From e2be560464ad971e7fe42d814e28e4423ada6067 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sat, 26 Jun 2021 05:44:59 -0400 Subject: [PATCH] Code coverage: fix the code coverage pipeline --- .buildkite/coverage-linux64/run_tests_base.jl | 3 +- .../coverage-linux64/upload_coverage.jl | 138 ++++++++++++++++-- 2 files changed, 124 insertions(+), 17 deletions(-) diff --git a/.buildkite/coverage-linux64/run_tests_base.jl b/.buildkite/coverage-linux64/run_tests_base.jl index 5386828aa4e142..8681d48b0ee9ed 100644 --- a/.buildkite/coverage-linux64/run_tests_base.jl +++ b/.buildkite/coverage-linux64/run_tests_base.jl @@ -30,7 +30,8 @@ const ncores = min(Sys.CPU_THREADS, Threads.nthreads()) @info "" ncores Sys.CPU_THREADS Threads.nthreads() try - Base.runtests(tests; ncores) + # Base.runtests(tests; ncores) # TODO: uncomment this line + Base.runtests(["compiler"]; ncores) # TODO: delete this line catch ex @error "" exception=(ex, catch_backtrace()) end diff --git a/.buildkite/coverage-linux64/upload_coverage.jl b/.buildkite/coverage-linux64/upload_coverage.jl index 04ddd9651861e6..04fd67286640d4 100644 --- a/.buildkite/coverage-linux64/upload_coverage.jl +++ b/.buildkite/coverage-linux64/upload_coverage.jl @@ -10,6 +10,24 @@ Pkg.precompile() import Coverage +function process_folders() + # `Coverage.process_folder` will have a LOT of `@info` statements that will make the log + # way too long. So before we run `Coverage.process_folder`, we disable logging for `@info` + # statements. After we run `Coverage.process_folder`, we re-enable logging for `@info` + # statements. + Logging.disable_logging(Logging.Info) + fcs_base = Coverage.process_folder("base") + fcs_stdlib = Coverage.process_folder("stdlib") + Logging.disable_logging(Logging.Debug) + + fcs = Coverage.merge_coverage_counts( + fcs_base, + fcs_stdlib, + ) + + return fcs +end + function get_external_stdlib_names(stdlib_dir::AbstractString) filename_list = filter(x -> isfile(joinpath(stdlib_dir, x)), readdir(stdlib_dir)) # find all of the files like `Pkg.version`, `Statistics.version`, etc. @@ -76,27 +94,103 @@ function print_coverage_summary( return nothing end -# `Coverage.process_folder` will have a LOT of `@info` statements that will make the log -# way too long. So before we run `Coverage.process_folder`, we disable logging for `@info` -# statements. After we run `Coverage.process_folder`, we re-enable logging for `@info` -# statements. -Logging.disable_logging(Logging.Info) -const fcs = Coverage.merge_coverage_counts( - Coverage.process_folder("base"), - Coverage.process_folder("stdlib"), -); -Logging.disable_logging(Logging.Debug) +function buildkite_env(name::String) + value = String(strip(ENV[name])) + if isempty(value) + throw(ErrorException("environment variable $(name) is empty")) + end + return value +end + +function buildkite_env(name_1::String, name_2::String, default::String) + value_1 = String(strip(ENV[name_1])) + value_2 = String(strip(ENV[name_2])) + !isempty(value_1) && return value_1 + !isempty(value_2) && return value_2 + return default +end + +function buildkite_branch_and_commit() + branch = buildkite_env("BUILDKITE_BRANCH") + commit = buildkite_env("BUILDKITE_COMMIT") + head_rev_parse = String(strip(read(`git rev-parse HEAD`, String))) + @info "" branch commit head_rev_parse + if strip(commit) == "HEAD" + commit = head_rev_parse + end + if commit !== head_rev_parse + msg = "mismatch" + @error msg commit head_rev_parse + throw(ErrorException(msg)) + end + if !occursin(r"^[a-f0-9]{40}$", commit) + msg = "BUILDKITE_COMMIT does not look like a long commit SHA" + @error msg commit + throw(ErrorException(msg)) + end + @info "" branch commit head_rev_parse + return (; branch, commit) +end + +function codecov_buildkite_add_local_to_kwargs() + branch, commit = buildkite_branch_and_commit() + kwargs = Coverage.Codecov.set_defaults( + Dict(); + branch, + commit, + ) + return kwargs +end + +function coveralls_buildkite_query_git_info() + branch, commit = buildkite_branch_and_commit() + remote_name = "origin" + remote = buildkite_env("BUILDKITE_REPO") + message = buildkite_env("BUILDKITE_MESSAGE") + author_name = buildkite_env( + "BUILDKITE_BUILD_AUTHOR", + "BUILDKITE_BUILD_CREATOR", + "", + ) + author_email = buildkite_env( + "BUILDKITE_BUILD_AUTHOR_EMAIL", + "BUILDKITE_BUILD_CREATOR_EMAIL", + "", + ) + remotes = [ + Dict( + "name" => remote_name, + "url" => remote, + ) + ] + head = Dict( + "id" => commit, + "author_name" => author_name, + "author_email" => author_email, + "committer_name" => author_name, + "committer_email" => author_email, + "message" => message, + ) + git_info = Dict( + "branch" => branch, + "remotes" => remotes + "head" => head, + ) + return git_info +end + +const fcs = process_folders() # Only include source code files. Exclude test files, benchmarking files, etc. filter!(fcs) do fc occursin(r"^base\/", fc.filename) || occursin("/src/", fc.filename) -end; +end # Exclude all external stdlibs (stdlibs that live in external repos). const external_stdlib_prefixes = get_external_stdlib_prefixes("stdlib") filter!(fcs) do fc all(x -> !startswith(fc.filename, x), external_stdlib_prefixes) -end; +end # Exclude all stdlib JLLs (stdlibs of the form `stdlib/*_jll/`). filter!(fcs) do fc @@ -108,8 +202,20 @@ sort!(fcs; by = fc -> fc.filename) print_coverage_summary.(fcs); print_coverage_summary(fcs, "Total") -# In order to upload to Codecov, you need to have the `CODECOV_TOKEN` environment variable defined. -Coverage.Codecov.submit_local(fcs) +let + kwargs = codecov_buildkite_add_local_to_kwargs() + @info "" kwargs -# In order to upload to Coveralls, you need to have the `COVERALLS_TOKEN` environment variable defined. -Coverage.Coveralls.submit_local(fcs) + # In order to upload to Codecov, you need to have the `CODECOV_TOKEN` environment variable defined. + Coverage.Codecov.submit_generic(fcs, kwargs) +end + +let + git_info = coveralls_buildkite_query_git_info() + @info "" git_info["branch"] + @info "" git_info["head"] + @info "" git_info["head"]["id"] + + # In order to upload to Coveralls, you need to have the `COVERALLS_TOKEN` environment variable defined. + Coverage.Coveralls.submit_local(fcs, git_info) +end