Generate end-to-end coverage reports for tests

Coverage reports can be generated via:

  bazel run //bazel/coverage

Due to bazelbuild/bazel#15166, coverage
collection does not yet encompass tests that end with a native sanitizer
report.

Temporarily updates .bazelversion to an unreleased version of Bazel that
includes required coverage fixes.

.bazelrc

@@ -50,3 +50,15 @@ build:maven --config=toolchain
 build:maven --stamp
 build:maven --define "maven_repo=https://oss.sonatype.org/service/local/staging/deploy/maven2"
 build:maven --java_runtime_version=local_jdk_8
+
+# Generic coverage configuration taken from https://github.com/fmeum/rules_jni
+coverage --combined_report=lcov
+coverage --experimental_use_llvm_covmap
+coverage --experimental_generate_llvm_lcov
+coverage --repo_env=CC=clang
+coverage --repo_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1
+coverage --repo_env=GCOV=llvm-profdata
+
+# Instrument all source files of non-test targets matching at least one of these regexes.
+coverage --instrumentation_filter=^//agent/src/main[:/],^//driver:,^//sanitizers/src/main[:/]
+coverage --test_tag_filters=-no-coverage

.bazelversion

@@ -1 +1 @@
-5.1.0
+5.2.0rc1

.gitignore

@@ -1,3 +1,4 @@
 /bazel-*
 .ijwb
 .clwb
+/coverage

WORKSPACE.bazel

@@ -1,6 +1,6 @@
 workspace(name = "jazzer")
 
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_jar")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file", "http_jar")
 load("//:repositories.bzl", "jazzer_dependencies")
 
 jazzer_dependencies()
@@ -116,3 +116,11 @@ maven_install(
 load("@maven//:defs.bzl", "pinned_maven_install")
 
 pinned_maven_install()
+
+http_file(
+    name = "genhtml",
+    downloaded_file_path = "genhtml",
+    executable = True,
+    sha256 = "4260f8d032743968d208afccfdb77fb6fda85b0ea4063a1f0b6a0c5602d22347",
+    urls = ["https://raw.githubusercontent.com/linux-test-project/lcov/master/bin/genhtml"],
+)

agent/BUILD.bazel

@@ -43,6 +43,11 @@ sh_test(
         ":jazzer_agent_deploy",
         "@local_jdk//:bin/jar",
     ],
+    tags = [
+        # Coverage instrumentation necessarily adds files to the jar that we
+        # wouldn't want to release and thus causes this test to fail.
+        "no-coverage",
+    ],
     target_compatible_with = SKIP_ON_WINDOWS,
 )
 

agent/src/main/java/com/code_intelligence/jazzer/utils/ClassNameGlobber.kt

@@ -14,12 +14,20 @@
 
 package com.code_intelligence.jazzer.utils
 
-import java.lang.IllegalArgumentException
-
 private val BASE_INCLUDED_CLASS_NAME_GLOBS = listOf(
     "**", // everything
 )
 
+// We use both a strong indicator for running as a Bazel test together with an indicator for a
+// Bazel coverage run to rule out false positives.
+private val IS_BAZEL_COVERAGE_RUN = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR") != null &&
+    System.getenv("COVERAGE_DIR") != null
+
+private val ADDITIONAL_EXCLUDED_NAME_GLOBS_FOR_BAZEL_COVERAGE = listOf(
+    "com.google.testing.coverage.**",
+    "org.jacoco.**",
+)
+
 private val BASE_EXCLUDED_CLASS_NAME_GLOBS = listOf(
     // JDK internals
     "\\[**", // array types
@@ -36,7 +44,7 @@ private val BASE_EXCLUDED_CLASS_NAME_GLOBS = listOf(
     "com.code_intelligence.jazzer.**",
     "jaz.Ter", // safe companion of the honeypot class used by sanitizers
     "jaz.Zer", // honeypot class used by sanitizers
-)
+) + if (IS_BAZEL_COVERAGE_RUN) ADDITIONAL_EXCLUDED_NAME_GLOBS_FOR_BAZEL_COVERAGE else listOf()
 
 class ClassNameGlobber(includes: List<String>, excludes: List<String>) {
     // If no include globs are provided, start with all classes.

bazel/coverage/BUILD.bazel

@@ -0,0 +1,10 @@
+# Run this target to generate and open an HTML coverage report.
+# Takes the same arguments as `bazel coverage`, but after a double dash (`--`).
+# The default is to run `bazel coverage //...`, which accumulates the coverage of all tests.
+sh_binary(
+    name = "coverage",
+    srcs = ["coverage.sh"],
+    data = [
+        "@genhtml//file:genhtml",
+    ],
+)

bazel/coverage/coverage.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env sh
+# Copyright 2022 Code Intelligence GmbH
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Use just like `bazel test` to generate and open an HTML coverage report.
+# Requires a local installation of Perl.
+
+RUNFILES_ROOT=$PWD
+cd "$BUILD_WORKSPACE_DIRECTORY" || exit 1
+if ! bazel coverage "${@:-//...}";
+then
+  exit $?
+fi
+"$RUNFILES_ROOT"/../genhtml/file/genhtml -o coverage \
+  --prefix "$PWD" \
+  --title "bazel coverage ${*:-//...}" \
+  bazel-out/_coverage/_coverage_report.dat
+xdg-open coverage/index.html > /dev/null 2>&1

driver/libfuzzer_fuzz_target.cpp

@@ -112,7 +112,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, const size_t size) {
     // Exit directly without invoking libFuzzer's atexit hook.
     driver_cleanup();
     // When running with LLVM coverage instrumentation, write out the profile as
-    // the exit hook that write it won't run.
+    // the exit hook that writes it won't run.
+    // TODO: Remove once https://github.com/bazelbuild/bazel/pull/15166 has been
+    //  fixed and use continuous mode instead.
     __llvm_profile_write_file();
     _Exit(Driver::kErrorExitCode);
   }