From 862f22a9acc16d9a1507a3186fdc9d49595fdc31 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Wed, 20 Apr 2022 16:29:57 +0200 Subject: [PATCH] Generate end-to-end coverage reports for tests Coverage reports can be generated via: bazel run //bazel/coverage Due to https://github.com/bazelbuild/bazel/pull/15166, coverage collection does not yet encompass tests that end with a native sanitizer report. Temporarily updates .bazelversion to an unreleased version of Bazel that includes required coverage fixes. --- .bazelrc | 12 ++++++++ .bazelversion | 2 +- .gitignore | 1 + WORKSPACE.bazel | 10 ++++++- agent/BUILD.bazel | 5 ++++ .../jazzer/utils/ClassNameGlobber.kt | 14 +++++++-- bazel/coverage/BUILD.bazel | 10 +++++++ bazel/coverage/coverage.sh | 30 +++++++++++++++++++ driver/libfuzzer_fuzz_target.cpp | 4 ++- 9 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 bazel/coverage/BUILD.bazel create mode 100755 bazel/coverage/coverage.sh diff --git a/.bazelrc b/.bazelrc index dd3094b92..2484b4190 100644 --- a/.bazelrc +++ b/.bazelrc @@ -50,3 +50,15 @@ build:maven --config=toolchain build:maven --stamp build:maven --define "maven_repo=https://oss.sonatype.org/service/local/staging/deploy/maven2" build:maven --java_runtime_version=local_jdk_8 + +# Generic coverage configuration taken from https://github.com/fmeum/rules_jni +coverage --combined_report=lcov +coverage --experimental_use_llvm_covmap +coverage --experimental_generate_llvm_lcov +coverage --repo_env=CC=clang +coverage --repo_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1 +coverage --repo_env=GCOV=llvm-profdata + +# Instrument all source files of non-test targets matching at least one of these regexes. +coverage --instrumentation_filter=^//agent/src/main[:/],^//driver:,^//sanitizers/src/main[:/] +coverage --test_tag_filters=-no-coverage diff --git a/.bazelversion b/.bazelversion index 831446cbd..7e9522d63 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -5.1.0 +93029d8f1f01be440d8e06cce9585ad912f9169b diff --git a/.gitignore b/.gitignore index e992903a7..b0ac4affd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /bazel-* .ijwb .clwb +/coverage diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 1f55a2e8f..f900cd9f7 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -1,6 +1,6 @@ workspace(name = "jazzer") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_jar") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file", "http_jar") load("//:repositories.bzl", "jazzer_dependencies") jazzer_dependencies() @@ -116,3 +116,11 @@ maven_install( load("@maven//:defs.bzl", "pinned_maven_install") pinned_maven_install() + +http_file( + name = "genhtml", + downloaded_file_path = "genhtml", + executable = True, + sha256 = "4260f8d032743968d208afccfdb77fb6fda85b0ea4063a1f0b6a0c5602d22347", + urls = ["https://raw.githubusercontent.com/linux-test-project/lcov/master/bin/genhtml"], +) diff --git a/agent/BUILD.bazel b/agent/BUILD.bazel index 7e10b5a39..1737adcdc 100644 --- a/agent/BUILD.bazel +++ b/agent/BUILD.bazel @@ -43,6 +43,11 @@ sh_test( ":jazzer_agent_deploy", "@local_jdk//:bin/jar", ], + tags = [ + # Coverage instrumentation necessarily adds files to the jar that we + # wouldn't want to release and thus causes this test to fail. + "no-coverage", + ], target_compatible_with = SKIP_ON_WINDOWS, ) diff --git a/agent/src/main/java/com/code_intelligence/jazzer/utils/ClassNameGlobber.kt b/agent/src/main/java/com/code_intelligence/jazzer/utils/ClassNameGlobber.kt index 763f230d4..44249c811 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/utils/ClassNameGlobber.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/utils/ClassNameGlobber.kt @@ -14,12 +14,20 @@ package com.code_intelligence.jazzer.utils -import java.lang.IllegalArgumentException - private val BASE_INCLUDED_CLASS_NAME_GLOBS = listOf( "**", // everything ) +// We use both a strong indicator for running as a Bazel test together with an indicator for a +// Bazel coverage run to rule out false positives. +private val IS_BAZEL_COVERAGE_RUN = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR") != null && + System.getenv("COVERAGE_DIR") != null + +private val ADDITIONAL_EXCLUDED_NAME_GLOBS_FOR_BAZEL_COVERAGE = listOf( + "com.google.testing.coverage.**", + "org.jacoco.**", +) + private val BASE_EXCLUDED_CLASS_NAME_GLOBS = listOf( // JDK internals "\\[**", // array types @@ -36,7 +44,7 @@ private val BASE_EXCLUDED_CLASS_NAME_GLOBS = listOf( "com.code_intelligence.jazzer.**", "jaz.Ter", // safe companion of the honeypot class used by sanitizers "jaz.Zer", // honeypot class used by sanitizers -) +) + if (IS_BAZEL_COVERAGE_RUN) ADDITIONAL_EXCLUDED_NAME_GLOBS_FOR_BAZEL_COVERAGE else listOf() class ClassNameGlobber(includes: List, excludes: List) { // If no include globs are provided, start with all classes. diff --git a/bazel/coverage/BUILD.bazel b/bazel/coverage/BUILD.bazel new file mode 100644 index 000000000..b3dc98616 --- /dev/null +++ b/bazel/coverage/BUILD.bazel @@ -0,0 +1,10 @@ +# Run this target to generate and open an HTML coverage report. +# Takes the same arguments as `bazel coverage`, but after a double dash (`--`). +# The default is to run `bazel coverage //...`, which accumulates the coverage of all tests. +sh_binary( + name = "coverage", + srcs = ["coverage.sh"], + data = [ + "@genhtml//file:genhtml", + ], +) diff --git a/bazel/coverage/coverage.sh b/bazel/coverage/coverage.sh new file mode 100755 index 000000000..626fdc706 --- /dev/null +++ b/bazel/coverage/coverage.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env sh +# Copyright 2022 Code Intelligence GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Use just like `bazel test` to generate and open an HTML coverage report. +# Requires a local installation of Perl. + +RUNFILES_ROOT=$PWD +cd "$BUILD_WORKSPACE_DIRECTORY" || exit 1 +if ! bazel coverage "${@:-//...}"; +then + exit $? +fi +"$RUNFILES_ROOT"/../genhtml/file/genhtml -o coverage \ + --prefix "$PWD" \ + --title "bazel coverage ${*:-//...}" \ + bazel-out/_coverage/_coverage_report.dat +xdg-open coverage/index.html > /dev/null 2>&1 diff --git a/driver/libfuzzer_fuzz_target.cpp b/driver/libfuzzer_fuzz_target.cpp index 9c2c6ecb4..1f3a5c311 100644 --- a/driver/libfuzzer_fuzz_target.cpp +++ b/driver/libfuzzer_fuzz_target.cpp @@ -112,7 +112,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, const size_t size) { // Exit directly without invoking libFuzzer's atexit hook. driver_cleanup(); // When running with LLVM coverage instrumentation, write out the profile as - // the exit hook that write it won't run. + // the exit hook that writes it won't run. + // TODO: Remove once https://github.com/bazelbuild/bazel/pull/15166 has been + // fixed and use continuous mode instead. __llvm_profile_write_file(); _Exit(Driver::kErrorExitCode); }