Skip to content

Commit

Permalink
Generate end-to-end coverage reports for tests
Browse files Browse the repository at this point in the history
Coverage reports can be generated via:

  bazel run //bazel/coverage

Due to bazelbuild/bazel#15166, coverage
collection does not yet encompass tests that end with a native sanitizer
report.

Temporarily updates .bazelversion to an unreleased version of Bazel that
includes required coverage fixes.
  • Loading branch information
fmeum committed May 20, 2022
1 parent c950a1e commit d6c0adf
Show file tree
Hide file tree
Showing 10 changed files with 83 additions and 6 deletions.
12 changes: 12 additions & 0 deletions .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,15 @@ build:maven --config=toolchain
build:maven --stamp
build:maven --define "maven_repo=https://oss.sonatype.org/service/local/staging/deploy/maven2"
build:maven --java_runtime_version=local_jdk_8

# Generic coverage configuration taken from https://github.com/fmeum/rules_jni
coverage --combined_report=lcov
coverage --experimental_use_llvm_covmap
coverage --experimental_generate_llvm_lcov
coverage --repo_env=CC=clang
coverage --repo_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1
coverage --repo_env=GCOV=llvm-profdata

# Instrument all source files of non-test targets matching at least one of these regexes.
coverage --instrumentation_filter=^//agent/src/main[:/],^//driver:,^//sanitizers/src/main[:/]
coverage --test_tag_filters=-no-coverage
2 changes: 1 addition & 1 deletion .bazelversion
Original file line number Diff line number Diff line change
@@ -1 +1 @@
5.1.0
5.2.0rc1
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/bazel-*
.ijwb
.clwb
/coverage
10 changes: 9 additions & 1 deletion WORKSPACE.bazel
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
workspace(name = "jazzer")

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_jar")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file", "http_jar")
load("//:repositories.bzl", "jazzer_dependencies")

jazzer_dependencies()
Expand Down Expand Up @@ -116,3 +116,11 @@ maven_install(
load("@maven//:defs.bzl", "pinned_maven_install")

pinned_maven_install()

http_file(
name = "genhtml",
downloaded_file_path = "genhtml",
executable = True,
sha256 = "4260f8d032743968d208afccfdb77fb6fda85b0ea4063a1f0b6a0c5602d22347",
urls = ["https://raw.githubusercontent.com/linux-test-project/lcov/master/bin/genhtml"],
)
5 changes: 5 additions & 0 deletions agent/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ sh_test(
":jazzer_agent_deploy",
"@local_jdk//:bin/jar",
],
tags = [
# Coverage instrumentation necessarily adds files to the jar that we
# wouldn't want to release and thus causes this test to fail.
"no-coverage",
],
target_compatible_with = SKIP_ON_WINDOWS,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,20 @@

package com.code_intelligence.jazzer.utils

import java.lang.IllegalArgumentException

private val BASE_INCLUDED_CLASS_NAME_GLOBS = listOf(
"**", // everything
)

// We use both a strong indicator for running as a Bazel test together with an indicator for a
// Bazel coverage run to rule out false positives.
private val IS_BAZEL_COVERAGE_RUN = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR") != null &&
System.getenv("COVERAGE_DIR") != null

private val ADDITIONAL_EXCLUDED_NAME_GLOBS_FOR_BAZEL_COVERAGE = listOf(
"com.google.testing.coverage.**",
"org.jacoco.**",
)

private val BASE_EXCLUDED_CLASS_NAME_GLOBS = listOf(
// JDK internals
"\\[**", // array types
Expand All @@ -36,7 +44,7 @@ private val BASE_EXCLUDED_CLASS_NAME_GLOBS = listOf(
"com.code_intelligence.jazzer.**",
"jaz.Ter", // safe companion of the honeypot class used by sanitizers
"jaz.Zer", // honeypot class used by sanitizers
)
) + if (IS_BAZEL_COVERAGE_RUN) ADDITIONAL_EXCLUDED_NAME_GLOBS_FOR_BAZEL_COVERAGE else listOf()

class ClassNameGlobber(includes: List<String>, excludes: List<String>) {
// If no include globs are provided, start with all classes.
Expand Down
10 changes: 10 additions & 0 deletions bazel/coverage/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Run this target to generate and open an HTML coverage report.
# Takes the same arguments as `bazel coverage`, but after a double dash (`--`).
# The default is to run `bazel coverage //...`, which accumulates the coverage of all tests.
sh_binary(
name = "coverage",
srcs = ["coverage.sh"],
data = [
"@genhtml//file:genhtml",
],
)
30 changes: 30 additions & 0 deletions bazel/coverage/coverage.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/usr/bin/env sh
# Copyright 2022 Code Intelligence GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# Use just like `bazel test` to generate and open an HTML coverage report.
# Requires a local installation of Perl.

RUNFILES_ROOT=$PWD
cd "$BUILD_WORKSPACE_DIRECTORY" || exit 1
if ! bazel coverage "${@:-//...}";
then
exit $?
fi
"$RUNFILES_ROOT"/../genhtml/file/genhtml -o coverage \
--prefix "$PWD" \
--title "bazel coverage ${*:-//...}" \
bazel-out/_coverage/_coverage_report.dat
xdg-open coverage/index.html > /dev/null 2>&1
4 changes: 3 additions & 1 deletion driver/libfuzzer_fuzz_target.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, const size_t size) {
// Exit directly without invoking libFuzzer's atexit hook.
driver_cleanup();
// When running with LLVM coverage instrumentation, write out the profile as
// the exit hook that write it won't run.
// the exit hook that writes it won't run.
// TODO: Remove once https://github.com/bazelbuild/bazel/pull/15166 has been
// fixed and use continuous mode instead.
__llvm_profile_write_file();
_Exit(Driver::kErrorExitCode);
}
Expand Down
1 change: 1 addition & 0 deletions third_party/jacoco_internal.BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ java_library(
]),
javacopts = [
"-Xep:EqualsHashCode:OFF",
"-Xep:ReturnValueIgnored:OFF",
],
deps = [
"@org_ow2_asm_asm//jar",
Expand Down

0 comments on commit d6c0adf

Please sign in to comment.