Carbon fuzzing 3/3: added actual fuzzer implementation and a fuzzvert…

…er utility for investigating crashing protos (#1156) * finished fuzzer and added fuzzverter util * fixed typo * renamed cmd line params * fixed libproto_mutator download path * small fixes * small fixes * small fixes * renamed sample corpus proto * small fixes * try building on github with LIBCPP_DEBUG enabled * temporarily marked proto fuzzer as a manual test * code review * use a dedicated proto-fuzzer feature to work around LIBCPP_DEBUG=1 crash in proto code * code review comments, added README.md * minor fixes to the text * Update bazel/cc_toolchains/clang_cc_toolchain_config.bzl Co-authored-by: Jon Meow <[email protected]> * use Carbon source representation for "empty Main()" instead of text format proto representation * fixed typo * made FuzzerUtil produce the full carbon source (proto converted + Main if needed) to decrease code duplication a bit * typo * switched to text proto format per code review * Update executable_semantics/prelude.h Co-authored-by: Jon Meow <[email protected]> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <[email protected]> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <[email protected]> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <[email protected]> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <[email protected]> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <[email protected]> * review comments * removed unnecessary file mode variables * Update executable_semantics/fuzzing/fuzzverter.cpp Co-authored-by: Jon Meow <[email protected]> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <[email protected]> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <[email protected]> * code review comments * Update executable_semantics/syntax/BUILD Co-authored-by: Jon Meow <[email protected]> * buildifier Co-authored-by: Jon Meow <[email protected]>
carbon-language · Apr 11, 2022 · 22462a0 · 22462a0
1 parent db0de10
commit 22462a0
Show file tree

Hide file tree

Showing 18 changed files with 497 additions and 20 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -56,6 +56,9 @@ build:asan --features=asan
 # Configuration for enabling LibFuzzer (along with ASan).
 build:fuzzer --features=fuzzer
 
+# Proto fuzzer specific configuration.
+build:proto-fuzzer --features=proto-fuzzer
+
 # Always allow tests to symbolize themselves with whatever `llvm-symbolize` is
 # in the users environment.
 build --test_env=ASAN_SYMBOLIZER_PATH

diff --git a/WORKSPACE b/WORKSPACE
@@ -248,6 +248,20 @@ rules_proto_dependencies()
 
 rules_proto_toolchains()
 
+###############################################################################
+# libprotobuf_mutator - for structured fuzzer testing.
+###############################################################################
+
+libprotobuf_mutator_version = "1.0"
+
+http_archive(
+    name = "com_google_libprotobuf_mutator",
+    build_file = "@//:third_party/libprotobuf_mutator/BUILD.txt",
+    sha256 = "792f250fb546bde8590e72d64311ea00a70c175fd77df6bb5e02328fa15fe28e",
+    strip_prefix = "libprotobuf-mutator-%s" % libprotobuf_mutator_version,
+    urls = ["https://github.com/google/libprotobuf-mutator/archive/v%s.tar.gz" % libprotobuf_mutator_version],
+)
+
 ###############################################################################
 # Example conversion repositories
 ###############################################################################

diff --git a/bazel/cc_toolchains/clang_cc_toolchain_config.bzl b/bazel/cc_toolchains/clang_cc_toolchain_config.bzl
@@ -482,6 +482,17 @@ def _impl(ctx):
         )],
     )
 
+    proto_fuzzer = feature(
+        name = "proto-fuzzer",
+        enabled = False,
+        requires = [feature_set(["nonhost"])],
+
+        # TODO: this should really be `fuzzer`, but `-fsanitize=fuzzer` triggers
+        # a clang crash when running `bazel test --config=fuzzer ...`. See
+        # https://github.com/carbon-language/carbon-lang/issues/1173
+        implies = ["asan"],
+    )
+
     linux_flags_feature = feature(
         name = "linux_flags",
         enabled = True,
@@ -528,7 +539,12 @@ def _impl(ctx):
                     "-D_LIBCPP_DEBUG=1",
                 ])],
                 with_features = [
-                    with_feature_set(not_features = ["opt"]),
+                    # _LIBCPP_DEBUG=1 causes protobuf code to crash when linked
+                    # with `-fsanitize=fuzzer`, possibly because of ODR
+                    # violations caused by Carbon source and pre-compiled llvm
+                    # Fuzzer driver library built with different _LIBCPP_DEBUG
+                    # values.
+                    with_feature_set(not_features = ["opt", "proto-fuzzer"]),
                 ],
             ),
         ],
@@ -751,6 +767,7 @@ def _impl(ctx):
         asan,
         enable_asan_in_fastbuild,
         fuzzer,
+        proto_fuzzer,
         layering_check,
         module_maps,
         use_module_maps,

diff --git a/executable_semantics/BUILD b/executable_semantics/BUILD
@@ -17,13 +17,13 @@ filegroup(
 cc_binary(
     name = "executable_semantics",
     srcs = ["main.cpp"],
-    data = [":standard_libraries"],
     deps = [
         "//common:error",
         "//executable_semantics/common:arena",
         "//executable_semantics/common:nonnull",
         "//executable_semantics/interpreter:exec_program",
         "//executable_semantics/syntax",
+        "//executable_semantics/syntax:prelude",
         "@llvm-project//llvm:Support",
     ],
 )

diff --git a/executable_semantics/fuzzing/BUILD b/executable_semantics/fuzzing/BUILD
@@ -2,6 +2,8 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+load("//bazel/fuzzing:rules.bzl", "cc_fuzz_test")
+
 cc_library(
     name = "ast_to_proto_lib",
     srcs = ["ast_to_proto.cpp"],
@@ -15,6 +17,18 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "fuzzer_util",
+    srcs = ["fuzzer_util.cpp"],
+    hdrs = ["fuzzer_util.h"],
+    deps = [
+        "//common:check",
+        "//common/fuzzing:carbon_cc_proto",
+        "//common/fuzzing:proto_to_carbon_lib",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
 cc_test(
     name = "ast_to_proto_test",
     srcs = ["ast_to_proto_test.cpp"],
@@ -36,6 +50,22 @@ cc_test(
     ],
 )
 
+cc_binary(
+    name = "fuzzverter",
+    srcs = ["fuzzverter.cpp"],
+    deps = [
+        ":ast_to_proto_lib",
+        ":fuzzer_util",
+        "//common:error",
+        "//common/fuzzing:carbon_cc_proto",
+        "//executable_semantics/common:error",
+        "//executable_semantics/common:nonnull",
+        "//executable_semantics/syntax",
+        "@com_google_protobuf//:protobuf_headers",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
 cc_test(
     name = "proto_to_carbon_test",
     srcs = ["proto_to_carbon_test.cpp"],
@@ -57,3 +87,22 @@ cc_test(
         "@llvm-project//llvm:Support",
     ],
 )
+
+# Needs `--config=proto-fuzzer` for `bazel build` / `bazel test`.
+cc_fuzz_test(
+    name = "executable_semantics_fuzzer",
+    size = "small",
+    srcs = ["executable_semantics_fuzzer.cpp"],
+    corpus = glob(["fuzzer_corpus/**"]),
+    tags = ["manual"],
+    deps = [
+        ":fuzzer_util",
+        "//common/fuzzing:carbon_cc_proto",
+        "//executable_semantics/interpreter:exec_program",
+        "//executable_semantics/syntax",
+        "//executable_semantics/syntax:prelude",
+        "@com_google_libprotobuf_mutator//:libprotobuf_mutator",
+        "@com_google_protobuf//:protobuf_headers",
+        "@llvm-project//llvm:Support",
+    ],
+)
diff --git a/executable_semantics/fuzzing/README.md b/executable_semantics/fuzzing/README.md
@@ -0,0 +1,89 @@
+# Executable semantics structured fuzzer
+
+<!--
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+-->
+
+## Overview
+
+Fuzz testing is based on generating a large amount of random inputs for a
+software component in order to trigger bugs and unexpected behavior. Basic
+fuzzing uses randomly generated arrays of bytes as inputs, which works great for
+some applications but is problematic for testing the logic that operates on
+highly structured data, as most random inputs are immediately rejected as
+invalid before any interesting parts of the code get a chance to run.
+
+Structured fuzzing addresses this issue by ensuring the randomly generated data
+is itself structured, and as such has a high chance of presenting a valid input.
+
+`executable_semantics_fuzzer` is a structured fuzzer based on
+[libprotobuf-mutator](https://github.com/google/libprotobuf-mutator), which is a
+library to randomly mutate
+[protobuffers](https://github.com/protocolbuffers/protobuf).
+
+The input to the fuzzer is an instance of `Carbon::Fuzzing::Carbon` proto
+randomly generated by the `libprotobuf-mutator` framework.
+`executable_semantics_fuzzer` converts the proto to a Carbon source code string,
+and tries to parse and execute the code using `executable_semantics`
+implementation.
+
+## Fuzzer data format
+
+`libprotobuf-mutator` supports fuzzer inputs in either text or binary protocol
+buffer format. `executable_semantics_fuzzer` uses text proto format with
+`Carbon` proto message definition in `common/fuzzing/carbon.proto`.
+
+## Running the fuzzer
+
+The fuzzer can be run in 'unit test' mode, where the fuzzer executes on each
+input file from the `fuzzer_corpus/` folder, or in 'fuzzing' mode, where the
+fuzzer will keep generating random inputs and executing the logic on them until
+a crash is triggered, or forever in a bug-free program ;).
+
+To run in 'unit test' mode:
+
+```bash
+bazel test --config=proto-fuzzer --test_output=all //executable_semantics/fuzzing:executable_semantics_fuzzer
+```
+
+To run in 'fuzzing' mode:
+
+```bash
+bazel build --config=proto-fuzzer //executable_semantics/fuzzing:executable_semantics_fuzzer
+
+bazel-bin/executable_semantics/fuzzing/executable_semantics_fuzzer
+```
+
+It's also possible to run the fuzzer on a single input:
+
+```bash
+bazel-bin/executable_semantics/fuzzing/executable_semantics_fuzzer /tmp/crash.textproto
+```
+
+## Investigating a crash
+
+To reproduce a crash, run the fuzzer on the crashing input as described above.
+
+A separate tool called `fuzzverter` can be used for things like converting a
+crashing input to Carbon source code for running `executable_semantics` on the
+code directly.
+
+To convert a `Fuzzing::Carbon` text proto to Carbon source:
+
+```bash
+bazel-bin/executable_semantics/fuzzing/fuzzverter --mode proto_to_carbon --input /tmp/crash.textproto
+```
+
+## Generating new fuzzer corpus entries
+
+The ability of the fuzzing framework to generate 'interesting' inputs can be
+improved by providing 'seed' inputs known as the fuzzer corpus. The inputs need
+to be a `Fuzzing::Carbon` text proto.
+
+To generate a text proto from Carbon source:
+
+```bash
+bazel-bin/executable_semantics/fuzzing/fuzzverter --mode carbon_to_proto --input /tmp/crash.carbon --output /tmp/crash.textproto
+```
diff --git a/executable_semantics/fuzzing/executable_semantics_fuzzer.cpp b/executable_semantics/fuzzing/executable_semantics_fuzzer.cpp
@@ -0,0 +1,42 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <google/protobuf/text_format.h>
+#include <libprotobuf_mutator/src/libfuzzer/libfuzzer_macro.h>
+
+#include "common/fuzzing/carbon.pb.h"
+#include "executable_semantics/fuzzing/fuzzer_util.h"
+#include "executable_semantics/interpreter/exec_program.h"
+#include "executable_semantics/syntax/parse.h"
+#include "executable_semantics/syntax/prelude.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace Carbon {
+
+// Parses and executes a fuzzer-generated program.
+void ParseAndExecute(const Fuzzing::CompilationUnit& compilation_unit) {
+  const std::string source = ProtoToCarbonWithMain(compilation_unit);
+
+  Arena arena;
+  ErrorOr<AST> ast = ParseFromString(&arena, "Fuzzer.carbon", source,
+                                     /*trace=*/false);
+  if (!ast.ok()) {
+    llvm::errs() << "Parsing failed: " << ast.error().message() << "\n";
+    return;
+  }
+  AddPrelude("executable_semantics/data/prelude.carbon", &arena,
+             &ast->declarations);
+  const ErrorOr<int> result = ExecProgram(&arena, *ast, /*trace=*/false);
+  if (!result.ok()) {
+    llvm::errs() << "Execution failed: " << result.error().message() << "\n";
+    return;
+  }
+  llvm::outs() << "Executed OK: " << *result << "\n";
+}
+
+}  // namespace Carbon
+
+DEFINE_TEXT_PROTO_FUZZER(const Carbon::Fuzzing::Carbon& input) {
+  Carbon::ParseAndExecute(input.compilation_unit());
+}
diff --git a/executable_semantics/fuzzing/fuzzer_corpus/empty.textproto b/executable_semantics/fuzzing/fuzzer_corpus/empty.textproto
diff --git a/executable_semantics/fuzzing/fuzzer_util.cpp b/executable_semantics/fuzzing/fuzzer_util.cpp
@@ -0,0 +1,32 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "executable_semantics/fuzzing/fuzzer_util.h"
+
+#include "common/check.h"
+#include "common/fuzzing/proto_to_carbon.h"
+
+namespace Carbon {
+
+// Appended to fuzzer-generated Carbon source when the source is missing
+// `Main()` definition, to prevent early error return in semantic analysis.
+static constexpr char EmptyMain[] = R"(
+fn Main() -> i32 {
+  return 0;
+}
+)";
+
+auto ProtoToCarbonWithMain(const Fuzzing::CompilationUnit& compilation_unit)
+    -> std::string {
+  const bool has_main = std::any_of(
+      compilation_unit.declarations().begin(),
+      compilation_unit.declarations().end(),
+      [](const Fuzzing::Declaration& decl) {
+        return decl.kind_case() == Fuzzing::Declaration::kFunction &&
+               decl.function().name() == "Main";
+      });
+  return Carbon::ProtoToCarbon(compilation_unit) + (has_main ? "" : EmptyMain);
+}
+
+}  // namespace Carbon
diff --git a/executable_semantics/fuzzing/fuzzer_util.h b/executable_semantics/fuzzing/fuzzer_util.h
@@ -0,0 +1,19 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef EXECUTABLE_SEMANTICS_FUZZING_FUZZER_UTIL_H_
+#define EXECUTABLE_SEMANTICS_FUZZING_FUZZER_UTIL_H_
+
+#include "common/fuzzing/carbon.pb.h"
+
+namespace Carbon {
+
+// Converts `compilation_unit` to Carbon. Adds an default `Main()`
+// definition if one is not present in the proto.
+auto ProtoToCarbonWithMain(const Fuzzing::CompilationUnit& compilation_unit)
+    -> std::string;
+
+}  // namespace Carbon
+
+#endif  // EXECUTABLE_SEMANTICS_FUZZING_FUZZER_UTIL_H_