fuzz: H1 capture fuzz test performance improvements. (envoyproxy#10281)

The main contribution in this patch is a "persistent" mode for
h1_capture_[direct_response_]fuzz_test. Based on profiling observations, we were spending 30-40% of
time rebuilding the Envoy server on each run. This is avoided by having fuzzer variants that makes
the integration test proxy static.

There is a downside of this approach, since different fuzz invocations may interfere with each
other. Ideally we would snapshot/fork for each fuzz invocation, but Envoy doesn't like forking once
events/dispatchers are up. So, for now we have two builds of the fuzzer, where we trade fuzz engine
efficacy for fuzz target performance. Some form of VM snapshotting would be ideal.

The persistent mode takes the H1 replay tests to O(10 exec/s) from O(1 exec/s). This is still not
great. Doing some perf analysis, it seems that we're spending the bulk of time in ASAN. Running
the fuzzers without ASAN gives O(100 exec/s), which seems reasonable for a LPM-style integration test.
It's future work why ASAN is so expensive, ASAN advertises itself as generally a 2x slowdown. There
is also some secondary effect from the cost of mocks used in the integration test TCP client (mock
watermark buffer), this speaks to our general mocking performance problem in fuzzing.

In addition to the above, this patch has an optimization for the direct response fuzzer (don't
initiate upstream connections) and a --config=plain-fuzz mode for peformance work without
confounding ASAN.

Risk level: Low
Testing: Manual bazel runs of the fuzzers, observing exec/s.

Signed-off-by: Harvey Tuch <[email protected]>

Author: htuch

.bazelrc

@@ -190,8 +190,14 @@ build:asan-fuzzer --config=clang-asan
 build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer
 build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
 build:asan-fuzzer --copt=-fsanitize=fuzzer-no-link
+build:asan-fuzzer --copt=-fno-omit-frame-pointer
 # Remove UBSAN halt_on_error to avoid crashing on protobuf errors.
 build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1
 
+# Fuzzing without ASAN. This is useful for profiling fuzzers without any ASAN artifacts.
+build:plain-fuzzer --define=FUZZING_ENGINE=libfuzzer
+build:plain-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+build:plain-fuzzer --copt=-fsanitize=fuzzer-no-link
+
 try-import %workspace%/clang.bazelrc
 try-import %workspace%/user.bazelrc

test/fuzz/fuzz_runner.h

@@ -62,6 +62,12 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv);
 // See https://llvm.org/docs/LibFuzzer.html#fuzz-target.
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size);
 
+#ifdef PERSISTENT_FUZZER
+#define PERSISTENT_FUZZ_VAR static
+#else
+#define PERSISTENT_FUZZ_VAR
+#endif
+
 #define DEFINE_TEST_ONE_INPUT_IMPL                                                                 \
   extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {                        \
     EnvoyTestOneInput(data, size);                                                                 \

test/integration/BUILD

@@ -975,19 +975,29 @@ envoy_cc_test(
     ],
 )
 
+H1_FUZZ_LIB_DEPS = [
+    ":capture_fuzz_proto_cc_proto",
+    ":http_integration_lib",
+    "//source/common/common:assert_lib",
+    "//source/common/common:logger_lib",
+    "//test/fuzz:fuzz_runner_lib",
+    "//test/integration:integration_lib",
+    "//test/test_common:environment_lib",
+]
+
 envoy_cc_test_library(
     name = "h1_fuzz_lib",
     srcs = ["h1_fuzz.cc"],
     hdrs = ["h1_fuzz.h"],
-    deps = [
-        ":capture_fuzz_proto_cc_proto",
-        ":http_integration_lib",
-        "//source/common/common:assert_lib",
-        "//source/common/common:logger_lib",
-        "//test/fuzz:fuzz_runner_lib",
-        "//test/integration:integration_lib",
-        "//test/test_common:environment_lib",
-    ],
+    deps = H1_FUZZ_LIB_DEPS,
+)
+
+envoy_cc_test_library(
+    name = "h1_fuzz_persistent_lib",
+    srcs = ["h1_fuzz.cc"],
+    hdrs = ["h1_fuzz.h"],
+    copts = ["-DPERSISTENT_FUZZER"],
+    deps = H1_FUZZ_LIB_DEPS,
 )
 
 envoy_cc_fuzz_test(
@@ -997,6 +1007,14 @@ envoy_cc_fuzz_test(
     deps = [":h1_fuzz_lib"],
 )
 
+envoy_cc_fuzz_test(
+    name = "h1_capture_persistent_fuzz_test",
+    srcs = ["h1_capture_fuzz_test.cc"],
+    copts = ["-DPERSISTENT_FUZZER"],
+    corpus = "h1_corpus",
+    deps = [":h1_fuzz_persistent_lib"],
+)
+
 envoy_cc_fuzz_test(
     name = "h1_capture_direct_response_fuzz_test",
     srcs = ["h1_capture_direct_response_fuzz_test.cc"],
@@ -1007,6 +1025,17 @@ envoy_cc_fuzz_test(
     ],
 )
 
+envoy_cc_fuzz_test(
+    name = "h1_capture_direct_response_persistent_fuzz_test",
+    srcs = ["h1_capture_direct_response_fuzz_test.cc"],
+    copts = ["-DPERSISTENT_FUZZER"],
+    corpus = "h1_corpus",
+    deps = [
+        ":h1_fuzz_persistent_lib",
+        "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto",
+    ],
+)
+
 envoy_cc_test(
     name = "scoped_rds_integration_test",
     srcs = [

test/integration/h1_capture_direct_response_fuzz_test.cc

@@ -31,8 +31,8 @@ void H1FuzzIntegrationTest::initialize() {
 DEFINE_PROTO_FUZZER(const test::integration::CaptureFuzzTestCase& input) {
   RELEASE_ASSERT(!TestEnvironment::getIpVersionsForTest().empty(), "");
   const auto ip_version = TestEnvironment::getIpVersionsForTest()[0];
-  H1FuzzIntegrationTest h1_fuzz_integration_test(ip_version);
-  h1_fuzz_integration_test.replay(input);
+  PERSISTENT_FUZZ_VAR H1FuzzIntegrationTest h1_fuzz_integration_test(ip_version);
+  h1_fuzz_integration_test.replay(input, true);
 }
 
 } // namespace Envoy

test/integration/h1_capture_fuzz_test.cc

@@ -7,8 +7,8 @@ DEFINE_PROTO_FUZZER(const test::integration::CaptureFuzzTestCase& input) {
   // Pick an IP version to use for loopback, it doesn't matter which.
   RELEASE_ASSERT(!TestEnvironment::getIpVersionsForTest().empty(), "");
   const auto ip_version = TestEnvironment::getIpVersionsForTest()[0];
-  H1FuzzIntegrationTest h1_fuzz_integration_test(ip_version);
-  h1_fuzz_integration_test.replay(input);
+  PERSISTENT_FUZZ_VAR H1FuzzIntegrationTest h1_fuzz_integration_test(ip_version);
+  h1_fuzz_integration_test.replay(input, false);
 }
 
 } // namespace Envoy

test/integration/h1_fuzz.cc

@@ -10,9 +10,14 @@
 
 namespace Envoy {
 
-void H1FuzzIntegrationTest::replay(const test::integration::CaptureFuzzTestCase& input) {
-  initialize();
-  fake_upstreams_[0]->set_allow_unexpected_disconnects(true);
+void H1FuzzIntegrationTest::replay(const test::integration::CaptureFuzzTestCase& input,
+                                   bool ignore_response) {
+  PERSISTENT_FUZZ_VAR bool initialized = [this]() -> bool {
+    initialize();
+    fake_upstreams_[0]->set_allow_unexpected_disconnects(true);
+    return true;
+  }();
+  UNREFERENCED_PARAMETER(initialized);
   IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("http"));
   FakeRawConnectionPtr fake_upstream_connection;
   for (int i = 0; i < input.events().size(); ++i) {
@@ -31,6 +36,9 @@ void H1FuzzIntegrationTest::replay(const test::integration::CaptureFuzzTestCase&
       // TODO(htuch): Should we wait for some data?
       break;
     case test::integration::Event::kUpstreamSendBytes:
+      if (ignore_response) {
+        break;
+      }
       if (fake_upstream_connection == nullptr) {
         if (!fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection, max_wait_ms_)) {
           // If we timed out, we fail out.

test/integration/h1_fuzz.h

@@ -15,7 +15,8 @@ class H1FuzzIntegrationTest : public HttpIntegrationTest {
       : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, version) {}
 
   void initialize() override;
-  void replay(const test::integration::CaptureFuzzTestCase&);
+  void replay(const test::integration::CaptureFuzzTestCase&, bool ignore_response);
   const std::chrono::milliseconds max_wait_ms_{10};
 };
+
 } // namespace Envoy