Make the fuzzing CI job faster (bytecodealliance#1727)

* CI: Only build fuzz targets, don't run them over the corpora

We've only ever caught a single potential regression by running the fuzz targets
over a sample of their corpora. However, this is also our slowest CI
job. Running the fuzz targets over their corpora simply isn't paying for itself.

Instead, just ensure that we can build the fuzz targets with `cargo fuzz` and
all of the libFuzzer and sanitizer instrumentation that it enables. This will
ensure that we don't break the fuzz targets, and we leave finding regressions in
the fuzz corpora to oss-fuzz.

* fuzz: feature gate peepmatic's fuzz targets

This makes it so that the CI's fuzz target-building job doesn't build peepmatic,
and transitively Z3.

Author: fitzgen

.github/workflows/main.yml

@@ -108,95 +108,20 @@ jobs:
     - run: cargo check --target armv7-unknown-linux-gnueabihf -p wasi-common
 
 
-  # Download our libFuzzer corpus and make sure that we can still handle all the
-  # inputs.
-  fuzz_corpora:
-    name: Fuzz Corpora
+  fuzz_targets:
+    name: Fuzz Targets
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
       with:
         submodules: true
-    - uses: actions/checkout@v1
-      with:
-        repository: bytecodealliance/wasmtime-libfuzzer-corpus
-        path: ./wasmtime/fuzz/corpus
-        ref: refs/heads/master
     - uses: ./.github/actions/install-rust
       with:
         toolchain: nightly
     - run: cargo install cargo-fuzz --vers "^0.7"
     - run: cargo fetch
       working-directory: ./fuzz
-    - run: cargo fuzz build --release --debug-assertions --features binaryen
-    # Our corpora are too large to run in full on every pull request, they just
-    # take too long. Instead, we sample some of them and make sure that running
-    # our fuzzers over the sampled inputs still works OK.
-    - run: |
-        find fuzz/corpus/compile -type f \
-            | shuf -n 3000 \
-            | xargs cargo fuzz run compile --release --debug-assertions --features binaryen
-      env:
-        RUST_BACKTRACE: 1
-    - run: |
-        find fuzz/corpus/instantiate -type f \
-            | shuf -n 2000 \
-            | xargs cargo fuzz run instantiate --release --debug-assertions --features binaryen
-      env:
-        RUST_BACKTRACE: 1
-    - run: |
-        find fuzz/corpus/instantiate_translated -type f \
-            | shuf -n 1000 \
-            | xargs cargo fuzz run instantiate_translated --release --debug-assertions --features binaryen
-      env:
-        RUST_BACKTRACE: 1
-    - run: |
-        find fuzz/corpus/api_calls -type f \
-            | shuf -n 100 \
-            | xargs cargo fuzz run api_calls --release --debug-assertions --features binaryen
-      env:
-        RUST_BACKTRACE: 1
-    - run: |
-        find fuzz/corpus/differential -type f \
-            | shuf -n 100 \
-            | xargs cargo fuzz run differential --release --debug-assertions --features binaryen
-      env:
-        RUST_BACKTRACE: 1
-    - run: |
-        find fuzz/corpus/peepmatic_compile -type f \
-            | shuf \
-            | head -n 10000 \
-            | xargs cargo fuzz run peepmatic_compile --release --debug-assertions --features binaryen
-      env:
-        RUST_BACKTRACE: 1
-    - run: |
-        find fuzz/corpus/peepmatic_fst_differential -type f \
-            | shuf \
-            | head -n 10000 \
-            | xargs cargo fuzz run peepmatic_fst_differential --release --debug-assertions --features binaryen
-      env:
-        RUST_BACKTRACE: 1
-    - run: |
-        find fuzz/corpus/peepmatic_interp -type f \
-            | shuf \
-            | head -n 5000 \
-            | xargs cargo fuzz run peepmatic_interp --release --debug-assertions --features binaryen
-      env:
-        RUST_BACKTRACE: 1
-    - run: |
-        find fuzz/corpus/peepmatic_parser -type f \
-            | shuf \
-            | head -n 10000 \
-            | xargs cargo fuzz run peepmatic_parser --release --debug-assertions --features binaryen
-      env:
-        RUST_BACKTRACE: 1
-    - run: |
-        find fuzz/corpus/peepmatic_simple_automata -type f \
-            | shuf \
-            | head -n 1000 \
-            | xargs cargo fuzz run peepmatic_simple_automata --release --debug-assertions --features binaryen
-      env:
-        RUST_BACKTRACE: 1
+    - run: cargo fuzz build --features binaryen
 
   rebuild_peephole_optimizers:
     name: Rebuild Peephole Optimizers

fuzz/Cargo.toml

@@ -14,7 +14,7 @@ cranelift-reader = { path = "../cranelift/reader" }
 cranelift-wasm = { path = "../cranelift/wasm" }
 libfuzzer-sys = "0.3.2"
 target-lexicon = "0.10"
-peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing" }
+peepmatic-fuzzing = { path = "../cranelift/peepmatic/crates/fuzzing", optional = true }
 wasmtime = { path = "../crates/wasmtime" }
 wasmtime-fuzzing = { path = "../crates/fuzzing" }
 
@@ -35,21 +35,21 @@ name = "instantiate_translated"
 path = "fuzz_targets/instantiate_translated.rs"
 test = false
 doc = false
-required-features = ['binaryen']
+required-features = ["binaryen"]
 
 [[bin]]
 name = "api_calls"
 path = "fuzz_targets/api_calls.rs"
 test = false
 doc = false
-required-features = ['binaryen']
+required-features = ["binaryen"]
 
 [[bin]]
 name = "differential"
 path = "fuzz_targets/differential.rs"
 test = false
 doc = false
-required-features = ['binaryen']
+required-features = ["binaryen"]
 
 [[bin]]
 name = "spectests"
@@ -62,30 +62,35 @@ name = "peepmatic_simple_automata"
 path = "fuzz_targets/peepmatic_simple_automata.rs"
 test = false
 doc = false
+required-features = ["peepmatic-fuzzing"]
 
 [[bin]]
 name = "peepmatic_fst_differential"
 path = "fuzz_targets/peepmatic_fst_differential.rs"
 test = false
 doc = false
+required-features = ["peepmatic-fuzzing"]
 
 [[bin]]
 name = "peepmatic_parser"
 path = "fuzz_targets/peepmatic_parser.rs"
 test = false
 doc = false
+required-features = ["peepmatic-fuzzing"]
 
 [[bin]]
 name = "peepmatic_compile"
 path = "fuzz_targets/peepmatic_compile.rs"
 test = false
 doc = false
+required-features = ["peepmatic-fuzzing"]
 
 [[bin]]
 name = "peepmatic_interp"
 path = "fuzz_targets/peepmatic_interp.rs"
 test = false
 doc = false
+required-features = ["peepmatic-fuzzing"]
 
 [features]
-binaryen = ['wasmtime-fuzzing/binaryen']
+binaryen = ["wasmtime-fuzzing/binaryen"]