diff --git a/.github/workflows/master_benchmarks.yml b/.github/workflows/master_benchmarks.yml new file mode 100644 index 000000000..227913271 --- /dev/null +++ b/.github/workflows/master_benchmarks.yml @@ -0,0 +1,34 @@ +on: + push: + branches: main + +jobs: + run_benchmarks: + name: Run PR Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Valgrind + run: sudo apt update && sudo apt install -y valgrind + - uses: taiki-e/install-action@cargo-binstall + - name: Install gungraun-runner + run: | + version=$(cargo metadata --format-version=1 |\ + jq '.packages[] | select(.name == "gungraun").version' |\ + tr -d '"' + ) + cargo binstall --no-confirm gungraun-runner --version $version + - name: Run Benchmarks + run: cargo bench --manifest-path ./benches/Cargo.toml > benchmark_results.txt + - name: Track Benchmarks with Bencher + run: | + bencher run \ + --host 'https://bencher.php.rs/api' \ + --project ext-php-rs \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch master \ + --testbed "$(php --version | head -n 1)" \ + --err \ + --adapter rust_gungraun \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --file "./benchmark_results.txt" diff --git a/.github/workflows/pr_benchmarks_archive.yml b/.github/workflows/pr_benchmarks_archive.yml new file mode 100644 index 000000000..338011423 --- /dev/null +++ b/.github/workflows/pr_benchmarks_archive.yml @@ -0,0 +1,18 @@ +on: + pull_request_target: + types: [closed] + +jobs: + archive_fork_pr_branch: + name: Archive closed PR branch with Bencher + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: bencherdev/bencher@main + - name: Archive closed fork PR branch with Bencher + run: | + bencher archive \ + --host 'https://bencher.php.rs/api' \ + --project ext-php-rs \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch "$GITHUB_HEAD_REF" diff --git a/.github/workflows/pr_benchmarks_run.yml b/.github/workflows/pr_benchmarks_run.yml new file mode 100644 index 000000000..6da8d5ebe --- /dev/null +++ b/.github/workflows/pr_benchmarks_run.yml @@ -0,0 +1,34 @@ +name: Run Benchmarks + +on: + pull_request: + types: [opened, reopened, edited, synchronize] + +jobs: + run_benchmarks: + name: Run PR Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Valgrind + run: sudo apt update && sudo apt install -y valgrind + - uses: taiki-e/install-action@cargo-binstall + - name: Install gungraun-runner + run: | + version=$(cargo metadata --manifest-path ./benches/Cargo.toml --format-version=1 |\ + jq '.packages[] | select(.name == "gungraun").version' |\ + tr -d '"' + ) + cargo binstall --no-confirm gungraun-runner --version $version + - name: Run Benchmarks + run: cargo bench --manifest-path ./benches/Cargo.toml > benchmark_results.txt + - name: Upload Benchmark Results + uses: actions/upload-artifact@v4 + with: + name: benchmark_results.txt + path: ./benchmark_results.txt + - name: Upload Pull Request Event + uses: actions/upload-artifact@v4 + with: + name: event.json + path: ${{ github.event_path }} diff --git a/.github/workflows/pr_benchmarks_upload.yml b/.github/workflows/pr_benchmarks_upload.yml new file mode 100644 index 000000000..88acf9d79 --- /dev/null +++ b/.github/workflows/pr_benchmarks_upload.yml @@ -0,0 +1,57 @@ +name: Upload PR Benchmark Results + +on: + workflow_run: + workflows: [Run Benchmarks] + types: [completed] + +jobs: + upload_benchmarks: + if: github.event.workflow_run.conclusion == 'success' + permissions: + pull-requests: write + runs-on: ubuntu-latest + env: + BENCHMARK_RESULTS: benchmark_results.txt + PR_EVENT: event.json + steps: + - name: Download Benchmark Results + uses: dawidd6/action-download-artifact@v6 + with: + name: ${{ env.BENCHMARK_RESULTS }} + run_id: ${{ github.event.workflow_run.id }} + - name: Download PR Event + uses: dawidd6/action-download-artifact@v6 + with: + name: ${{ env.PR_EVENT }} + run_id: ${{ github.event.workflow_run.id }} + - name: Export PR Event Data + uses: actions/github-script@v6 + with: + script: | + let fs = require('fs'); + let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'})); + core.exportVariable("PR_HEAD", prEvent.pull_request.head.ref); + core.exportVariable("PR_HEAD_SHA", prEvent.pull_request.head.sha); + core.exportVariable("PR_BASE", prEvent.pull_request.base.ref); + core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha); + core.exportVariable("PR_NUMBER", prEvent.number); + - uses: bencherdev/bencher@main + - name: Track Benchmarks with Bencher + run: | + bencher run \ + --host 'https://bencher.php.rs/api' \ + --project ext-php-rs \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch "$PR_HEAD" \ + --hash "$PR_HEAD_SHA" \ + --start-point "$PR_BASE" \ + --start-point-hash "$PR_BASE_SHA" \ + --start-point-clone-thresholds \ + --start-point-reset \ + --testbed "$(php --version | head -n 1)" \ + --err \ + --adapter rust_gungraun \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-number "$PR_NUMBER" \ + --file "$BENCHMARK_RESULTS" diff --git a/Cargo.toml b/Cargo.toml index b07f582d0..e2d49412a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ static = ["bindgen/static"] [workspace] members = ["crates/macros", "crates/cli", "tests"] +exclude = ["benches"] [package.metadata.docs.rs] rustdoc-args = ["--cfg", "docs"] diff --git a/benches/.cargo/config.toml b/benches/.cargo/config.toml new file mode 100644 index 000000000..88c4308c0 --- /dev/null +++ b/benches/.cargo/config.toml @@ -0,0 +1,14 @@ +[target.'cfg(not(target_os = "windows"))'] +rustflags = ["-C", "link-arg=-Wl,-undefined,dynamic_lookup"] + +[target.x86_64-pc-windows-msvc] +linker = "rust-lld" + +[target.i686-pc-windows-msvc] +linker = "rust-lld" + +[target.'cfg(target_env = "musl")'] +rustflags = ["-C", "target-feature=-crt-static"] + +[profile.bench] +debug = true diff --git a/benches/.gitignore b/benches/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/benches/.gitignore @@ -0,0 +1 @@ +/target diff --git a/benches/Cargo.toml b/benches/Cargo.toml new file mode 100644 index 000000000..61deab0f3 --- /dev/null +++ b/benches/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "benches" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +ext-php-rs = { path = "../" } +gungraun = { version = "0.17.0", features = ["client_requests"] } + +[features] +default = ["enum", "runtime", "closure"] +enum = ["ext-php-rs/enum"] +anyhow = ["ext-php-rs/anyhow"] +runtime = ["ext-php-rs/runtime"] +closure = ["ext-php-rs/closure"] +static = ["ext-php-rs/static"] + +[lib] +crate-type = ["cdylib"] +bench = false + +[[bench]] +name = "binary_bench" +harness = false diff --git a/benches/benches/binary_bench.rs b/benches/benches/binary_bench.rs new file mode 100644 index 000000000..8cd81bfde --- /dev/null +++ b/benches/benches/binary_bench.rs @@ -0,0 +1,102 @@ +use std::{ + process::Command, + sync::{LazyLock, Once}, +}; + +use gungraun::{ + binary_benchmark, binary_benchmark_group, main, BinaryBenchmarkConfig, Callgrind, + FlamegraphConfig, +}; + +static BUILD: Once = Once::new(); +static EXT_TARGET_DIR: LazyLock = LazyLock::new(|| { + let mut dir = std::env::current_dir().expect("Could not get cwd"); + dir.push("target"); + dir.push("release"); + dir.display().to_string() +}); + +fn setup() { + BUILD.call_once(|| { + let mut command = Command::new("cargo"); + command.arg("build"); + + command.arg("--release"); + + // Build features list dynamically based on compiled features + // Note: Using vec_init_then_push pattern here is intentional due to conditional compilation + #[allow(clippy::vec_init_then_push)] + { + let mut features = vec![]; + #[cfg(feature = "enum")] + features.push("enum"); + #[cfg(feature = "closure")] + features.push("closure"); + #[cfg(feature = "anyhow")] + features.push("anyhow"); + #[cfg(feature = "runtime")] + features.push("runtime"); + #[cfg(feature = "static")] + features.push("static"); + + if !features.is_empty() { + command.arg("--no-default-features"); + command.arg("--features").arg(features.join(",")); + } + } + + let result = command.output().expect("failed to execute cargo build"); + + assert!( + result.status.success(), + "Extension build failed:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + }); +} + +#[binary_benchmark] +#[bench::single_function_call(args = ("benches/function_call.php", 1))] +#[bench::multiple_function_calls(args = ("benches/function_call.php", 10))] +#[bench::lots_of_function_calls(args = ("benches/function_call.php", 100_000))] +fn function_calls(path: &str, cnt: usize) -> gungraun::Command { + setup(); + + gungraun::Command::new("php") + .arg(format!("-dextension={}/libbenches.so", *EXT_TARGET_DIR)) + .arg(path) + .arg(cnt.to_string()) + .build() +} + +#[binary_benchmark] +#[bench::single_callback_call(args = ("benches/callback_call.php", 1))] +#[bench::multiple_callback_calls(args = ("benches/callback_call.php", 10))] +#[bench::lots_of_callback_calls(args = ("benches/callback_call.php", 100_000))] +fn callback_calls(path: &str, cnt: usize) -> gungraun::Command { + setup(); + + gungraun::Command::new("php") + .arg(format!("-dextension={}/libbenches.so", *EXT_TARGET_DIR)) + .arg(path) + .arg(cnt.to_string()) + .build() +} + +binary_benchmark_group!( + name = function; + benchmarks = function_calls +); + +binary_benchmark_group!( + name = callback; + benchmarks = callback_calls +); + +main!( + config = BinaryBenchmarkConfig::default() + .tool(Callgrind::with_args(["--instr-atstart=no", "--I1=32768,8,64", "--D1=32768,8,64", "--LL=67108864,16,64"]) + .flamegraph(FlamegraphConfig::default())); + binary_benchmark_groups = function, callback +); diff --git a/benches/benches/callback_call.php b/benches/benches/callback_call.php new file mode 100644 index 000000000..04123721b --- /dev/null +++ b/benches/benches/callback_call.php @@ -0,0 +1,5 @@ + $i * 2, (int) $argv[1]); diff --git a/benches/benches/function_call.php b/benches/benches/function_call.php new file mode 100644 index 000000000..3c54f6b6d --- /dev/null +++ b/benches/benches/function_call.php @@ -0,0 +1,11 @@ + u64 { + // A simple function that does not do much work + n +} + +#[php_function] +pub fn bench_callback_function(callback: ZendCallable, n: usize) { + // Call the provided PHP callable with a fixed argument + start_instrumentation(); + for i in 0..n { + callback + .try_call(vec![&i]) + .expect("Failed to call function"); + } + stop_instrumentation(); +} + +#[php_function] +pub fn start_instrumentation() { + gungraun::client_requests::callgrind::start_instrumentation(); + // gungraun::client_requests::callgrind::toggle_collect(); +} + +#[php_function] +pub fn stop_instrumentation() { + gungraun::client_requests::callgrind::stop_instrumentation(); +} + +#[php_module] +pub fn build_module(module: ModuleBuilder) -> ModuleBuilder { + module + .function(wrap_function!(bench_function)) + .function(wrap_function!(bench_callback_function)) + .function(wrap_function!(start_instrumentation)) + .function(wrap_function!(stop_instrumentation)) +} diff --git a/flake.lock b/flake.lock index 594401cd2..bc46cd1f5 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1762977756, - "narHash": "sha256-4PqRErxfe+2toFJFgcRKZ0UI9NSIOJa+7RXVtBhy4KE=", + "lastModified": 1765186076, + "narHash": "sha256-hM20uyap1a0M9d344I692r+ik4gTMyj60cQWO+hAYP8=", "owner": "nixos", "repo": "nixpkgs", - "rev": "c5ae371f1a6a7fd27823bc500d9390b38c05fa55", + "rev": "addf7cf5f383a3101ecfba091b98d0a1263dc9b8", "type": "github" }, "original": { @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1763087910, - "narHash": "sha256-eB9Z1mWd1U6N61+F8qwDggX0ihM55s4E0CluwNukJRU=", + "lastModified": 1765248027, + "narHash": "sha256-ngar+yP06x3+2k2Iey29uU0DWx5ur06h3iPBQXlU+yI=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "cf4a68749733d45c0420726596367acd708eb2e8", + "rev": "7b50ad68415ae5be7ee4cc68fa570c420741b644", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 3d9f93a15..e9396afa2 100644 --- a/flake.nix +++ b/flake.nix @@ -28,12 +28,14 @@ php-dev libclang.lib clang + valgrind ]; nativeBuildInputs = [ pkgs.rust-bin.stable.latest.default ]; shellHook = '' export LIBCLANG_PATH="${pkgs.libclang.lib}/lib" + export GUNGRAUN_VALGRIND_INCLUDE="${pkgs.valgrind.dev}/include" ''; }; };