From d7ff239b9fc402bb914d7f36d49e8b7486dc85ba Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Tue, 2 Dec 2025 11:22:56 -0600 Subject: [PATCH 01/10] Use apm-sdks-benchmarks macrobenchmarks --- .gitlab-ci.yml | 2 +- .gitlab/macrobenchmarks.yml | 152 ++---------------------------------- 2 files changed, 8 insertions(+), 146 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21759852a4e..f5c12d7f529 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ stages: - publish - shared-pipeline - benchmarks - - macrobenchmarks + - java-spring-petclinic-macrobenchmarks - tests - exploration-tests - ci-visibility-tests diff --git a/.gitlab/macrobenchmarks.yml b/.gitlab/macrobenchmarks.yml index 9c7a9cd4cad..ea0c1d69a37 100644 --- a/.gitlab/macrobenchmarks.yml +++ b/.gitlab/macrobenchmarks.yml @@ -1,146 +1,8 @@ -include: - - project: 'DataDog/benchmarking-platform-tools' - file: 'images/templates/gitlab/notify-slo-breaches.template.yml' - - project: 'DataDog/benchmarking-platform-tools' - file: 'images/templates/gitlab/check-slo-breaches.template.yml' - -.macrobenchmarks: - stage: macrobenchmarks - rules: - - if: $POPULATE_CACHE - when: never - - if: ($NIGHTLY_BENCHMARKS || $CI_PIPELINE_SOURCE != "schedule") && $CI_COMMIT_REF_NAME == "master" - when: always - interruptible: false - - when: manual - interruptible: true - allow_failure: true - tags: ["runner:apm-k8s-same-cpu"] - needs: ["build"] - timeout: 1h - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-java-petclinic - script: - - git clone --branch java/petclinic https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform - - ./steps/run-benchmarks.sh - artifacts: - name: "artifacts" - when: always - paths: - - platform/artifacts/ - expire_in: 3 months - variables: - K6_OPTIONS_WARMUP_RATE: 2000 - K6_OPTIONS_WARMUP_DURATION: 5m - K6_OPTIONS_WARMUP_GRACEFUL_STOP: 10s - K6_OPTIONS_WARMUP_PRE_ALLOCATED_VUS: 4 - K6_OPTIONS_WARMUP_MAX_VUS: 4 - - K6_OPTIONS_NORMAL_OPERATION_RATE: 1500 - K6_OPTIONS_NORMAL_OPERATION_DURATION: 10m - K6_OPTIONS_NORMAL_OPERATION_GRACEFUL_STOP: 10s - K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 4 - K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 4 - - K6_OPTIONS_HIGH_LOAD_RATE: 4000 - K6_OPTIONS_HIGH_LOAD_DURATION: 5m - K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 10s - K6_OPTIONS_HIGH_LOAD_PRE_ALLOCATED_VUS: 4 - K6_OPTIONS_HIGH_LOAD_MAX_VUS: 4 - retry: - max: 2 - when: - - unknown_failure - - data_integrity_failure - - runner_system_failure - - scheduler_failure - - api_failure - -baseline: - extends: .macrobenchmarks - variables: - BP_BENCHMARKS_CONFIGURATION: baseline - TRACER_OPTS: -Ddd.service=bp-java-petclinic - JAVA_OPTS: -javaagent:/app/memcheck/stability-testing-memwatch.jar -Xmx128M - -only-tracing: - extends: .macrobenchmarks - variables: - BP_BENCHMARKS_CONFIGURATION: only-tracing - TRACER_OPTS: -javaagent:/app/dd-java-agent.jar -Ddd.env=${BP_BENCHMARKS_CONFIGURATION} -Ddd.service=bp-java-petclinic - JAVA_OPTS: -javaagent:/app/memcheck/stability-testing-memwatch.jar -Xmx128M +# macrobenchmarks are located in the `apm-sdks-benchmarks` repository. +stages: + - java-spring-petclinic-macrobenchmarks -otel-latest: - extends: .macrobenchmarks - variables: - BP_BENCHMARKS_CONFIGURATION: otel-latest - TRACER_OPTS: -javaagent:/app/otel-java-agent.jar -Ddd.env=otel-latest -Ddd.service=bp-java-petclinic - JAVA_OPTS: -javaagent:/app/memcheck/stability-testing-memwatch.jar -Xmx128M - - -check-slo-breaches: - extends: .check-slo-breaches - stage: macrobenchmarks - interruptible: true - rules: - - if: $POPULATE_CACHE - when: never - - when: on_success - needs: - - job: baseline - artifacts: true - - job: only-tracing - artifacts: true - - job: otel-latest - artifacts: true - - job: benchmarks-startup - artifacts: true - - job: benchmarks-load - artifacts: true - - job: benchmarks-dacapo - artifacts: true - script: - # macrobenchmarks are located here, files are already in "converted" format - - export ARTIFACTS_DIR="$(pwd)/platform/artifacts/" && mkdir -p "${ARTIFACTS_DIR}" - - # Need to move the artifacts the benchmarks-* job - - | - export BENCHMARKS_ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${BENCHMARKS_ARTIFACTS_DIR}" - for benchmarkType in startup load dacapo; do - find "$BENCHMARKS_ARTIFACTS_DIR/$benchmarkType" -name "benchmark-baseline.json" -o -name "benchmark-candidate.json" | while read file; do - relpath="${file#$BENCHMARKS_ARTIFACTS_DIR/$benchmarkType/}" - prefix="${relpath%/benchmark-*}" # Remove the trailing /benchmark-(baseline|candidate).json - prefix="${prefix#./}" # Remove any leading ./ - prefix="${prefix//\//-}" # Replace / with - - case "$file" in - *benchmark-baseline.json) type="baseline" ;; - *benchmark-candidate.json) type="candidate" ;; - esac - echo "Moving $file to $ARTIFACTS_DIR/${type}-${benchmarkType}-${prefix}.converted.json" - cp "$file" "$ARTIFACTS_DIR/${type}-${benchmarkType}-${prefix}.converted.json" - done - done - - ls -lah "$ARTIFACTS_DIR" - - bp-runner .gitlab/benchmarks/bp-runner.fail-on-breach.yml - artifacts: - name: "artifacts" - when: always - paths: - - platform/artifacts/ - expire_in: 1 week - variables: - DDOCTOSTS_POLICY: "self.gitlab.github-access.read" - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance. - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-java" - UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built. - UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for. - -notify-slo-breaches: - extends: .notify-slo-breaches - stage: macrobenchmarks - needs: ["check-slo-breaches"] - rules: - - if: $POPULATE_CACHE - when: never - - when: always - variables: - CHANNEL: "apm-java" +include: + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-spring-petclinic.yml' + ref: 'sarahchen6/migrate-java-macrobenchmarks' From b37678591b430fc23e42645a682e6eae126e61af Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Tue, 2 Dec 2025 11:42:09 -0600 Subject: [PATCH 02/10] Move to gitlab-ci file --- .gitlab-ci.yml | 3 +++ .gitlab/macrobenchmarks.yml | 14 +++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f5c12d7f529..2c9006c2768 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,9 @@ include: - local: ".gitlab/macrobenchmarks.yml" - local: ".gitlab/exploration-tests.yml" - local: ".gitlab/ci-visibility-tests.yml" + - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' + file: '.gitlab/ci-java-spring-petclinic.yml' + ref: 'sarahchen6/migrate-java-macrobenchmarks' stages: - build diff --git a/.gitlab/macrobenchmarks.yml b/.gitlab/macrobenchmarks.yml index ea0c1d69a37..7226ed7972b 100644 --- a/.gitlab/macrobenchmarks.yml +++ b/.gitlab/macrobenchmarks.yml @@ -1,8 +1,8 @@ -# macrobenchmarks are located in the `apm-sdks-benchmarks` repository. -stages: - - java-spring-petclinic-macrobenchmarks +# # macrobenchmarks are located in the `apm-sdks-benchmarks` repository. +# stages: +# - java-spring-petclinic-macrobenchmarks -include: - - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' - file: '.gitlab/ci-java-spring-petclinic.yml' - ref: 'sarahchen6/migrate-java-macrobenchmarks' +# include: +# - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' +# file: '.gitlab/ci-java-spring-petclinic.yml' +# ref: 'sarahchen6/migrate-java-macrobenchmarks' From bf66715302f234bc6fee109408acbc60e373c285 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Tue, 2 Dec 2025 11:46:46 -0600 Subject: [PATCH 03/10] Test main --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c9006c2768..650e2bc7325 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,14 +6,16 @@ include: - local: ".gitlab/ci-visibility-tests.yml" - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' file: '.gitlab/ci-java-spring-petclinic.yml' - ref: 'sarahchen6/migrate-java-macrobenchmarks' + ref: 'main' + # ref: 'sarahchen6/migrate-java-macrobenchmarks' stages: - build - publish - shared-pipeline - benchmarks - - java-spring-petclinic-macrobenchmarks + - java-spring-petclinic-tests + # - java-spring-petclinic-macrobenchmarks - tests - exploration-tests - ci-visibility-tests From ddf61d52ea5d9d4b3fa0c9a50f15c1dde8023ccf Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Tue, 2 Dec 2025 11:58:37 -0600 Subject: [PATCH 04/10] Fix includes and stages --- .gitlab-ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 650e2bc7325..3a57427533b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,11 @@ include: - local: ".gitlab/one-pipeline.locked.yml" - local: ".gitlab/benchmarks.yml" - - local: ".gitlab/macrobenchmarks.yml" - local: ".gitlab/exploration-tests.yml" - local: ".gitlab/ci-visibility-tests.yml" - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' file: '.gitlab/ci-java-spring-petclinic.yml' - ref: 'main' - # ref: 'sarahchen6/migrate-java-macrobenchmarks' + ref: 'sarahchen6/migrate-java-macrobenchmarks' stages: - build @@ -15,7 +13,7 @@ stages: - shared-pipeline - benchmarks - java-spring-petclinic-tests - # - java-spring-petclinic-macrobenchmarks + - java-spring-petclinic-macrobenchmarks - tests - exploration-tests - ci-visibility-tests From d69aeaff514000b29641bb4ab6b7a78cb14cbfd5 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Tue, 2 Dec 2025 17:31:45 -0600 Subject: [PATCH 05/10] Delete macrobenchmarks file --- .gitlab/macrobenchmarks.yml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .gitlab/macrobenchmarks.yml diff --git a/.gitlab/macrobenchmarks.yml b/.gitlab/macrobenchmarks.yml deleted file mode 100644 index 7226ed7972b..00000000000 --- a/.gitlab/macrobenchmarks.yml +++ /dev/null @@ -1,8 +0,0 @@ -# # macrobenchmarks are located in the `apm-sdks-benchmarks` repository. -# stages: -# - java-spring-petclinic-macrobenchmarks - -# include: -# - project: 'DataDog/apm-reliability/apm-sdks-benchmarks' -# file: '.gitlab/ci-java-spring-petclinic.yml' -# ref: 'sarahchen6/migrate-java-macrobenchmarks' From a9e13c98d320a523aa171224ac1dca70b7d059d1 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Wed, 3 Dec 2025 10:33:26 -0600 Subject: [PATCH 06/10] Empty commit to run benchmarks again From 4a173b815f3d5e650a4856e975571d697b182d3d Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Wed, 3 Dec 2025 11:58:47 -0600 Subject: [PATCH 07/10] Empty commit to run benchmarks again 2 From de5ab068e039f425abfc844e2cd29744467e23c6 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Thu, 4 Dec 2025 09:31:44 -0500 Subject: [PATCH 08/10] Empty commit to run benchmarks again 3 From 6c41b8c5cf14581b3581b70a4bbeb7f1f70fb088 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Thu, 4 Dec 2025 09:40:43 -0500 Subject: [PATCH 09/10] Empty commit to run benchmarks again 4 From bd827c37f7fab30d5a4f3dff36fcf9e8f7f7fe47 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Fri, 5 Dec 2025 11:05:02 -0500 Subject: [PATCH 10/10] Exclude startup benchmarks from slo check for now --- .../benchmarks/bp-runner.fail-on-breach.yml | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.gitlab/benchmarks/bp-runner.fail-on-breach.yml b/.gitlab/benchmarks/bp-runner.fail-on-breach.yml index bb2211a27fe..1a97b99cb1d 100644 --- a/.gitlab/benchmarks/bp-runner.fail-on-breach.yml +++ b/.gitlab/benchmarks/bp-runner.fail-on-breach.yml @@ -35,14 +35,15 @@ experiments: thresholds: - throughput > 1100.0 op/s - # Startup macrobenchmarks - # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Atracing%3AGlobalTracer&trendsType=scenario - # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Aappsec%3AGlobalTracer&trendsType=scenario - # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Aiast%3AGlobalTracer&trendsType=scenario - - name: "startup:petclinic:(tracing|appsec|iast):GlobalTracer" - thresholds: - - execution_time < 280 ms - # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Aprofiling%3AGlobalTracer&trendsType=scenario - - name: "startup:petclinic:profiling:GlobalTracer" - thresholds: - - execution_time < 420 ms + # TODO: Startup benchmarks need to be moved from dd-trace-java to apm-sdks-benchmarks (https://github.com/DataDog/dd-trace-java/blob/master/benchmark/startup/petclinic/benchmark.json) + # # Startup macrobenchmarks + # # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Atracing%3AGlobalTracer&trendsType=scenario + # # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Aappsec%3AGlobalTracer&trendsType=scenario + # # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Aiast%3AGlobalTracer&trendsType=scenario + # - name: "startup:petclinic:(tracing|appsec|iast):GlobalTracer" + # thresholds: + # - execution_time < 280 ms + # # https://benchmarking.us1.prod.dog/trends?projectId=4&branch=master&trendsTab=per_scenario&scenario=startup%3Apetclinic%3Aprofiling%3AGlobalTracer&trendsType=scenario + # - name: "startup:petclinic:profiling:GlobalTracer" + # thresholds: + # - execution_time < 420 ms