Skip to content

Commit cde148a

Browse files
ty-everettty-everett
committed
Test benchmarking PR script
1 parent bc0ab58 commit cde148a

File tree

3 files changed

+429
-0
lines changed

3 files changed

+429
-0
lines changed

‎.github/workflows/benchmark-report.yml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: Benchmark Comparison
2+
3+
on:
4+
pull_request:
5+
types: [opened, synchronize, reopened]
6+
7+
jobs:
8+
benchmarks:
9+
name: Benchmarks (Node 22)
10+
runs-on: ubuntu-latest
11+
permissions:
12+
contents: read
13+
pull-requests: write
14+
steps:
15+
- name: Checkout PR branch
16+
uses: actions/checkout@v4
17+
with:
18+
fetch-depth: 0
19+
20+
- name: Setup Node.js
21+
uses: actions/setup-node@v4
22+
with:
23+
node-version: 22
24+
25+
- name: Install dependencies
26+
run: npm ci
27+
28+
- name: Build PR branch
29+
run: npm run build
30+
31+
- name: Run PR benchmarks
32+
run: node scripts/run-benchmarks.js --repo . --output pr-benchmarks.json
33+
34+
- name: Checkout master baseline
35+
uses: actions/checkout@v4
36+
with:
37+
ref: master
38+
path: master
39+
40+
- name: Install dependencies (master)
41+
working-directory: master
42+
run: npm ci
43+
44+
- name: Build master
45+
working-directory: master
46+
run: npm run build
47+
48+
- name: Run master benchmarks
49+
run: node scripts/run-benchmarks.js --repo master --output master-benchmarks.json
50+
51+
- name: Capture baseline sha
52+
id: baseline_sha
53+
run: echo "value=$(cd master && git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
54+
55+
- name: Generate benchmark report
56+
run: >
57+
node scripts/format-benchmark-comment.js
58+
--baseline master-benchmarks.json
59+
--branch pr-benchmarks.json
60+
--output benchmark-report.md
61+
--branch-ref $GITHUB_SHA
62+
--baseline-ref ${{ steps.baseline_sha.outputs.value }}
63+
64+
- name: Upload benchmark artifacts
65+
uses: actions/upload-artifact@v4
66+
with:
67+
name: benchmark-results
68+
path: |
69+
pr-benchmarks.json
70+
master-benchmarks.json
71+
benchmark-report.md
72+
73+
- name: Comment on PR
74+
if: github.event.pull_request.number != ''
75+
uses: actions/github-script@v7
76+
with:
77+
github-token: ${{ secrets.GITHUB_TOKEN }}
78+
script: |
79+
const fs = require('fs');
80+
const body = fs.readFileSync('benchmark-report.md', 'utf8');
81+
await github.rest.issues.createComment({
82+
...context.repo,
83+
issue_number: context.payload.pull_request.number,
84+
body
85+
});

‎scripts/format-benchmark-comment.js‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
#!/usr/bin/env node
2+
import { readFile, writeFile } from 'fs/promises'
3+
import path from 'path'
4+
5+
const THRESHOLD = 0.05
6+
7+
function parseArgs () {
8+
const args = process.argv.slice(2)
9+
const options = {
10+
baseline: null,
11+
branch: null,
12+
output: null,
13+
baselineRef: 'master',
14+
branchRef: 'PR HEAD'
15+
}
16+
17+
for (let i = 0; i < args.length; i++) {
18+
const arg = args[i]
19+
if (arg === '--baseline' && args[i + 1] != null) {
20+
options.baseline = path.resolve(args[++i])
21+
} else if (arg === '--branch' && args[i + 1] != null) {
22+
options.branch = path.resolve(args[++i])
23+
} else if (arg === '--output' && args[i + 1] != null) {
24+
options.output = path.resolve(args[++i])
25+
} else if (arg === '--baseline-ref' && args[i + 1] != null) {
26+
options.baselineRef = args[++i]
27+
} else if (arg === '--branch-ref' && args[i + 1] != null) {
28+
options.branchRef = args[++i]
29+
}
30+
}
31+
32+
if (options.baseline == null || options.branch == null) {
33+
throw new Error('Both --baseline and --branch paths are required.')
34+
}
35+
36+
return options
37+
}
38+
39+
function loadJson (filePath) {
40+
return readFile(filePath, 'utf8').then((data) => JSON.parse(data))
41+
}
42+
43+
function formatMs (value) {
44+
return value == null ? '—' : `${value.toFixed(2)} ms`
45+
}
46+
47+
function pctChange (baseline, value) {
48+
if (baseline == null || value == null || baseline === 0) return null
49+
return ((value - baseline) / baseline) * 100
50+
}
51+
52+
function changeBadge (change) {
53+
if (change == null) return 'n/a'
54+
const sign = change > 0 ? '+' : ''
55+
return `${sign}${change.toFixed(2)}%`
56+
}
57+
58+
function shortSha (sha) {
59+
if (sha == null) return 'unknown'
60+
return sha.slice(0, 7)
61+
}
62+
63+
async function main () {
64+
const options = parseArgs()
65+
const baseline = await loadJson(options.baseline)
66+
const branch = await loadJson(options.branch)
67+
68+
const benchIds = new Set([
69+
...Object.keys(baseline ?? {}),
70+
...Object.keys(branch ?? {})
71+
])
72+
73+
const rows = []
74+
const warnings = []
75+
const kudos = []
76+
const notes = []
77+
78+
for (const id of benchIds) {
79+
const baselineEntry = baseline[id] ?? {}
80+
const branchEntry = branch[id] ?? {}
81+
const label = branchEntry.label ?? baselineEntry.label ?? id
82+
83+
if (baselineEntry.skipped === true) {
84+
notes.push(`- ${label}: baseline skipped (${baselineEntry.reason ?? 'no script found'}).`)
85+
}
86+
if (branchEntry.skipped === true) {
87+
notes.push(`- ${label}: PR branch skipped (${branchEntry.reason ?? 'no script found'}).`)
88+
}
89+
90+
const metrics = new Set([
91+
...Object.keys(baselineEntry.metrics ?? {}),
92+
...Object.keys(branchEntry.metrics ?? {})
93+
])
94+
95+
if (metrics.size === 0) {
96+
rows.push(`| ${label} | _no metrics_ | — | — | — | — |`)
97+
continue
98+
}
99+
100+
for (const metric of metrics) {
101+
const baseVal = baselineEntry.metrics?.[metric]
102+
const prVal = branchEntry.metrics?.[metric]
103+
const delta = (baseVal != null && prVal != null) ? prVal - baseVal : null
104+
const change = pctChange(baseVal, prVal)
105+
106+
if (typeof change === 'number') {
107+
if (change > THRESHOLD * 100 + 0.0001) {
108+
warnings.push(`${label} – ${metric} is ${change.toFixed(2)}% slower (${formatMs(prVal)} vs ${formatMs(baseVal)}).`)
109+
} else if (change < -THRESHOLD * 100 - 0.0001) {
110+
kudos.push(`${label} – ${metric} is ${Math.abs(change).toFixed(2)}% faster (${formatMs(prVal)} vs ${formatMs(baseVal)}).`)
111+
}
112+
}
113+
114+
rows.push(`| ${label} | ${metric} | ${formatMs(prVal)} | ${formatMs(baseVal)} | ${delta == null ? '—' : `${delta >= 0 ? '+' : ''}${delta.toFixed(2)} ms`} | ${changeBadge(change)} |`)
115+
}
116+
}
117+
118+
const summaryParts = []
119+
if (warnings.length > 0) {
120+
summaryParts.push(`⚠️ ${warnings.length} regression${warnings.length === 1 ? '' : 's'} detected (>${THRESHOLD * 100}% slower).`)
121+
} else {
122+
summaryParts.push('✅ No regressions over the 5% threshold detected.')
123+
}
124+
if (kudos.length > 0) {
125+
summaryParts.push(`🎉 ${kudos.length} significant speedup${kudos.length === 1 ? '' : 's'} (>${THRESHOLD * 100}% faster).`)
126+
}
127+
128+
const summary = summaryParts.join(' ')
129+
const header = `## 🏁 Benchmark Comparison (Node 22)\n\nComparing this PR (${shortSha(options.branchRef)}) against master (${shortSha(options.baselineRef)}).\n\n${summary}\n`
130+
131+
const warningSection = warnings.length > 0
132+
? `\n### Regressions\n${warnings.map((w) => `- ⚠️ ${w}`).join('\n')}\n`
133+
: ''
134+
const kudosSection = kudos.length > 0
135+
? `\n### Speedups\n${kudos.map((m) => `- 🎉 ${m}`).join('\n')}\n`
136+
: ''
137+
138+
const table = `\n| Benchmark | Metric | PR Branch | Master | Δ | Change |\n| --- | --- | --- | --- | --- | --- |\n${rows.join('\n')}\n`
139+
const notesSection = notes.length > 0
140+
? `\n### Notes\n${notes.join('\n')}\n`
141+
: ''
142+
143+
const body = `${header}${warningSection}${kudosSection}${table}${notesSection}`
144+
145+
if (options.output != null) {
146+
await writeFile(options.output, body, 'utf8')
147+
} else {
148+
process.stdout.write(body)
149+
}
150+
}
151+
152+
main().catch((err) => {
153+
console.error(err)
154+
process.exit(1)
155+
})

0 commit comments

Comments
 (0)