use prometheus instead of benchstat

Author: jianzhangbjz

.github/workflows/benchmark.yaml

@@ -33,14 +33,136 @@ jobs:
           mkdir -p /tmp/artifacts/
           ARTIFACT_PATH=/tmp/artifacts make test-benchmark
 
-      - name: Compare with baseline
+      - name: Convert Benchmark Output to Prometheus Metrics
         run: |
-          go install golang.org/x/perf/cmd/benchstat@latest
-          benchstat benchmarks/baseline.txt /tmp/artifacts/new.txt | tee /tmp/artifacts/output
+          mkdir -p /tmp/artifacts/prometheus/
+          cat << 'EOF' > benchmark_to_prometheus.py
+          import sys
+          import re
 
-      - name: Upload benchmark results      
+          def parse_benchmark_output(benchmark_output):
+              metrics = []
+              for line in benchmark_output.split("\n"):
+                  match = re.match(r"Benchmark([\w\d]+)-\d+\s+\d+\s+([\d]+)\s+ns/op\s+([\d]+)\s+B/op\s+([\d]+)\s+allocs/op", line)
+                  if match:
+                      benchmark_name = match.group(1).lower()
+                      time_ns = match.group(2)
+                      memory_bytes = match.group(3)
+                      allocs = match.group(4)
+
+                      metrics.append(f"benchmark_{benchmark_name}_ns {time_ns}")
+                      metrics.append(f"benchmark_{benchmark_name}_allocs {allocs}")
+                      metrics.append(f"benchmark_{benchmark_name}_mem_bytes {memory_bytes}")
+
+              return "\n".join(metrics)
+
+          if __name__ == "__main__":
+              benchmark_output = sys.stdin.read()
+              metrics = parse_benchmark_output(benchmark_output)
+              print(metrics)
+          EOF
+
+          cat /tmp/artifacts/new.txt | python3 benchmark_to_prometheus.py > /tmp/artifacts/prometheus/metrics.txt
+
+      # - name: Compare with baseline
+      #   run: |
+      #     go install golang.org/x/perf/cmd/benchstat@latest
+      #     benchstat benchmarks/baseline.txt /tmp/artifacts/new.txt | tee /tmp/artifacts/output
+
+      - name: Upload Benchmark Metrics     
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-metrics
+          path: /tmp/artifacts/prometheus/
+
+  start-prometheus:
+    needs: benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Download Benchmark Metrics
+        uses: actions/download-artifact@v4
+        with:
+          name: benchmark-metrics
+          path: ./
+
+      - name: List Downloaded Metrics files
+        run: ls -lh
+
+      - name: Set Up Prometheus Config
+        run: |
+          cat << 'EOF' > prometheus.yml
+          global:
+            scrape_interval: 5s
+          scrape_configs:
+            - job_name: 'benchmark_metrics'
+              static_configs:
+                - targets: ['localhost:9000']
+          EOF
+
+      - name: Run Prometheus
+        run: |
+          docker run -d --name prometheus -p 9090:9090 \
+            -v ${{ github.workspace }}/prometheus.yml:/etc/prometheus/prometheus.yml \
+            -v ${{ github.workspace }}/prometheus-data:/prometheus \
+            prom/prometheus --config.file=/etc/prometheus/prometheus.yml \
+            --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=1h
+
+      - name: Wait for Prometheus to start
+        run: sleep 10
+        
+      - name: Check Prometheus is running
+        run: curl -s http://localhost:9090/-/ready || (docker logs prometheus && exit 1)
+          
+      - name: Start HTTP Server to Expose Metrics
+        run: |
+          cat << 'EOF' > server.py
+          from http.server import SimpleHTTPRequestHandler, HTTPServer
+
+          class MetricsHandler(SimpleHTTPRequestHandler):
+              def do_GET(self):
+                  if self.path == "/metrics":
+                      self.send_response(200)
+                      self.send_header("Content-type", "text/plain")
+                      self.end_headers()
+                      with open("metrics.txt", "r") as f:
+                          self.wfile.write(f.read().encode())
+                  else:
+                      self.send_response(404)
+                      self.end_headers()
+
+          if __name__ == "__main__":
+              server = HTTPServer(('0.0.0.0', 9000), MetricsHandler)
+              print("Serving on port 9000...")
+              server.serve_forever()
+          EOF
+          
+          nohup python3 server.py &
+
+      - name: Wait for Prometheus to Collect Data
+        run: sleep 30
+
+      - name: Trigger Prometheus Snapshot
+        run: |
+          curl -X POST http://localhost:9090/api/v1/admin/tsdb/snapshot
+
+      - name: Stop Prometheus
+        run: docker stop prometheus
+
+      - name: Find and Upload Prometheus Snapshot
+        run: |
+          SNAPSHOT_PATH=$(ls -td /prometheus/snapshots/* | head -1)
+          echo "Prometheus snapshot stored in: $SNAPSHOT_PATH"
+          tar -czf prometheus_snapshot.tar.gz -C "$SNAPSHOT_PATH" .
+          mv prometheus_snapshot.tar.gz $GITHUB_WORKSPACE/
+
+      - name: Upload Prometheus Snapshot
         uses: actions/upload-artifact@v4
         with:
-          name: benchmark-artifacts
-          path: /tmp/artifacts/
+          name: prometheus-snapshot
+          path: prometheus_snapshot.tar.gz