diff --git a/.github/workflows/reusable-build.yaml b/.github/workflows/reusable-build.yaml index 1713a664..f9b4ad0b 100644 --- a/.github/workflows/reusable-build.yaml +++ b/.github/workflows/reusable-build.yaml @@ -84,6 +84,13 @@ jobs: else echo "Chart $chartFilename exists" fi + htmlReportFilename=$scenarioPath$approach.html + if [ ! -f "$htmlReportFilename" ]; then + echo "Verification error: k6 HTML report $htmlReportFilename missing" + exit 1 + else + echo "k6 HTML report $htmlReportFilename exists" + fi done done resultsCsvFilename=build/results/results.csv @@ -101,7 +108,7 @@ jobs: scenarios=${scenariosPath%%.*} resultsDir=results/ci/$scenarios/${{ matrix.os }}/java-${{ matrix.java-version }}/ mkdir -p $resultsDir - cp build/results/results* $resultsDir + cp -r build/results/* $resultsDir git pull - name: Commit results diff --git a/src/main/bash/benchmark-scenario.sh b/src/main/bash/benchmark-scenario.sh index 813466b5..06636ddd 100755 --- a/src/main/bash/benchmark-scenario.sh +++ b/src/main/bash/benchmark-scenario.sh @@ -68,6 +68,7 @@ jvmCsvFile="$resultDir/$approach"-jvm.csv latencyCsvFile="$resultDir/$approach"-latency.csv systemCsvFile="$resultDir/$approach"-system.csv chartFile="$resultDir/$approach".png +htmlReportFile="$resultDir/$approach".html resultsCsvFile="$resultsDir/results.csv" clientErrorLogFile="$resultDir/$approach"-client-error.log serviceErrorLogFile="$resultDir/$approach"-service-error.log @@ -140,7 +141,7 @@ load_and_measure_system() { (sleep 2 && ./src/main/bash/system-measure.sh "$systemCsvFile" "$durationInSeconds") & systemMeasurePid=$! - load "$durationInSeconds" + load "$phase" "$durationInSeconds" mv "$jvmCsvTmpFile" "$jvmCsvFile" && log "Saved $jvmCsvFile" wait_for_system_csv_file @@ -174,14 +175,44 @@ verify_chart_results() { log "Results file $resultsCsvFile does not exist; terminating" exit 1 fi + if [ ! -f "$htmlReportFile" ]; then + log "k6 HTML report file $htmlReportFile does not exist; terminating" + exit 1 + fi } load() { - _durationInSeconds=$1 + phase=$1 + _durationInSeconds=$2 k6ConfigFile=src/main/resources/scenarios/"$k6Config" log "Issuing requests for ${_durationInSeconds}s using ${k6ConfigFile}..." - k6 run --env DURATION_IN_SECONDS="${_durationInSeconds}" --out csv="$k6OutputTmpFile" --env K6_CSV_TIME_FORMAT="unix_milli" --env DELAY_CALL_DEPTH="$delayCallDepth" --env DELAY_IN_MILLIS="$delayInMillis" --env SERVICE_API_BASE_URL="$serviceApiBaseUrl" --env VUS="$connections" --env RPS="$requestsPerSecond" "$k6ConfigFile" 2>&1 | tee "$k6LogTmpFile" + + # Only generate HTML report during test phase, not during warmup + if [ "$phase" == "test" ]; then + K6_WEB_DASHBOARD=true K6_WEB_DASHBOARD_EXPORT="$htmlReportFile" \ + k6 run \ + --env DURATION_IN_SECONDS="${_durationInSeconds}" \ + --out csv="$k6OutputTmpFile" \ + --env K6_CSV_TIME_FORMAT="unix_milli" \ + --env DELAY_CALL_DEPTH="$delayCallDepth" \ + --env DELAY_IN_MILLIS="$delayInMillis" \ + --env SERVICE_API_BASE_URL="$serviceApiBaseUrl" \ + --env VUS="$connections" \ + --env RPS="$requestsPerSecond" \ + "$k6ConfigFile" 2>&1 | tee "$k6LogTmpFile" + else + k6 run \ + --env DURATION_IN_SECONDS="${_durationInSeconds}" \ + --out csv="$k6OutputTmpFile" \ + --env K6_CSV_TIME_FORMAT="unix_milli" \ + --env DELAY_CALL_DEPTH="$delayCallDepth" \ + --env DELAY_IN_MILLIS="$delayInMillis" \ + --env SERVICE_API_BASE_URL="$serviceApiBaseUrl" \ + --env VUS="$connections" \ + --env RPS="$requestsPerSecond" \ + "$k6ConfigFile" 2>&1 | tee "$k6LogTmpFile" + fi # csv: metric_name,timestamp,metric_value,check,error,error_code,expected_response,group,method,name,proto,scenario,service,status # shellcheck disable=SC2002 diff --git a/src/main/bash/generate-results-markdown.sh b/src/main/bash/generate-results-markdown.sh index a7c96a97..ef9d89e5 100755 --- a/src/main/bash/generate-results-markdown.sh +++ b/src/main/bash/generate-results-markdown.sh @@ -80,8 +80,12 @@ while IFS=',' read -r scenario k6Config serverProfiles delayCallDepth delayInMil IFS=',' read -ra approach_array <<< "$approaches" for approach in "${approach_array[@]}"; do image_path="$results_dir/$scenario/$approach.png" + html_report_path="$results_dir/$scenario/$approach.html" if [[ -f "$image_path" ]]; then markdown+="#### $approach\n\n" + if [[ -f "$html_report_path" ]]; then + markdown+="[k6 Report](./$scenario/$approach.html)\n\n" + fi markdown+="![$approach](./$scenario/$approach.png)\n\n" else markdown+="#### $approach (No image available)\n\n" diff --git a/src/main/resources/scenarios/scenarios-default.csv b/src/main/resources/scenarios/scenarios-default.csv index e3ae5e5c..76607783 100644 --- a/src/main/resources/scenarios/scenarios-default.csv +++ b/src/main/resources/scenarios/scenarios-default.csv @@ -1,5 +1,5 @@ scenario,k6Config,serverProfiles,delayCallDepth,delayInMillis,connections,requestsPerSecond,warmupDurationInSeconds,testDurationInSeconds -smoketest,get-time.js,,1,100,5,5,0,5 +smoketest,get-time.js,,1,100,5,5,0,15 1k-vus-and-rps-get-time-no-delay,get-time.js,,0,0,1000,1000,10,180 5k-vus-and-rps-get-time,get-time.js,,0,100,5000,5000,10,180 5k-vus-and-rps-get-movies,get-movies.js,,0,100,5000,5000,10,180 diff --git a/src/main/resources/scenarios/scenarios-smoketest.csv b/src/main/resources/scenarios/scenarios-smoketest.csv index 7ab57371..40738326 100644 --- a/src/main/resources/scenarios/scenarios-smoketest.csv +++ b/src/main/resources/scenarios/scenarios-smoketest.csv @@ -1,5 +1,5 @@ scenario,k6Config,serverProfiles,delayCallDepth,delayInMillis,connections,requestsPerSecond,warmupDurationInSeconds,testDurationInSeconds -smoketest-get-time,get-time.js,,0,0,100,200,2,6 -smoketest-get-movies-h2,get-movies.js,,1,100,10,10,0,6 -smoketest-get-movies-postgres,get-movies.js,postgres,1,100,10,10,0,6 -smoketest-get-movies-postgres-no-cache,get-movies.js,postgres|no-cache,1,100,10,10,0,6 +smoketest-get-time,get-time.js,,0,0,100,200,2,15 +smoketest-get-movies-h2,get-movies.js,,1,100,10,10,0,15 +smoketest-get-movies-postgres,get-movies.js,postgres,1,100,10,10,0,15 +smoketest-get-movies-postgres-no-cache,get-movies.js,postgres|no-cache,1,100,10,10,0,15