From 84fb7d86e58b104720b44876018f0cf081f47174 Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Wed, 16 Oct 2024 03:18:31 -0700 Subject: [PATCH 01/12] Add basic vector_db_benchmark test, add method to create benchmark running command --- .../__runner__/runner.py | 65 ++++++++++++++----- .../test-suites/vector_db_benchmark_test.yml | 32 +++++++++ 2 files changed, 81 insertions(+), 16 deletions(-) create mode 100644 redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml diff --git a/redis_benchmarks_specification/__runner__/runner.py b/redis_benchmarks_specification/__runner__/runner.py index 2ba57d7d..bc2fe859 100644 --- a/redis_benchmarks_specification/__runner__/runner.py +++ b/redis_benchmarks_specification/__runner__/runner.py @@ -203,6 +203,27 @@ def run_client_runner_logic(args, project_name, project_name_suffix, project_ver override_memtier_test_time, ) +def prepare_vector_db_benchmark_parameters( + clientconfig, + full_benchmark_path, + port, + server, + password, +): + benchmark_command = [] + if port is not None: + benchmark_command.extend(["REDIS_PORT={}".format(port)]) + if password is not None: + benchmark_command.extend(["REDIS_AUTH={}".format(password)]) + benchmark_command.extend([ + full_benchmark_path, + "--host", + f"{server}", + ]) + benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-m-8-ef-16")]) + benchmark_command.extend(["--datasets", clientconfig.get("datasets", "glove-100-angular")]) + benchmark_command_str = " ".join(benchmark_command) + return None, benchmark_command_str def prepare_memtier_benchmark_parameters( clientconfig, @@ -699,22 +720,7 @@ def delete_temporary_files( ) arbitrary_command = False - if "memtier_benchmark" not in benchmark_tool: - # prepare the benchmark command - ( - benchmark_command, - benchmark_command_str, - ) = prepare_benchmark_parameters( - benchmark_config, - full_benchmark_path, - port, - host, - local_benchmark_output_filename, - False, - benchmark_tool_workdir, - False, - ) - else: + if "memtier_benchmark" in benchmark_tool: ( _, benchmark_command_str, @@ -736,6 +742,33 @@ def delete_temporary_files( override_memtier_test_time, override_test_runs, ) + elif "vector_db_benchmark" in benchmark_tool: + ( + _, + benchmark_command_str, + ) = prepare_vector_db_benchmark_parameters( + benchmark_config["clientconfig"], + full_benchmark_path, + port, + host, + password, + ) + else: + # prepare the benchmark command + ( + benchmark_command, + benchmark_command_str, + ) = prepare_benchmark_parameters( + benchmark_config, + full_benchmark_path, + port, + host, + local_benchmark_output_filename, + False, + benchmark_tool_workdir, + False, + ) + if ( arbitrary_command diff --git a/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml b/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml new file mode 100644 index 00000000..e1e4c1fb --- /dev/null +++ b/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml @@ -0,0 +1,32 @@ +version: 0.4 +name: vector_db_benchmark_test +description: Test if vector-db-benchmark can be run with this tool +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-groups: +- redisearch +tested-commands: +- search +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: vector-db-benchmark:test + tool: vector_db_benchmark + dataset: glove-100-angular + engine: redis-m-16-ef-64 + #arguments: '"--data-size" "100" --command "LPUSH __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram' + resources: + requests: + cpus: '4' + memory: 2g + +priority: 38 From ca41ef760e224aceaf73f351f205f306704a96e4 Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Thu, 5 Dec 2024 01:55:32 -0800 Subject: [PATCH 02/12] Add new builder yaml to build redisearch.so --- .../__builder__/builder.py | 11 ++- .../__runner__/runner.py | 11 +-- .../__self_contained_coordinator__/clients.py | 20 +++++ .../__self_contained_coordinator__/runners.py | 39 +++++--- .../self_contained_coordinator.py | 90 ++++++++++++------- ...1.5.0-amd64-debian-bullseye-redisearch.yml | 24 +++++ .../setups/topologies/topologies.yml | 11 +++ .../test-suites/vector_db_benchmark_test.yml | 6 +- 8 files changed, 156 insertions(+), 56 deletions(-) create mode 100644 redis_benchmarks_specification/setups/builders/gcc:11.5.0-amd64-debian-bullseye-redisearch.yml diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index 616ee57e..25cee4c9 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -381,6 +381,9 @@ def builder_process_stream( # build_vars_str, # ) build_command = "sh -c 'make -j'" + print(f"Marcin debug: build_config: {build_config}") + if "build_command" in build_config: + build_command = build_config["build_command"] if b"build_command" in testDetails: build_command = testDetails[b"build_command"].decode() server_name = "redis" @@ -427,7 +430,7 @@ def builder_process_stream( volumes={ redis_temporary_dir: {"bind": "/mnt/redis/", "mode": "rw"}, }, - auto_remove=True, + auto_remove=False, privileged=True, working_dir="/mnt/redis/", command=build_command, @@ -647,9 +650,13 @@ def generate_benchmark_stream_request( build_stream_fields["git_timestamp_ms"] = git_timestamp_ms prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}" + print(f"Marcin debug: build_artifacts: {build_artifacts}") for artifact in build_artifacts: bin_key = f"zipped:artifacts:{prefix}:{id}:{artifact}.zip" - bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read() + if artifact == "redisearch.so": + bin_artifact = open(f"{redis_temporary_dir}modules/redisearch/src/bin/linux-x64-release/search-community/{artifact}", "rb").read() + else: + bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read() bin_artifact_len = len(bytes(bin_artifact)) assert bin_artifact_len > 0 conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS) diff --git a/redis_benchmarks_specification/__runner__/runner.py b/redis_benchmarks_specification/__runner__/runner.py index bc2fe859..c079c084 100644 --- a/redis_benchmarks_specification/__runner__/runner.py +++ b/redis_benchmarks_specification/__runner__/runner.py @@ -210,17 +210,18 @@ def prepare_vector_db_benchmark_parameters( server, password, ): + logging.info(f"Marcin debug prepare_vdb: {full_benchmark_path}") benchmark_command = [] - if port is not None: - benchmark_command.extend(["REDIS_PORT={}".format(port)]) - if password is not None: - benchmark_command.extend(["REDIS_AUTH={}".format(password)]) + # if port is not None: + # benchmark_command.extend(["REDIS_PORT={}".format(port)]) + # if password is not None: + # benchmark_command.extend(["REDIS_AUTH={}".format(password)]) benchmark_command.extend([ full_benchmark_path, "--host", f"{server}", ]) - benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-m-8-ef-16")]) + benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-hnsw-m-16-ef-128")]) benchmark_command.extend(["--datasets", clientconfig.get("datasets", "glove-100-angular")]) benchmark_command_str = " ".join(benchmark_command) return None, benchmark_command_str diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py index e09df382..69739698 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py @@ -22,3 +22,23 @@ def prepare_memtier_benchmark_parameters( benchmark_command_str = benchmark_command_str + " " + clientconfig["arguments"] return None, benchmark_command_str + + +def prepare_vector_db_benchmark_parameters( + clientconfig, + full_benchmark_path, + port, + server, +): + benchmark_command = [] + if port is not None: + benchmark_command.extend(["REDIS_PORT={}".format(port)]) + benchmark_command.extend([ + full_benchmark_path, + "--host", + f"{server}", + ]) + benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-m-8-ef-16")]) + benchmark_command.extend(["--datasets", clientconfig.get("datasets", "glove-100-angular")]) + benchmark_command_str = " ".join(benchmark_command) + return None, benchmark_command_str diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/runners.py b/redis_benchmarks_specification/__self_contained_coordinator__/runners.py index 8bb29922..eb8aa1f5 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/runners.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/runners.py @@ -56,6 +56,7 @@ ) from redis_benchmarks_specification.__self_contained_coordinator__.clients import ( prepare_memtier_benchmark_parameters, + prepare_vector_db_benchmark_parameters, ) from redis_benchmarks_specification.__self_contained_coordinator__.cpuset import ( extract_db_cpu_limit, @@ -347,9 +348,12 @@ def process_self_contained_coordinator_stream( # backwards compatible if benchmark_tool is None: benchmark_tool = "redis-benchmark" - full_benchmark_path = "/usr/local/bin/{}".format( - benchmark_tool - ) + if benchmark_tool == "vector_db_benchmark": + full_benchmark_path = "python /code/run.py" + else: + full_benchmark_path = "/usr/local/bin/{}".format( + benchmark_tool + ) # setup the benchmark ( @@ -370,34 +374,45 @@ def process_self_contained_coordinator_stream( local_benchmark_output_filename ) ) - if "memtier_benchmark" not in benchmark_tool: - # prepare the benchmark command + if "memtier_benchmark" in benchmark_tool: ( - benchmark_command, + _, benchmark_command_str, - ) = prepare_benchmark_parameters( - benchmark_config, + ) = prepare_memtier_benchmark_parameters( + benchmark_config["clientconfig"], full_benchmark_path, redis_proc_start_port, "localhost", local_benchmark_output_filename, - False, benchmark_tool_workdir, - False, ) - else: + elif "vector_db_benchmark" in benchmark_tool: ( _, benchmark_command_str, - ) = prepare_memtier_benchmark_parameters( + ) = prepare_vector_db_benchmark_parameters( benchmark_config["clientconfig"], full_benchmark_path, redis_proc_start_port, "localhost", + ) + else: + # prepare the benchmark command + ( + benchmark_command, + benchmark_command_str, + ) = prepare_benchmark_parameters( + benchmark_config, + full_benchmark_path, + redis_proc_start_port, + "localhost", local_benchmark_output_filename, + False, benchmark_tool_workdir, + False, ) + client_container_image = extract_client_container_image( benchmark_config ) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 2387f98c..6ca6fb70 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -7,6 +7,7 @@ import traceback import re import docker +import docker.errors import redis import os from pathlib import Path @@ -55,6 +56,7 @@ from redis_benchmarks_specification.__runner__.runner import ( print_results_table_stdout, prepare_memtier_benchmark_parameters, + prepare_vector_db_benchmark_parameters, ) from redis_benchmarks_specification.__self_contained_coordinator__.args import ( create_self_contained_coordinator_args, @@ -357,6 +359,7 @@ def self_contained_coordinator_blocking_read( get_runners_consumer_group_name(platform_name), consumer_name ) ) + logging.info(f"Marcin Debug githubevent_conn {github_event_conn}") newTestInfo = github_event_conn.xreadgroup( get_runners_consumer_group_name(platform_name), consumer_name, @@ -364,6 +367,7 @@ def self_contained_coordinator_blocking_read( count=1, block=0, ) + logging.info(f"New test info: {newTestInfo}") if len(newTestInfo[0]) < 2 or len(newTestInfo[0][1]) < 1: stream_id = ">" else: @@ -549,7 +553,7 @@ def process_self_contained_coordinator_stream( ) new_executable = f"{mnt_point}{server_name}-server" logging.info( - "changing executable from {executable} to {new_executable}" + f"changing executable from {executable} to {new_executable}" ) executable = new_executable @@ -906,9 +910,12 @@ def process_self_contained_coordinator_stream( # backwards compatible if benchmark_tool is None: benchmark_tool = "redis-benchmark" - full_benchmark_path = "/usr/local/bin/{}".format( - benchmark_tool - ) + if benchmark_tool == "vector_db_benchmark": + full_benchmark_path = "python /code/run.py" + else: + full_benchmark_path = "/usr/local/bin/{}".format( + benchmark_tool + ) # setup the benchmark ( @@ -929,41 +936,52 @@ def process_self_contained_coordinator_stream( local_benchmark_output_filename ) ) - if "memtier_benchmark" not in benchmark_tool: + if "memtier_benchmark" in benchmark_tool: # prepare the benchmark command ( - benchmark_command, + _, benchmark_command_str, - ) = prepare_benchmark_parameters( - benchmark_config, + arbitrary_command, + ) = prepare_memtier_benchmark_parameters( + benchmark_config["clientconfig"], full_benchmark_path, redis_proc_start_port, "localhost", + None, local_benchmark_output_filename, False, - benchmark_tool_workdir, False, + False, + None, + None, + None, + None, + override_test_time, ) - else: + elif "vector_db_benchmark" in benchmark_tool: ( _, benchmark_command_str, - arbitrary_command, - ) = prepare_memtier_benchmark_parameters( + ) = prepare_vector_db_benchmark_parameters( benchmark_config["clientconfig"], full_benchmark_path, redis_proc_start_port, "localhost", None, + ) + else: + ( + benchmark_command, + benchmark_command_str, + ) = prepare_benchmark_parameters( + benchmark_config, + full_benchmark_path, + redis_proc_start_port, + "localhost", local_benchmark_output_filename, False, + benchmark_tool_workdir, False, - False, - None, - None, - None, - None, - override_test_time, ) client_container_image = extract_client_container_image( @@ -995,23 +1013,27 @@ def process_self_contained_coordinator_stream( ) # run the benchmark benchmark_start_time = datetime.datetime.now() - - client_container_stdout = docker_client.containers.run( - image=client_container_image, - volumes={ - temporary_dir_client: { - "bind": client_mnt_point, - "mode": "rw", + try: + client_container_stdout = docker_client.containers.run( + image=client_container_image, + volumes={ + temporary_dir_client: { + "bind": client_mnt_point, + "mode": "rw", + }, }, - }, - auto_remove=True, - privileged=True, - working_dir=benchmark_tool_workdir, - command=benchmark_command_str, - network_mode="host", - detach=False, - cpuset_cpus=client_cpuset_cpus, - ) + #auto_remove=True, + privileged=True, + working_dir=benchmark_tool_workdir, + command=benchmark_command_str, + network_mode="host", + detach=False, + cpuset_cpus=client_cpuset_cpus, + ) + except docker.errors.ContainerError as e: + logging.info("stdout: {}".format(e.container.logs(stdout=True))) + logging.info("stderr: {}".format(e.container.logs(stderr=True))) + raise e benchmark_end_time = datetime.datetime.now() benchmark_duration_seconds = ( diff --git a/redis_benchmarks_specification/setups/builders/gcc:11.5.0-amd64-debian-bullseye-redisearch.yml b/redis_benchmarks_specification/setups/builders/gcc:11.5.0-amd64-debian-bullseye-redisearch.yml new file mode 100644 index 00000000..64604a0a --- /dev/null +++ b/redis_benchmarks_specification/setups/builders/gcc:11.5.0-amd64-debian-bullseye-redisearch.yml @@ -0,0 +1,24 @@ +version: 0.1 +id: gcc:11.5.0-amd64-debian-bullseye-redisearch +os: debian-bullseye +arch: amd64 +compiler: "gcc" +cpp_compiler: "g++" +kind: docker +build_image: gcc:11.5.0-bullseye +run_image: debian:bullseye +description: "Using GNU Compiler Containers (https://hub.docker.com/_/gcc?tab=description) + pre-configured environment with all the tools required to build with gcc." +build_artifacts: + - redisearch.so + - redis-server +build_command: "sh -c 'BUILD_WITH_MODULES=yes make --ignore-errors -j && cd modules/redisearch/src && make setup && make build && cd -'" +metadata: + compiler: "gcc" + compiler_version: "11.5.0" + os: debian-bullseye + arch: amd64 + +env: + REDIS_CFLAGS: "-g -fno-omit-frame-pointer" + diff --git a/redis_benchmarks_specification/setups/topologies/topologies.yml b/redis_benchmarks_specification/setups/topologies/topologies.yml index 59e9d610..59267010 100644 --- a/redis_benchmarks_specification/setups/topologies/topologies.yml +++ b/redis_benchmarks_specification/setups/topologies/topologies.yml @@ -10,6 +10,17 @@ spec: cpus: "1" memory: "10g" + - name: oss-standalone-with-redisearch + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --loadmodule /mnt/redis/modules/redisearch/src/bin/linux-x64-release/search-community/redisearch.so + resources: + requests: + cpus: "1" + memory: "10g" + - name: oss-standalone-02-io-threads type: oss-standalone redis_topology: diff --git a/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml b/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml index e1e4c1fb..d744c14a 100644 --- a/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml +++ b/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml @@ -14,15 +14,15 @@ tested-groups: tested-commands: - search redis-topologies: -- oss-standalone +- oss-standalone-with-redisearch build-variants: -- gcc:8.5.0-amd64-debian-buster-default +- gcc:11.5.0-amd64-debian-bullseye-redisearch - dockerhub clientconfig: run_image: vector-db-benchmark:test tool: vector_db_benchmark dataset: glove-100-angular - engine: redis-m-16-ef-64 + engine: redis-hnsw-m-16-ef-128 #arguments: '"--data-size" "100" --command "LPUSH __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram' resources: requests: From 89767aa9be4c52627e7b294301138408a3d1794b Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Thu, 5 Dec 2024 01:58:05 -0800 Subject: [PATCH 03/12] Set autoremove to tru --- redis_benchmarks_specification/__builder__/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index 25cee4c9..a90fc403 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -430,7 +430,7 @@ def builder_process_stream( volumes={ redis_temporary_dir: {"bind": "/mnt/redis/", "mode": "rw"}, }, - auto_remove=False, + auto_remove=True, privileged=True, working_dir="/mnt/redis/", command=build_command, From fa718be2c4aa8ed11383e94458971b15d18938f9 Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Tue, 10 Dec 2024 23:00:49 -0800 Subject: [PATCH 04/12] Add loading vector-db-benchmark results into vdb_results variable --- .../__runner__/runner.py | 5 +++-- .../post_processing.py | 18 ++++++++++++++++++ .../self_contained_coordinator.py | 7 +++++++ ...0.5.0-amd64-debian-bullseye-redisearch.yml} | 8 ++++---- .../setups/topologies/topologies.yml | 2 +- .../test-suites/vector_db_benchmark_test.yml | 2 +- 6 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py rename redis_benchmarks_specification/setups/builders/{gcc:11.5.0-amd64-debian-bullseye-redisearch.yml => gcc:10.5.0-amd64-debian-bullseye-redisearch.yml} (61%) diff --git a/redis_benchmarks_specification/__runner__/runner.py b/redis_benchmarks_specification/__runner__/runner.py index c079c084..db945715 100644 --- a/redis_benchmarks_specification/__runner__/runner.py +++ b/redis_benchmarks_specification/__runner__/runner.py @@ -209,8 +209,8 @@ def prepare_vector_db_benchmark_parameters( port, server, password, + client_mnt_point ): - logging.info(f"Marcin debug prepare_vdb: {full_benchmark_path}") benchmark_command = [] # if port is not None: # benchmark_command.extend(["REDIS_PORT={}".format(port)]) @@ -221,9 +221,10 @@ def prepare_vector_db_benchmark_parameters( "--host", f"{server}", ]) - benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-hnsw-m-16-ef-128")]) + benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-test")]) benchmark_command.extend(["--datasets", clientconfig.get("datasets", "glove-100-angular")]) benchmark_command_str = " ".join(benchmark_command) + benchmark_command_str = f"bash -c 'ITERATIONS=1 {benchmark_command_str} && mv /code/results {client_mnt_point}.'" return None, benchmark_command_str def prepare_memtier_benchmark_parameters( diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py b/redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py new file mode 100644 index 00000000..f6078063 --- /dev/null +++ b/redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py @@ -0,0 +1,18 @@ +import os +import json + +def post_process_vector_db(temporary_dir): + results_dir = os.path.join(temporary_dir, 'results') + results = {} + for file in os.listdir(results_dir): + if "upload" in file: + with open(os.path.join(results_dir, file), 'r') as f: + upload_results = json.load(f) + results['upload_time'] = upload_results['results']['upload_time'] + else: + with open(os.path.join(results_dir, file), 'r') as f: + query_results = json.load(f) + results['rps'] = query_results['results']['rps'] + results['precision'] = query_results['results']['mean_precisions'] + results['total_time'] = query_results['results']['total_time'] + return results diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 6ca6fb70..91ecc1af 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -15,6 +15,7 @@ import time from docker.models.containers import Container +from redis_benchmarks_specification.__self_contained_coordinator__.post_processing import post_process_vector_db from redisbench_admin.profilers.profilers_local import ( check_compatible_system_and_kernel_and_prepare_profile, ) @@ -823,6 +824,7 @@ def process_self_contained_coordinator_stream( ) ) if restore_build_artifacts: + print(f"Marcin debug1: Restoring build artifacts") restore_build_artifacts_from_test_details( build_artifacts, github_event_conn, @@ -968,6 +970,7 @@ def process_self_contained_coordinator_stream( redis_proc_start_port, "localhost", None, + client_mnt_point, ) else: ( @@ -1155,6 +1158,10 @@ def process_self_contained_coordinator_stream( and git_timestamp_ms is not None ): datapoint_time_ms = git_timestamp_ms + if "vector_db_benchmark" in benchmark_tool: + print(f"Debug: Post-processing vector-db-benchmark results") + vdb_results = post_process_vector_db(temporary_dir_client) + print(f"Debug: results: {vdb_results}") post_process_benchmark_results( benchmark_tool, local_benchmark_output_filename, diff --git a/redis_benchmarks_specification/setups/builders/gcc:11.5.0-amd64-debian-bullseye-redisearch.yml b/redis_benchmarks_specification/setups/builders/gcc:10.5.0-amd64-debian-bullseye-redisearch.yml similarity index 61% rename from redis_benchmarks_specification/setups/builders/gcc:11.5.0-amd64-debian-bullseye-redisearch.yml rename to redis_benchmarks_specification/setups/builders/gcc:10.5.0-amd64-debian-bullseye-redisearch.yml index 64604a0a..4a24fb5b 100644 --- a/redis_benchmarks_specification/setups/builders/gcc:11.5.0-amd64-debian-bullseye-redisearch.yml +++ b/redis_benchmarks_specification/setups/builders/gcc:10.5.0-amd64-debian-bullseye-redisearch.yml @@ -1,21 +1,21 @@ version: 0.1 -id: gcc:11.5.0-amd64-debian-bullseye-redisearch +id: gcc:10.5.0-amd64-debian-bullseye-redisearch os: debian-bullseye arch: amd64 compiler: "gcc" cpp_compiler: "g++" kind: docker -build_image: gcc:11.5.0-bullseye +build_image: gcc:10.5.0-bullseye run_image: debian:bullseye description: "Using GNU Compiler Containers (https://hub.docker.com/_/gcc?tab=description) pre-configured environment with all the tools required to build with gcc." build_artifacts: - redisearch.so - redis-server -build_command: "sh -c 'BUILD_WITH_MODULES=yes make --ignore-errors -j && cd modules/redisearch/src && make setup && make build && cd -'" +build_command: "sh -c 'apt update && apt install python3-pip -y && BUILD_WITH_MODULES=yes make --ignore-errors -j && cd modules/redisearch/src && make setup && make build && cd -'" metadata: compiler: "gcc" - compiler_version: "11.5.0" + compiler_version: "10.5.0" os: debian-bullseye arch: amd64 diff --git a/redis_benchmarks_specification/setups/topologies/topologies.yml b/redis_benchmarks_specification/setups/topologies/topologies.yml index 59267010..61d8cecd 100644 --- a/redis_benchmarks_specification/setups/topologies/topologies.yml +++ b/redis_benchmarks_specification/setups/topologies/topologies.yml @@ -15,7 +15,7 @@ spec: redis_topology: primaries: 1 replicas: 0 - redis_arguments: --loadmodule /mnt/redis/modules/redisearch/src/bin/linux-x64-release/search-community/redisearch.so + redis_arguments: --loadmodule /mnt/redis/redisearch.so resources: requests: cpus: "1" diff --git a/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml b/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml index d744c14a..14bdad12 100644 --- a/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml +++ b/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml @@ -16,7 +16,7 @@ tested-commands: redis-topologies: - oss-standalone-with-redisearch build-variants: -- gcc:11.5.0-amd64-debian-bullseye-redisearch +- gcc:10.5.0-amd64-debian-bullseye-redisearch - dockerhub clientconfig: run_image: vector-db-benchmark:test From f8cd5b3599bf565bb13e2e91eb58d0fa2e924f14 Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Wed, 11 Dec 2024 02:07:29 -0800 Subject: [PATCH 05/12] Keep results in results_dict --- .../self_contained_coordinator.py | 63 ++++++++++--------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 91ecc1af..2312d104 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -1160,41 +1160,42 @@ def process_self_contained_coordinator_stream( datapoint_time_ms = git_timestamp_ms if "vector_db_benchmark" in benchmark_tool: print(f"Debug: Post-processing vector-db-benchmark results") - vdb_results = post_process_vector_db(temporary_dir_client) - print(f"Debug: results: {vdb_results}") - post_process_benchmark_results( - benchmark_tool, - local_benchmark_output_filename, - datapoint_time_ms, - start_time_str, - client_container_stdout, - None, - ) - full_result_path = local_benchmark_output_filename - if "memtier_benchmark" in benchmark_tool: - full_result_path = "{}/{}".format( - temporary_dir_client, + results_dict = post_process_vector_db(temporary_dir_client) + print(f"Debug: results: {results_dict}") + else: + post_process_benchmark_results( + benchmark_tool, local_benchmark_output_filename, + datapoint_time_ms, + start_time_str, + client_container_stdout, + None, ) - logging.info( - "Reading results json from {}".format( - full_result_path + full_result_path = local_benchmark_output_filename + if "memtier_benchmark" in benchmark_tool: + full_result_path = "{}/{}".format( + temporary_dir_client, + local_benchmark_output_filename, + ) + logging.info( + "Reading results json from {}".format( + full_result_path + ) ) - ) - with open( - full_result_path, - "r", - ) as json_file: - results_dict = json.load(json_file) - print_results_table_stdout( - benchmark_config, - default_metrics, - results_dict, - setup_type, - test_name, - None, - ) + with open( + full_result_path, + "r", + ) as json_file: + results_dict = json.load(json_file) + print_results_table_stdout( + benchmark_config, + default_metrics, + results_dict, + setup_type, + test_name, + None, + ) dataset_load_duration_seconds = 0 try: From 406c29f8b69f2d0259021bd0bad22fc4a14a01bd Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Wed, 11 Dec 2024 05:03:33 -0800 Subject: [PATCH 06/12] Reformat files --- .../__builder__/builder.py | 5 +++- .../__runner__/runner.py | 26 +++++++++---------- .../__self_contained_coordinator__/clients.py | 20 +++++++++----- .../post_processing.py | 15 ++++++----- .../__self_contained_coordinator__/runners.py | 1 - .../self_contained_coordinator.py | 26 ++++++++++++++----- 6 files changed, 58 insertions(+), 35 deletions(-) diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index a90fc403..5e46bd15 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -654,7 +654,10 @@ def generate_benchmark_stream_request( for artifact in build_artifacts: bin_key = f"zipped:artifacts:{prefix}:{id}:{artifact}.zip" if artifact == "redisearch.so": - bin_artifact = open(f"{redis_temporary_dir}modules/redisearch/src/bin/linux-x64-release/search-community/{artifact}", "rb").read() + bin_artifact = open( + f"{redis_temporary_dir}modules/redisearch/src/bin/linux-x64-release/search-community/{artifact}", + "rb", + ).read() else: bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read() bin_artifact_len = len(bytes(bin_artifact)) diff --git a/redis_benchmarks_specification/__runner__/runner.py b/redis_benchmarks_specification/__runner__/runner.py index db945715..71c96cc0 100644 --- a/redis_benchmarks_specification/__runner__/runner.py +++ b/redis_benchmarks_specification/__runner__/runner.py @@ -203,30 +203,31 @@ def run_client_runner_logic(args, project_name, project_name_suffix, project_ver override_memtier_test_time, ) + def prepare_vector_db_benchmark_parameters( - clientconfig, - full_benchmark_path, - port, - server, - password, - client_mnt_point + clientconfig, full_benchmark_path, port, server, password, client_mnt_point ): benchmark_command = [] # if port is not None: # benchmark_command.extend(["REDIS_PORT={}".format(port)]) # if password is not None: # benchmark_command.extend(["REDIS_AUTH={}".format(password)]) - benchmark_command.extend([ - full_benchmark_path, - "--host", - f"{server}", - ]) + benchmark_command.extend( + [ + full_benchmark_path, + "--host", + f"{server}", + ] + ) benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-test")]) - benchmark_command.extend(["--datasets", clientconfig.get("datasets", "glove-100-angular")]) + benchmark_command.extend( + ["--datasets", clientconfig.get("datasets", "glove-100-angular")] + ) benchmark_command_str = " ".join(benchmark_command) benchmark_command_str = f"bash -c 'ITERATIONS=1 {benchmark_command_str} && mv /code/results {client_mnt_point}.'" return None, benchmark_command_str + def prepare_memtier_benchmark_parameters( clientconfig, full_benchmark_path, @@ -771,7 +772,6 @@ def delete_temporary_files( False, ) - if ( arbitrary_command and oss_cluster_api_enabled diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py index 69739698..788b9641 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py @@ -33,12 +33,18 @@ def prepare_vector_db_benchmark_parameters( benchmark_command = [] if port is not None: benchmark_command.extend(["REDIS_PORT={}".format(port)]) - benchmark_command.extend([ - full_benchmark_path, - "--host", - f"{server}", - ]) - benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-m-8-ef-16")]) - benchmark_command.extend(["--datasets", clientconfig.get("datasets", "glove-100-angular")]) + benchmark_command.extend( + [ + full_benchmark_path, + "--host", + f"{server}", + ] + ) + benchmark_command.extend( + ["--engines", clientconfig.get("engines", "redis-m-8-ef-16")] + ) + benchmark_command.extend( + ["--datasets", clientconfig.get("datasets", "glove-100-angular")] + ) benchmark_command_str = " ".join(benchmark_command) return None, benchmark_command_str diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py b/redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py index f6078063..15676c67 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py @@ -1,18 +1,19 @@ import os import json + def post_process_vector_db(temporary_dir): - results_dir = os.path.join(temporary_dir, 'results') + results_dir = os.path.join(temporary_dir, "results") results = {} for file in os.listdir(results_dir): if "upload" in file: - with open(os.path.join(results_dir, file), 'r') as f: + with open(os.path.join(results_dir, file), "r") as f: upload_results = json.load(f) - results['upload_time'] = upload_results['results']['upload_time'] + results["upload_time"] = upload_results["results"]["upload_time"] else: - with open(os.path.join(results_dir, file), 'r') as f: + with open(os.path.join(results_dir, file), "r") as f: query_results = json.load(f) - results['rps'] = query_results['results']['rps'] - results['precision'] = query_results['results']['mean_precisions'] - results['total_time'] = query_results['results']['total_time'] + results["rps"] = query_results["results"]["rps"] + results["precision"] = query_results["results"]["mean_precisions"] + results["total_time"] = query_results["results"]["total_time"] return results diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/runners.py b/redis_benchmarks_specification/__self_contained_coordinator__/runners.py index eb8aa1f5..067c7823 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/runners.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/runners.py @@ -412,7 +412,6 @@ def process_self_contained_coordinator_stream( False, ) - client_container_image = extract_client_container_image( benchmark_config ) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 2312d104..3020e07f 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -15,7 +15,9 @@ import time from docker.models.containers import Container -from redis_benchmarks_specification.__self_contained_coordinator__.post_processing import post_process_vector_db +from redis_benchmarks_specification.__self_contained_coordinator__.post_processing import ( + post_process_vector_db, +) from redisbench_admin.profilers.profilers_local import ( check_compatible_system_and_kernel_and_prepare_profile, ) @@ -1025,7 +1027,7 @@ def process_self_contained_coordinator_stream( "mode": "rw", }, }, - #auto_remove=True, + auto_remove=True, privileged=True, working_dir=benchmark_tool_workdir, command=benchmark_command_str, @@ -1034,8 +1036,16 @@ def process_self_contained_coordinator_stream( cpuset_cpus=client_cpuset_cpus, ) except docker.errors.ContainerError as e: - logging.info("stdout: {}".format(e.container.logs(stdout=True))) - logging.info("stderr: {}".format(e.container.logs(stderr=True))) + logging.info( + "stdout: {}".format( + e.container.logs(stdout=True) + ) + ) + logging.info( + "stderr: {}".format( + e.container.logs(stderr=True) + ) + ) raise e benchmark_end_time = datetime.datetime.now() @@ -1159,8 +1169,12 @@ def process_self_contained_coordinator_stream( ): datapoint_time_ms = git_timestamp_ms if "vector_db_benchmark" in benchmark_tool: - print(f"Debug: Post-processing vector-db-benchmark results") - results_dict = post_process_vector_db(temporary_dir_client) + print( + f"Debug: Post-processing vector-db-benchmark results" + ) + results_dict = post_process_vector_db( + temporary_dir_client + ) print(f"Debug: results: {results_dict}") else: post_process_benchmark_results( From 9b32c5b4d479d76c7bd7a6859febcc31b5018f00 Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Wed, 11 Dec 2024 22:00:27 -0800 Subject: [PATCH 07/12] Reformat self_contained_coordinator.py --- .../self_contained_coordinator.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 3020e07f..deda9d05 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -1019,21 +1019,23 @@ def process_self_contained_coordinator_stream( # run the benchmark benchmark_start_time = datetime.datetime.now() try: - client_container_stdout = docker_client.containers.run( - image=client_container_image, - volumes={ - temporary_dir_client: { - "bind": client_mnt_point, - "mode": "rw", + client_container_stdout = ( + docker_client.containers.run( + image=client_container_image, + volumes={ + temporary_dir_client: { + "bind": client_mnt_point, + "mode": "rw", + }, }, - }, - auto_remove=True, - privileged=True, - working_dir=benchmark_tool_workdir, - command=benchmark_command_str, - network_mode="host", - detach=False, - cpuset_cpus=client_cpuset_cpus, + auto_remove=True, + privileged=True, + working_dir=benchmark_tool_workdir, + command=benchmark_command_str, + network_mode="host", + detach=False, + cpuset_cpus=client_cpuset_cpus, + ) ) except docker.errors.ContainerError as e: logging.info( From d104fe1facf72e37fb402e81eea70f01601366e0 Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Wed, 11 Dec 2024 23:16:13 -0800 Subject: [PATCH 08/12] Use redis-test engine --- .../__self_contained_coordinator__/clients.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py index 788b9641..07c4986d 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py @@ -41,7 +41,7 @@ def prepare_vector_db_benchmark_parameters( ] ) benchmark_command.extend( - ["--engines", clientconfig.get("engines", "redis-m-8-ef-16")] + ["--engines", clientconfig.get("engines", "redis-test")] ) benchmark_command.extend( ["--datasets", clientconfig.get("datasets", "glove-100-angular")] From 745c933c8693ee806eab00a0ccde23c4bdfd78f9 Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Wed, 11 Dec 2024 23:17:55 -0800 Subject: [PATCH 09/12] Checkout runner.py from main --- .../__runner__/runner.py | 67 +++++-------------- 1 file changed, 16 insertions(+), 51 deletions(-) diff --git a/redis_benchmarks_specification/__runner__/runner.py b/redis_benchmarks_specification/__runner__/runner.py index 71c96cc0..2ba57d7d 100644 --- a/redis_benchmarks_specification/__runner__/runner.py +++ b/redis_benchmarks_specification/__runner__/runner.py @@ -204,30 +204,6 @@ def run_client_runner_logic(args, project_name, project_name_suffix, project_ver ) -def prepare_vector_db_benchmark_parameters( - clientconfig, full_benchmark_path, port, server, password, client_mnt_point -): - benchmark_command = [] - # if port is not None: - # benchmark_command.extend(["REDIS_PORT={}".format(port)]) - # if password is not None: - # benchmark_command.extend(["REDIS_AUTH={}".format(password)]) - benchmark_command.extend( - [ - full_benchmark_path, - "--host", - f"{server}", - ] - ) - benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-test")]) - benchmark_command.extend( - ["--datasets", clientconfig.get("datasets", "glove-100-angular")] - ) - benchmark_command_str = " ".join(benchmark_command) - benchmark_command_str = f"bash -c 'ITERATIONS=1 {benchmark_command_str} && mv /code/results {client_mnt_point}.'" - return None, benchmark_command_str - - def prepare_memtier_benchmark_parameters( clientconfig, full_benchmark_path, @@ -723,7 +699,22 @@ def delete_temporary_files( ) arbitrary_command = False - if "memtier_benchmark" in benchmark_tool: + if "memtier_benchmark" not in benchmark_tool: + # prepare the benchmark command + ( + benchmark_command, + benchmark_command_str, + ) = prepare_benchmark_parameters( + benchmark_config, + full_benchmark_path, + port, + host, + local_benchmark_output_filename, + False, + benchmark_tool_workdir, + False, + ) + else: ( _, benchmark_command_str, @@ -745,32 +736,6 @@ def delete_temporary_files( override_memtier_test_time, override_test_runs, ) - elif "vector_db_benchmark" in benchmark_tool: - ( - _, - benchmark_command_str, - ) = prepare_vector_db_benchmark_parameters( - benchmark_config["clientconfig"], - full_benchmark_path, - port, - host, - password, - ) - else: - # prepare the benchmark command - ( - benchmark_command, - benchmark_command_str, - ) = prepare_benchmark_parameters( - benchmark_config, - full_benchmark_path, - port, - host, - local_benchmark_output_filename, - False, - benchmark_tool_workdir, - False, - ) if ( arbitrary_command From d04e6b91b2eeac464562270c2f5adc8e93f8ef9c Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Thu, 12 Dec 2024 01:34:03 -0800 Subject: [PATCH 10/12] Fix prepare_vector_db method --- .../__self_contained_coordinator__/clients.py | 16 +++++++--------- .../self_contained_coordinator.py | 4 +++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py index 07c4986d..42e319a9 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py @@ -25,14 +25,13 @@ def prepare_memtier_benchmark_parameters( def prepare_vector_db_benchmark_parameters( - clientconfig, - full_benchmark_path, - port, - server, + clientconfig, full_benchmark_path, port, server, password, client_mnt_point ): benchmark_command = [] - if port is not None: - benchmark_command.extend(["REDIS_PORT={}".format(port)]) + # if port is not None: + # benchmark_command.extend(["REDIS_PORT={}".format(port)]) + # if password is not None: + # benchmark_command.extend(["REDIS_AUTH={}".format(password)]) benchmark_command.extend( [ full_benchmark_path, @@ -40,11 +39,10 @@ def prepare_vector_db_benchmark_parameters( f"{server}", ] ) - benchmark_command.extend( - ["--engines", clientconfig.get("engines", "redis-test")] - ) + benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-test")]) benchmark_command.extend( ["--datasets", clientconfig.get("datasets", "glove-100-angular")] ) benchmark_command_str = " ".join(benchmark_command) + benchmark_command_str = f"bash -c 'ITERATIONS=1 {benchmark_command_str} && mv /code/results {client_mnt_point}.'" return None, benchmark_command_str diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index deda9d05..14515aea 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -59,7 +59,6 @@ from redis_benchmarks_specification.__runner__.runner import ( print_results_table_stdout, prepare_memtier_benchmark_parameters, - prepare_vector_db_benchmark_parameters, ) from redis_benchmarks_specification.__self_contained_coordinator__.args import ( create_self_contained_coordinator_args, @@ -112,6 +111,9 @@ extract_db_cpu_limit, generate_cpuset_cpus, ) +from redis_benchmarks_specification.__self_contained_coordinator__.clients import ( + prepare_vector_db_benchmark_parameters, +) from redis_benchmarks_specification.__self_contained_coordinator__.docker import ( generate_standalone_redis_server_args, ) From f4b215e61e84faf8019d388d005531334330a207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Po=C5=BAniak?= Date: Mon, 16 Dec 2024 05:21:07 -0800 Subject: [PATCH 11/12] Remove debug prints --- .../self_contained_coordinator.py | 6 ------ .../test-suites/vector_db_benchmark_test.yml | 13 ++++++++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 14515aea..45501952 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -364,7 +364,6 @@ def self_contained_coordinator_blocking_read( get_runners_consumer_group_name(platform_name), consumer_name ) ) - logging.info(f"Marcin Debug githubevent_conn {github_event_conn}") newTestInfo = github_event_conn.xreadgroup( get_runners_consumer_group_name(platform_name), consumer_name, @@ -828,7 +827,6 @@ def process_self_contained_coordinator_stream( ) ) if restore_build_artifacts: - print(f"Marcin debug1: Restoring build artifacts") restore_build_artifacts_from_test_details( build_artifacts, github_event_conn, @@ -1173,13 +1171,9 @@ def process_self_contained_coordinator_stream( ): datapoint_time_ms = git_timestamp_ms if "vector_db_benchmark" in benchmark_tool: - print( - f"Debug: Post-processing vector-db-benchmark results" - ) results_dict = post_process_vector_db( temporary_dir_client ) - print(f"Debug: results: {results_dict}") else: post_process_benchmark_results( benchmark_tool, diff --git a/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml b/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml index 14bdad12..cfc72475 100644 --- a/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml +++ b/redis_benchmarks_specification/test-suites/vector_db_benchmark_test.yml @@ -21,7 +21,7 @@ build-variants: clientconfig: run_image: vector-db-benchmark:test tool: vector_db_benchmark - dataset: glove-100-angular + datasets: laion-img-emb-512-1M-cosine engine: redis-hnsw-m-16-ef-128 #arguments: '"--data-size" "100" --command "LPUSH __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram' resources: @@ -29,4 +29,15 @@ clientconfig: cpus: '4' memory: 2g +exporter: + redistimeseries: + break_by: + - version + - commit + metrics: + - upload_time + - total_time + - rps + - precision + priority: 38 From 876098fd6acb794fd2ca0fdf62070ffc28dc2058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Po=C5=BAniak?= Date: Mon, 16 Dec 2024 05:26:17 -0800 Subject: [PATCH 12/12] Remove debug prints from builder --- redis_benchmarks_specification/__builder__/builder.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index 5e46bd15..ddd34045 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -381,7 +381,6 @@ def builder_process_stream( # build_vars_str, # ) build_command = "sh -c 'make -j'" - print(f"Marcin debug: build_config: {build_config}") if "build_command" in build_config: build_command = build_config["build_command"] if b"build_command" in testDetails: @@ -650,7 +649,6 @@ def generate_benchmark_stream_request( build_stream_fields["git_timestamp_ms"] = git_timestamp_ms prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}" - print(f"Marcin debug: build_artifacts: {build_artifacts}") for artifact in build_artifacts: bin_key = f"zipped:artifacts:{prefix}:{id}:{artifact}.zip" if artifact == "redisearch.so":