Added memtier_benchmark-3Mkeys-load-string-with-512B-values. Included 2,4,8,16,32,64 io-threads setups. (#262)

* Added memtier_benchmark-3Mkeys-load-string-with-512B-values: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 512 Bytes, with 650 clients running sequential SET commands

* Added tests for io-threads setups

* Removed duplicate test

* Fixed missing update tests

Author: fcostaoliveira

redis_benchmarks_specification/__common__/spec.py

@@ -1,4 +1,56 @@
 import math
+import logging
+
+
+def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):
+    redis_configuration_parameters = {}
+    modules_configuration_parameters_map = {}
+    dataset_load_timeout_secs = 120
+    dataset_name = None
+    dbconfig_present = False
+    if dbconfig_keyname in benchmark_config:
+        dbconfig_present = True
+        if type(benchmark_config[dbconfig_keyname]) == list:
+            for k in benchmark_config[dbconfig_keyname]:
+                if "configuration-parameters" in k:
+                    cp = k["configuration-parameters"]
+                    for item in cp:
+                        for k, v in item.items():
+                            redis_configuration_parameters[k] = v
+                if "dataset_load_timeout_secs" in k:
+                    dataset_load_timeout_secs = k["dataset_load_timeout_secs"]
+                if "dataset_name" in k:
+                    dataset_name = k["dataset_name"]
+        if type(benchmark_config[dbconfig_keyname]) == dict:
+            if "configuration-parameters" in benchmark_config[dbconfig_keyname]:
+                cp = benchmark_config[dbconfig_keyname]["configuration-parameters"]
+                for k, v in cp.items():
+                    redis_configuration_parameters[k] = v
+            if "dataset_load_timeout_secs" in benchmark_config[dbconfig_keyname]:
+                dataset_load_timeout_secs = benchmark_config[dbconfig_keyname][
+                    "dataset_load_timeout_secs"
+                ]
+            if "dataset_name" in benchmark_config[dbconfig_keyname]:
+                dataset_name = benchmark_config[dbconfig_keyname]["dataset_name"]
+
+    return (
+        dbconfig_present,
+        dataset_name,
+        redis_configuration_parameters,
+        dataset_load_timeout_secs,
+        modules_configuration_parameters_map,
+    )
+
+
+def extract_redis_configuration_from_topology(topologies_map, topology_spec_name):
+    redis_arguments = ""
+    topology_spec = topologies_map[topology_spec_name]
+    if "redis_arguments" in topology_spec:
+        redis_arguments = topology_spec["redis_arguments"]
+        logging.info(
+            f"extracted redis_arguments: {redis_arguments} from topology: {topology_spec_name}"
+        )
+    return redis_arguments
 
 
 def extract_client_cpu_limit(benchmark_config):

redis_benchmarks_specification/__self_contained_coordinator__/docker.py

@@ -8,7 +8,7 @@
 
 
 def generate_standalone_redis_server_args(
-    binary, port, dbdir, configuration_parameters=None
+    binary, port, dbdir, configuration_parameters=None, redis_arguments=""
 ):
     added_params = ["port", "protected-mode", "dir"]
     # start redis-server
@@ -30,6 +30,10 @@ def generate_standalone_redis_server_args(
                         parameter_value,
                     ]
                 )
+    if redis_arguments != "":
+        redis_arguments_arr = redis_arguments.split(" ")
+        logging.info(f"adding redis arguments {redis_arguments_arr}")
+        command.extend(redis_arguments_arr)
     return command
 
 

redis_benchmarks_specification/__self_contained_coordinator__/runners.py

@@ -32,7 +32,6 @@
 from redisbench_admin.run.run import calculate_client_tool_duration_and_check
 from redisbench_admin.utils.benchmark_config import (
     get_final_benchmark_config,
-    extract_redis_dbconfig_parameters,
 )
 from redisbench_admin.utils.local import get_local_run_full_filename
 from redisbench_admin.utils.results import post_process_benchmark_results
@@ -47,6 +46,7 @@
     extract_client_cpu_limit,
     extract_client_tool,
     extract_client_container_image,
+    extract_redis_dbconfig_parameters,
 )
 from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import (
     restore_build_artifacts_from_test_details,

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -80,7 +80,6 @@
 from redisbench_admin.run.run import calculate_client_tool_duration_and_check
 from redisbench_admin.utils.benchmark_config import (
     get_final_benchmark_config,
-    extract_redis_dbconfig_parameters,
     get_defaults,
 )
 from redisbench_admin.utils.local import get_local_run_full_filename
@@ -95,6 +94,8 @@
     extract_client_cpu_limit,
     extract_client_tool,
     extract_client_container_image,
+    extract_redis_dbconfig_parameters,
+    extract_redis_configuration_from_topology,
 )
 from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import (
     restore_build_artifacts_from_test_details,
@@ -756,13 +757,26 @@ def process_self_contained_coordinator_stream(
                                     )
                                 )
                         for topology_spec_name in benchmark_config["redis-topologies"]:
+                            setup_name = topology_spec_name
+                            setup_type = "oss-standalone"
+                            if topology_spec_name in topologies_map:
+                                topology_spec = topologies_map[topology_spec_name]
+                                setup_type = topology_spec["type"]
+                            logging.info(
+                                f"Running topology named {topology_spec_name} of type {setup_type}"
+                            )
                             test_result = False
                             redis_container = None
                             try:
                                 current_cpu_pos = cpuset_start_pos
                                 ceil_db_cpu_limit = extract_db_cpu_limit(
                                     topologies_map, topology_spec_name
                                 )
+                                redis_arguments = (
+                                    extract_redis_configuration_from_topology(
+                                        topologies_map, topology_spec_name
+                                    )
+                                )
                                 temporary_dir = tempfile.mkdtemp(dir=home)
                                 temporary_dir_client = tempfile.mkdtemp(dir=home)
                                 logging.info(
@@ -776,8 +790,6 @@ def process_self_contained_coordinator_stream(
                                     )
                                 )
 
-                                setup_name = "oss-standalone"
-                                setup_type = "oss-standalone"
                                 tf_triggering_env = "ci"
                                 github_actor = "{}-{}".format(
                                     tf_triggering_env, running_platform
@@ -814,6 +826,7 @@ def process_self_contained_coordinator_stream(
                                     redis_proc_start_port,
                                     mnt_point,
                                     redis_configuration_parameters,
+                                    redis_arguments,
                                 )
                                 command_str = " ".join(command)
                                 db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(
@@ -926,7 +939,7 @@ def process_self_contained_coordinator_stream(
                                         start_time_str,
                                         git_hash,
                                         test_name,
-                                        "oss-standalone",
+                                        setup_name,
                                     )
                                 )
                                 logging.info(

redis_benchmarks_specification/setups/topologies/topologies.yml

@@ -10,6 +10,71 @@ spec:
         cpus: "1"
         memory: "10g"
 
+  - name: oss-standalone-02-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 2 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "3"
+        memory: "10g"
+
+  - name: oss-standalone-04-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 4 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "5"
+        memory: "10g"
+
+  - name: oss-standalone-08-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 8 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "9"
+        memory: "10g"
+
+  - name: oss-standalone-16-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 16 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "17"
+        memory: "10g"
+
+  - name: oss-standalone-32-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 32 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "33"
+        memory: "10g"
+
+  - name: oss-standalone-64-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 64 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "65"
+        memory: "10g"
   - name: oss-standalone-1replica
     type: oss-standalone
     redis_topology:

redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml

@@ -0,0 +1,36 @@
+version: 0.4
+name: memtier_benchmark-3Mkeys-load-string-with-512B-values
+description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 512 Bytes, with 650 clients running sequential SET commands.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 3g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+- oss-standalone-02-io-threads
+- oss-standalone-04-io-threads
+- oss-standalone-08-io-threads
+- oss-standalone-16-io-threads
+- oss-standalone-32-io-threads
+- oss-standalone-64-io-threads
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "512" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 50 -t 13 --hide-histogram'
+  resources:
+    requests:
+      cpus: '13'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 17

utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-string-with-10B-values
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading STRINGs in which the value has a data size of 10 Bytes.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone-02-io-threads
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "10" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  resources:
+    requests:
+      cpus: "1"
+      memory: "2g"
+
+tested-groups:
+- string
+priority: 17

utils/tests/test_data/test-suites/topologies.yml

@@ -0,0 +1,22 @@
+spec:
+  setups:
+  - name: oss-standalone
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    resources:
+      requests:
+        cpus: "1"
+        memory: "10g"
+
+  - name: oss-standalone-02-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 2 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "1"
+        memory: "10g"

utils/tests/test_runner.py

@@ -258,7 +258,7 @@ def test_extract_testsuites():
         ]
     )
     tests = extract_testsuites(args)
-    assert len(tests) == 7
+    assert len(tests) == 9
 
     args = parser.parse_args(
         args=[
@@ -269,7 +269,7 @@ def test_extract_testsuites():
         ]
     )
     tests = extract_testsuites(args)
-    assert len(tests) == 7
+    assert len(tests) == 9
 
     args = parser.parse_args(
         args=[