Enable TLS benchmarks on client-runner and memtier_benchmark (#77)

* Enable TLS benchmarks on client-runner and memtier_benchmark

* Fixed flake8 error

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.19"
+version = "0.1.20"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

redis_benchmarks_specification/__runner__/args.py

@@ -76,4 +76,31 @@ def create_client_runner_args(project_name):
         action="store_true",
         help="At the end of every test send a FLUSHALL",
     )
+    parser.add_argument(
+        "--tls",
+        default=False,
+        action="store_true",
+        help="Enable SSL/TLS transport security",
+    )
+    parser.add_argument(
+        "--tls-skip-verify",
+        default=False,
+        action="store_true",
+        help="Skip verification of server certificate",
+    )
+    parser.add_argument(
+        "--cert",
+        default="",
+        help="Use specified client certificate for TLS",
+    )
+    parser.add_argument(
+        "--key",
+        default="",
+        help="Use specified private key for TLS",
+    )
+    parser.add_argument(
+        "--cacert",
+        default="",
+        help="Use specified CA certs bundle for TLS",
+    )
     return parser

redis_benchmarks_specification/__runner__/runner.py

@@ -3,11 +3,11 @@
 import logging
 import os
 import pathlib
-import shutil
 import sys
 import tempfile
 import traceback
 from pathlib import Path
+import shutil
 
 import docker
 import redis
@@ -110,6 +110,11 @@ def main():
             exit(1)
 
     running_platform = args.platform_name
+    tls_enabled = args.tls
+    tls_skip_verify = args.tls_skip_verify
+    tls_cert = args.cert
+    tls_key = args.key
+    tls_cacert = args.cacert
     docker_client = docker.from_env()
     home = str(Path.home())
     logging.info("Running the benchmark specs.")
@@ -124,6 +129,11 @@ def main():
         testsuite_spec_files,
         {},
         running_platform,
+        tls_enabled,
+        tls_skip_verify,
+        tls_cert,
+        tls_key,
+        tls_cacert,
     )
 
 
@@ -134,6 +144,11 @@ def prepare_memtier_benchmark_parameters(
     server,
     local_benchmark_output_filename,
     oss_cluster_api_enabled,
+    tls_enabled=False,
+    tls_skip_verify=False,
+    tls_cert=None,
+    tls_key=None,
+    tls_cacert=None,
 ):
     benchmark_command = [
         full_benchmark_path,
@@ -144,6 +159,17 @@ def prepare_memtier_benchmark_parameters(
         "--json-out-file",
         local_benchmark_output_filename,
     ]
+    if tls_enabled:
+        benchmark_command.append("--tls")
+        if tls_cert is not None and tls_cert != "":
+            benchmark_command.extend(["--cert", tls_cert])
+        if tls_key is not None and tls_key != "":
+            benchmark_command.extend(["--key", tls_key])
+        if tls_cacert is not None and tls_cacert != "":
+            benchmark_command.extend(["--cacert", tls_cacert])
+        if tls_skip_verify:
+            benchmark_command.append("--tls-skip-verify")
+
     if oss_cluster_api_enabled is True:
         benchmark_command.append("--cluster-mode")
     benchmark_command_str = " ".join(benchmark_command)
@@ -163,14 +189,29 @@ def process_self_contained_coordinator_stream(
     testsuite_spec_files,
     topologies_map,
     running_platform,
+    tls_enabled=False,
+    tls_skip_verify=False,
+    tls_cert=None,
+    tls_key=None,
+    tls_cacert=None,
 ):
     overall_result = True
     total_test_suite_runs = 0
     for test_file in testsuite_spec_files:
         client_containers = []
 
         with open(test_file, "r") as stream:
-            benchmark_config, test_name = get_final_benchmark_config(None, stream, "")
+            _, benchmark_config, test_name = get_final_benchmark_config(
+                None, stream, ""
+            )
+
+            if tls_enabled:
+                test_name = test_name + "-tls"
+                logging.info(
+                    "Given that TLS is enabled, appending -tls to the testname: {}.".format(
+                        test_name
+                    )
+                )
 
             for topology_spec_name in benchmark_config["redis-topologies"]:
                 test_result = False
@@ -189,8 +230,22 @@ def process_self_contained_coordinator_stream(
 
                     port = args.db_server_port
                     host = args.db_server_host
-                    r = redis.StrictRedis(host=host, port=port)
+
+                    ssl_cert_reqs = "required"
+                    if tls_skip_verify:
+                        ssl_cert_reqs = None
+                    r = redis.StrictRedis(
+                        host=host,
+                        port=port,
+                        ssl=tls_enabled,
+                        ssl_cert_reqs=ssl_cert_reqs,
+                        ssl_keyfile=tls_key,
+                        ssl_certfile=tls_cert,
+                        ssl_ca_certs=tls_cacert,
+                        ssl_check_hostname=False,
+                    )
                     r.ping()
+
                     ceil_client_cpu_limit = extract_client_cpu_limit(benchmark_config)
                     client_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(
                         ceil_client_cpu_limit, current_cpu_pos
@@ -202,6 +257,17 @@ def process_self_contained_coordinator_stream(
                     benchmark_tool_workdir = client_mnt_point
 
                     metadata = {}
+                    if tls_enabled:
+                        metadata["tls"] = "true"
+                        if tls_cert is not None and tls_cert != "":
+                            _, tls_cert = cp_to_workdir(temporary_dir_client, tls_cert)
+                        if tls_cacert is not None and tls_cacert != "":
+                            _, tls_cacert = cp_to_workdir(
+                                temporary_dir_client, tls_cacert
+                            )
+                        if tls_key is not None and tls_key != "":
+                            _, tls_key = cp_to_workdir(temporary_dir_client, tls_key)
+
                     if "preload_tool" in benchmark_config["dbconfig"]:
                         data_prepopulation_step(
                             benchmark_config,
@@ -213,6 +279,11 @@ def process_self_contained_coordinator_stream(
                             temporary_dir_client,
                             test_name,
                             host,
+                            tls_enabled,
+                            tls_skip_verify,
+                            tls_cert,
+                            tls_key,
+                            tls_cacert,
                         )
 
                     benchmark_tool = extract_client_tool(benchmark_config)
@@ -263,7 +334,12 @@ def process_self_contained_coordinator_stream(
                             port,
                             host,
                             local_benchmark_output_filename,
-                            benchmark_tool_workdir,
+                            False,
+                            tls_enabled,
+                            tls_skip_verify,
+                            tls_cert,
+                            tls_key,
+                            tls_cacert,
                         )
 
                     client_container_image = extract_client_container_image(
@@ -389,6 +465,18 @@ def process_self_contained_coordinator_stream(
                 overall_result &= test_result
 
 
+def cp_to_workdir(benchmark_tool_workdir, srcfile):
+    head, filename = os.path.split(srcfile)
+    dstfile = "{}/{}".format(benchmark_tool_workdir, filename)
+    shutil.copyfile(srcfile, dstfile)
+    logging.info(
+        "Copying to workdir the following file {}. Final workdir file {}".format(
+            srcfile, dstfile
+        )
+    )
+    return dstfile, filename
+
+
 def data_prepopulation_step(
     benchmark_config,
     benchmark_tool_workdir,
@@ -399,6 +487,11 @@ def data_prepopulation_step(
     temporary_dir,
     test_name,
     host,
+    tls_enabled=False,
+    tls_skip_verify=False,
+    tls_cert=None,
+    tls_key=None,
+    tls_cacert=None,
 ):
     # setup the benchmark
     (
@@ -426,6 +519,11 @@ def data_prepopulation_step(
             host,
             local_benchmark_output_filename,
             False,
+            tls_enabled,
+            tls_skip_verify,
+            tls_cert,
+            tls_key,
+            tls_cacert,
         )
 
         logging.info(

utils/tests/test_runner.py

@@ -0,0 +1,142 @@
+import yaml
+
+from redis_benchmarks_specification.__common__.spec import extract_client_tool
+from redis_benchmarks_specification.__runner__.runner import (
+    prepare_memtier_benchmark_parameters,
+)
+
+
+def test_prepare_memtier_benchmark_parameters():
+    with open(
+        "./redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-100B-expire-use-case.yml",
+        "r",
+    ) as yml_file:
+        benchmark_config = yaml.safe_load(yml_file)
+        client_tool = extract_client_tool(benchmark_config)
+        assert client_tool == "memtier_benchmark"
+        local_benchmark_output_filename = "1.json"
+        oss_api_enabled = False
+        (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters(
+            benchmark_config["clientconfig"],
+            client_tool,
+            12000,
+            "localhost",
+            local_benchmark_output_filename,
+            oss_api_enabled,
+        )
+        assert (
+            benchmark_command_str
+            == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+        )
+        oss_api_enabled = True
+        (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters(
+            benchmark_config["clientconfig"],
+            client_tool,
+            12000,
+            "localhost",
+            local_benchmark_output_filename,
+            oss_api_enabled,
+        )
+        assert (
+            benchmark_command_str
+            == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --cluster-mode "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+        )
+
+        oss_api_enabled = False
+        tls_enabled = False
+        tls_skip_verify = True
+        tls_cert = None
+        tls_key = None
+
+        # ensure that when tls is disabled we dont change the args
+        (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters(
+            benchmark_config["clientconfig"],
+            client_tool,
+            12000,
+            "localhost",
+            local_benchmark_output_filename,
+            oss_api_enabled,
+            tls_enabled,
+            tls_skip_verify,
+            tls_cert,
+            tls_key,
+        )
+        assert (
+            benchmark_command_str
+            == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+        )
+
+        tls_enabled = True
+        (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters(
+            benchmark_config["clientconfig"],
+            client_tool,
+            12000,
+            "localhost",
+            local_benchmark_output_filename,
+            oss_api_enabled,
+            tls_enabled,
+            tls_skip_verify,
+            tls_cert,
+            tls_key,
+        )
+        assert (
+            benchmark_command_str
+            == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --tls-skip-verify "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+        )
+
+        tls_skip_verify = False
+        (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters(
+            benchmark_config["clientconfig"],
+            client_tool,
+            12000,
+            "localhost",
+            local_benchmark_output_filename,
+            oss_api_enabled,
+            tls_enabled,
+            tls_skip_verify,
+            tls_cert,
+            tls_key,
+        )
+        assert (
+            benchmark_command_str
+            == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+        )
+
+        tls_skip_verify = False
+        tls_cert = "cert.file"
+        tls_key = "key.file"
+        (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters(
+            benchmark_config["clientconfig"],
+            client_tool,
+            12000,
+            "localhost",
+            local_benchmark_output_filename,
+            oss_api_enabled,
+            tls_enabled,
+            tls_skip_verify,
+            tls_cert,
+            tls_key,
+        )
+        assert (
+            benchmark_command_str
+            == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --cert cert.file --key key.file "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+        )
+
+        tls_cacert = "cacert.file"
+        (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters(
+            benchmark_config["clientconfig"],
+            client_tool,
+            12000,
+            "localhost",
+            local_benchmark_output_filename,
+            oss_api_enabled,
+            tls_enabled,
+            tls_skip_verify,
+            tls_cert,
+            tls_key,
+            tls_cacert,
+        )
+        assert (
+            benchmark_command_str
+            == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --cert cert.file --key key.file --cacert cacert.file "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+        )