Added APPEND/INCRBY/INCRBYFLOAT/SETRANGE benchmarks (#267)

* Enabled running forks source built benchmarks

* Fixed server_name Null check

* Enabled passing baseline/comparison hash and github_repo to ensure proper data filtering on compare. Removed refs/heads/ usage from builder

* skipping cli builder test on ci

* Added --baseline-target-branch and --comparison-target-branch to the compare tool

* Added GEOPOS and GEOSEARCH WITHCOORD new benchmarks

* Included the connection setup benchmark using HELLO

* Bumping version from 0.1.218 to 0.1.219

* Added APPEND/INCRBY/INCRBYFLOAT/SETRANGE benchmarks

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.219"
+version = "0.1.222"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

redis_benchmarks_specification/__builder__/builder.py

@@ -644,11 +644,11 @@ def generate_benchmark_stream_request(
         build_stream_fields["git_version"] = git_version
     if git_timestamp_ms is not None:
         build_stream_fields["git_timestamp_ms"] = git_timestamp_ms
+
+    prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}"
     for artifact in build_artifacts:
-        bin_key = "zipped:artifacts:{}:{}.zip".format(id, artifact)
-        bin_artifact = open(
-            "{}src/{}".format(redis_temporary_dir, artifact), "rb"
-        ).read()
+        bin_key = f"zipped:artifacts:{prefix}:{id}:{artifact}.zip"
+        bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read()
         bin_artifact_len = len(bytes(bin_artifact))
         assert bin_artifact_len > 0
         conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS)

redis_benchmarks_specification/__compare__/args.py

@@ -44,6 +44,8 @@ def create_compare_arguments(parser):
     )
     parser.add_argument("--github_repo", type=str, default="redis")
     parser.add_argument("--github_org", type=str, default="redis")
+    parser.add_argument("--baseline_github_org", type=str, default="")
+    parser.add_argument("--comparison_github_org", type=str, default="")
     parser.add_argument("--triggering_env", type=str, default="ci")
     parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN)
     parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="")

redis_benchmarks_specification/__compare__/compare.py

@@ -256,6 +256,8 @@ def compare_command_logic(args, project_name, project_version):
     comparison_target_branch = args.comparison_target_branch
     baseline_github_repo = args.baseline_github_repo
     comparison_github_repo = args.comparison_github_repo
+    baseline_github_org = args.baseline_github_org
+    comparison_github_org = args.comparison_github_org
     baseline_hash = args.baseline_hash
     comparison_hash = args.comparison_hash
 
@@ -324,6 +326,8 @@ def compare_command_logic(args, project_name, project_version):
         comparison_github_repo,
         baseline_target_branch,
         comparison_target_branch,
+        baseline_github_org,
+        comparison_github_org,
     )
     prepare_regression_comment(
         auto_approve,
@@ -555,6 +559,8 @@ def compute_regression_table(
     comparison_github_repo="redis",
     baseline_target_branch=None,
     comparison_target_branch=None,
+    baseline_github_org="redis",
+    comparison_github_org="redis",
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -657,6 +663,8 @@ def compute_regression_table(
         running_platform,
         baseline_github_repo,
         comparison_github_repo,
+        baseline_github_org,
+        comparison_github_org,
     )
     logging.info(
         "Printing differential analysis between {} and {}".format(
@@ -909,6 +917,8 @@ def from_rts_to_regression_table(
     running_platform=None,
     baseline_github_repo="redis",
     comparison_github_repo="redis",
+    baseline_github_org="redis",
+    comparison_github_org="redis",
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
     table_full = []
@@ -942,6 +952,8 @@ def from_rts_to_regression_table(
             "github_repo={}".format(baseline_github_repo),
             "triggering_env={}".format(tf_triggering_env),
         ]
+        if baseline_github_org != "":
+            filters_baseline.append(f"github_org={baseline_github_org}")
         if running_platform is not None:
             filters_baseline.append("running_platform={}".format(running_platform))
         filters_comparison = [
@@ -952,6 +964,8 @@ def from_rts_to_regression_table(
             "github_repo={}".format(comparison_github_repo),
             "triggering_env={}".format(tf_triggering_env),
         ]
+        if comparison_github_org != "":
+            filters_baseline.append(f"github_org={comparison_github_org}")
         if "hash" not in by_str_baseline:
             filters_baseline.append("hash==")
         if "hash" not in by_str_comparison:

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-string-append-1-100B
+description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 1 Bytes for each key initially. Then we append to the same keys with values ranging from 1 to 100B.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '"--data-size" "1" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- append
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 --data-size-range=1-100 --command "APPEND __key__ __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 1

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby.yml

@@ -0,0 +1,28 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-string-incrby
+description: Runs memtier_benchmark, for a keyspace length of 1M keys doing incrby of 1.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  resources:
+    requests:
+      memory: 1g
+tested-groups:
+- string
+tested-commands:
+- incrby
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 --command "INCRBY __key__ 1" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 21

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat.yml

@@ -0,0 +1,28 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-string-incrbyfloat
+description: Runs memtier_benchmark, for a keyspace length of 1M keys doing incrby of 0.01.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  resources:
+    requests:
+      memory: 1g
+tested-groups:
+- string
+tested-commands:
+- incrbyfloat
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 --command "INCRBYFLOAT __key__ 0.01" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 21

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-string-setrange-100B
+description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 100 Bytes for each key.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '"--data-size" "100" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- setrange
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 --data-size-range=1-95 --command "SETRANGE __key__ 5 __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 1

redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml

@@ -1,6 +1,14 @@
 version: 0.4
 name: memtier_benchmark-connection-hello
 description: 'Runs memtier_benchmark, for no keyspace, benchmarking the connection setup scenario using HELLO command.'
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
 tested-groups:
 - connection
 tested-commands: