[fix] Fixed tool-builder non blocking on event stream (#24)

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.4"
+version = "0.1.5"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>"]
 readme = "Readme.md"

redis_benchmarks_specification/__api__/api.py

@@ -27,6 +27,10 @@
     LOG_DATEFMT,
     LOG_LEVEL,
 )
+from redis_benchmarks_specification.__common__.package import (
+    populate_with_poetry_data,
+    get_version_string,
+)
 
 auth = HTTPBasicAuth()
 
@@ -120,9 +124,11 @@ def base():
 
 
 def main():
+    _, _, project_version = populate_with_poetry_data()
+    project_name = "redis-benchmarks-specification API"
     global conn
     parser = argparse.ArgumentParser(
-        description="redis-benchmarks-specification API",
+        description=get_version_string(project_name, project_version),
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
     logging.info("redis-benchmarks-specification API. v{}".format(__version__))
@@ -157,6 +163,7 @@ def main():
             level=LOG_LEVEL,
             datefmt=LOG_DATEFMT,
         )
+    logging.info(get_version_string(project_name, project_version))
     logging.info(
         "Using redis available at: {}:{}".format(
             GH_REDIS_SERVER_HOST, GH_REDIS_SERVER_PORT

redis_benchmarks_specification/__builder__/builder.py

@@ -25,6 +25,10 @@
     STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
     STREAM_KEYNAME_NEW_BUILD_EVENTS,
 )
+from redis_benchmarks_specification.__common__.package import (
+    populate_with_poetry_data,
+    get_version_string,
+)
 
 
 class ZipFileWithPermissions(ZipFile):
@@ -41,8 +45,10 @@ def _extract_member(self, member, targetpath, pwd):
 
 
 def main():
+    _, _, project_version = populate_with_poetry_data()
+    project_name = "redis-benchmarks-spec builder"
     parser = argparse.ArgumentParser(
-        description="redis-benchmarks-spec builder",
+        description=get_version_string(project_name, project_version),
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
     parser.add_argument(
@@ -60,7 +66,6 @@ def main():
         default=">",
     )
     args = parser.parse_args()
-
     if args.logname is not None:
         print("Writting log to {}".format(args.logname))
         logging.basicConfig(
@@ -77,7 +82,7 @@ def main():
             level=LOG_LEVEL,
             datefmt=LOG_DATEFMT,
         )
-
+    logging.info(get_version_string(project_name, project_version))
     builders_folder = os.path.abspath(args.setups_folder + "/builders")
     logging.info("Using package dir {} for inner file paths".format(builders_folder))
     different_build_specs = os.listdir(builders_folder)
@@ -120,12 +125,13 @@ def main():
         )
 
 
-def builder_consumer_group_create(conn):
+def builder_consumer_group_create(conn, id="$"):
     try:
         conn.xgroup_create(
             STREAM_KEYNAME_GH_EVENTS_COMMIT,
             STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
             mkstream=True,
+            id=id,
         )
         logging.info(
             "Created consumer group named {} to distribute work.".format(
@@ -149,6 +155,7 @@ def builder_process_stream(builders_folder, conn, different_build_specs, previou
         consumer_name,
         {STREAM_KEYNAME_GH_EVENTS_COMMIT: previous_id},
         count=1,
+        block=0,
     )
 
     if len(newTestInfo[0]) < 2 or len(newTestInfo[0][1]) < 1:

redis_benchmarks_specification/__common__/package.py

@@ -1,3 +1,28 @@
+from redis_benchmarks_specification import __version__
+
 import os
+import toml
 
 PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def populate_with_poetry_data():
+    project_name = "redis-benchmarks-specification"
+    project_version = __version__
+    project_description = None
+    try:
+        poetry_data = toml.load("pyproject.toml")["tool"]["poetry"]
+        project_name = poetry_data["name"]
+        project_version = poetry_data["version"]
+        project_description = poetry_data["description"]
+    except FileNotFoundError:
+        pass
+
+    return project_name, project_description, project_version
+
+
+def get_version_string(project_name, project_version):
+    version_str = "{project_name} {project_version}".format(
+        project_name=project_name, project_version=project_version
+    )
+    return version_str

redis_benchmarks_specification/__self_contained_coordinator__/args.py

@@ -13,9 +13,9 @@
 )
 
 
-def create_self_contained_coordinator_args():
+def create_self_contained_coordinator_args(project_name):
     parser = argparse.ArgumentParser(
-        description="redis-benchmarks-spec runner(self-contained)",
+        description=project_name,
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
     parser.add_argument(

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -39,6 +39,10 @@
     GH_REDIS_SERVER_USER,
     STREAM_GH_NEW_BUILD_RUNNERS_CG,
 )
+from redis_benchmarks_specification.__common__.package import (
+    get_version_string,
+    populate_with_poetry_data,
+)
 from redis_benchmarks_specification.__common__.spec import (
     extract_client_cpu_limit,
     extract_client_container_image,
@@ -50,7 +54,11 @@
 
 
 def main():
-    parser = create_self_contained_coordinator_args()
+    _, _, project_version = populate_with_poetry_data()
+    project_name = "redis-benchmarks-spec runner(self-contained)"
+    parser = create_self_contained_coordinator_args(
+        get_version_string(project_name, project_version)
+    )
     args = parser.parse_args()
 
     if args.logname is not None:
@@ -69,6 +77,7 @@ def main():
             level=LOG_LEVEL,
             datefmt=LOG_DATEFMT,
         )
+    logging.info(get_version_string(project_name, project_version))
     topologies_folder = os.path.abspath(args.setups_folder + "/topologies")
     logging.info("Using topologies folder dir {}".format(topologies_folder))
     topologies_files = pathlib.Path(topologies_folder).glob("*.yml")

utils/tests/test_builder.py

@@ -38,7 +38,7 @@ def test_commit_schema_to_stream_then_build():
             conn = redis.StrictRedis(port=16379)
             conn.ping()
             conn.flushall()
-            builder_consumer_group_create(conn)
+            builder_consumer_group_create(conn, "0")
             assert conn.xlen(STREAM_KEYNAME_GH_EVENTS_COMMIT) == 0
 
             result, reply_fields, error_msg = commit_schema_to_stream(