Fixed git_timestamp_ms usage on cli triggering (for historical/past data filling) (#255)

* Fixed coordinator filtering based uppon test regexp

* Fixed git_timestamp_ms usage on cli triggering (for historical/past data filling)

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.205"
+version = "0.1.207"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

redis_benchmarks_specification/__cli__/cli.py

@@ -73,6 +73,10 @@ def get_commits_by_branch(args, repo):
     commits = []
     for commit in repo.iter_commits():
         commit_datetime = commit.committed_datetime
+        git_timestamp_ms = int(
+            datetime.datetime.utcfromtimestamp(commit_datetime.timestamp()).timestamp()
+            * 1000
+        )
         if (
             args.from_date
             <= datetime.datetime.utcfromtimestamp(commit_datetime.timestamp())
@@ -87,6 +91,7 @@ def get_commits_by_branch(args, repo):
                         "git_branch": repo.active_branch.name,
                         "commit_summary": commit.summary,
                         "commit_datetime": str(commit_datetime),
+                        "git_timestamp_ms": git_timestamp_ms,
                     }
                 )
     return commits, total_commits
@@ -107,6 +112,14 @@ def get_commits_by_tags(args, repo):
 
     tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
     for tag in tags:
+
+        git_timestamp_ms = int(
+            datetime.datetime.utcfromtimestamp(
+                tag.commit.committed_datetime.timestamp()
+            ).timestamp()
+            * 1000
+        )
+
         if (
             args.from_date
             <= datetime.datetime.utcfromtimestamp(
@@ -137,6 +150,7 @@ def get_commits_by_tags(args, repo):
                             "git_version": git_version,
                             "commit_summary": tag.commit.summary,
                             "commit_datetime": commit_datetime,
+                            "git_timestamp_ms": git_timestamp_ms,
                         }
                     )
             except packaging.version.InvalidVersion:
@@ -265,6 +279,7 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
     for cdict in commits:
         commit_hash = cdict["git_hash"]
         commit_summary = cdict["commit_summary"]
+        commit_datetime = cdict["commit_datetime"]
         match_obj = re.search(hash_regexp_string, commit_hash)
         if match_obj is None:
             logging.info(
@@ -274,9 +289,7 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
             )
         else:
             print(
-                "Commit with hash: {} added. summary: {}".format(
-                    commit_hash, commit_summary
-                )
+                f"Commit with hash: {commit_hash} from {commit_datetime} added. summary: {commit_summary}"
             )
             filtered_hash_commits.append(cdict)
 

redis_benchmarks_specification/__common__/builder_schema.py

@@ -107,7 +107,8 @@ def get_commit_dict_from_sha(
             commit.commit.author.date.timestamp() * 1000.0
         )
     else:
-        use_git_timestamp = False
+        if "git_timestamp_ms" not in commit_dict:
+            use_git_timestamp = False
     commit_dict["use_git_timestamp"] = str(use_git_timestamp)
     commit_dict["git_hash"] = git_hash
     if gh_branch is not None:

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -532,7 +532,7 @@ def process_self_contained_coordinator_stream(
 
             tests_regexp = ".*"
             if b"tests_regexp" in testDetails:
-                tests_regexp = testDetails[b"tests_regexp"]
+                tests_regexp = testDetails[b"tests_regexp"].decode()
                 logging.info(
                     f"detected a regexp definition on the streamdata {tests_regexp}"
                 )

redis_benchmarks_specification/__spec__/cli.py

@@ -92,13 +92,12 @@ def cli_command_logic(args, project_name, project_version):
     total_commits = 0
     if args.use_branch:
         for commit in repo.iter_commits():
-            if (
-                args.from_date
-                <= datetime.datetime.utcfromtimestamp(
-                    commit.committed_datetime.timestamp()
-                )
-                <= args.to_date
-            ):
+
+            commit_datetime_utc = datetime.datetime.utcfromtimestamp(
+                commit.committed_datetime.timestamp()
+            )
+            git_timestamp_ms = int(commit_datetime_utc.timestamp() * 1000)
+            if args.from_date <= commit_datetime_utc <= args.to_date:
                 if (
                     args.last_n > 0 and total_commits < args.last_n
                 ) or args.last_n == -1:
@@ -109,6 +108,7 @@ def cli_command_logic(args, project_name, project_version):
                             "git_hash": commit.hexsha,
                             "git_branch": repo.active_branch.name,
                             "commit_summary": commit.summary,
+                            "git_timestamp_ms": git_timestamp_ms,
                         }
                     )
     if args.use_tags: