In case of remote rdb fetch and remote run will copy directly from remote file to DB machine (remove extra hop) (#305)

Author: filipecosta90

README.md

@@ -24,6 +24,7 @@ Current supported benchmark tools:
 - [tsbs](https://github.com/RedisTimeSeries/tsbs)
 - [redisgraph-benchmark-go](https://github.com/RedisGraph/redisgraph-benchmark-go)
 - [ftsb_redisearch](https://github.com/RediSearch/ftsb)
+- [ann-benchmarks](https://github.com/RedisAI/ann-benchmarks)
 - [SOON][aibench](https://github.com/RedisAI/aibench)
 
 ## Installation

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.7.4"
+version = "0.7.5"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "README.md"

redisbench_admin/run_remote/remote_client.py

@@ -133,6 +133,18 @@ def run_remote_client_tool(
             stdout,
             tmp,
         )
+    else:
+        logging.error(
+            "Given the remote tool failed, inspecting the remote results file content ({})".format(
+                remote_results_file
+            )
+        )
+        command = ["cat {}".format(remote_results_file)]
+        recv_exit_status, stdout, stderr = execute_remote_commands(
+            client_public_ip, username, private_key, [command], client_ssh_port
+        )[0]
+        logging.warning("Remote results file content: {}".format(stdout))
+
     return (
         artifact_version,
         benchmark_duration_seconds,

redisbench_admin/utils/local.py

@@ -21,6 +21,7 @@ def check_dataset_local_requirements(
     dbconfig_keyname="dbconfig",
     number_primaries=1,
     is_cluster=False,
+    is_remote=False,
 ):
     dataset = None
     dataset_name = None
@@ -34,7 +35,7 @@ def check_dataset_local_requirements(
                 dataset_name = k["dataset_name"]
         if dataset is not None:
             full_path = check_if_needs_remote_fetch(
-                dataset, datasets_localtemp_dir, dirname
+                dataset, datasets_localtemp_dir, dirname, None, is_remote
             )
 
             if is_cluster is False:

redisbench_admin/utils/remote.py

@@ -177,6 +177,7 @@ def check_dataset_remote_requirements(
         "dbconfig",
         number_primaries,
         False,
+        True,
     )
     if dataset is not None:
         logging.info(
@@ -196,15 +197,28 @@ def check_dataset_remote_requirements(
                 fullpath, server_public_ip, remote_dataset_file
             )
         )
-        res = copy_file_to_remote_setup(
-            server_public_ip,
-            username,
-            private_key,
-            fullpath,
-            remote_dataset_file,
-            None,
-            db_ssh_port,
-        )
+        if "https" in dataset:
+            logging.info(
+                "Given dataset is a remote one ( {} ), copying it directly to DB machine ( {} ).".format(
+                    dataset,
+                    remote_dataset_file,
+                )
+            )
+            commands = []
+            commands.append("wget -O {} {}".format(remote_dataset_file, dataset))
+            execute_remote_commands(
+                server_public_ip, username, private_key, commands, db_ssh_port
+            )
+        else:
+            res = copy_file_to_remote_setup(
+                server_public_ip,
+                username,
+                private_key,
+                fullpath,
+                remote_dataset_file,
+                None,
+                db_ssh_port,
+            )
         if is_cluster:
             commands = []
             for master_shard_id in range(2, number_primaries + 1):