add optional sample timeout to tox sample runner (Azure#17139)

* add optional sample timeout to tox sample runner

* updated sample timeout to install sample specific dependencies reqs

* except with FileNotFoundError

* made timeout a required param and pass_if_timeout default to True

Co-authored-by: Swathi Pillalamarri <swathip@microsoft.com>

Author: kristapratico

scripts/devops_tasks/test_run_samples.py

@@ -11,6 +11,7 @@
 import os
 import logging
 from fnmatch import fnmatch
+from subprocess import check_call, CalledProcessError, TimeoutExpired
 from common_tasks import (
     run_check_call,
     process_glob_string,
@@ -20,6 +21,56 @@
 
 root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))
 
+"""
+Some samples may "run forever" or need to be timed out after a period of time. Add them here in the following format:
+TIMEOUT_SAMPLES = {
+    "<package-name>": {
+        "<sample_file_name.py>": (<timeout (seconds)>, <pass if timeout? (bool, default: True)>)
+    }
+}
+"""
+TIMEOUT_SAMPLES = {
+    "azure-eventhub": {
+        "authenticate_with_sas_token.py": (5),
+        "receive_batch_with_checkpoint.py": (5),
+        "recv.py": (5),
+        "recv_track_last_enqueued_event_prop.py": (5),
+        "recv_with_checkpoint_by_event_count.py": (5),
+        "recv_with_checkpoint_by_time_interval.py": (5),
+        "recv_with_checkpoint_store.py": (5),
+        "recv_with_custom_starting_position.py": (5),
+        "sample_code_eventhub.py": (10),
+        "authenticate_with_sas_token_async.py": (5),
+        "receive_batch_with_checkpoint_async.py": (5),
+        "recv_async.py": (5),
+        "recv_track_last_enqueued_event_prop_async.py": (5),
+        "recv_with_checkpoint_by_event_count_async.py": (5),
+        "recv_with_checkpoint_by_time_interval_async.py": (5),
+        "recv_with_checkpoint_store_async.py": (5),
+        "recv_with_custom_starting_position_async.py": (5),
+        "sample_code_eventhub_async.py": (10)
+    },
+    "azure-eventhub-checkpointstoreblob": {
+        "receive_events_using_checkpoint_store.py": (5),
+        "receive_events_using_checkpoint_store_storage_api_version.py": (5)
+    },
+    "azure-eventhub-checkpointstoreblob-aio": {
+        "receive_events_using_checkpoint_store_async.py": (5),
+        "receive_events_using_checkpoint_store_storage_api_version_async.py": (5)
+    },
+    "azure-servicebus": {
+        "failure_and_recovery.py": (5),
+        "receive_iterator_queue.py": (5),
+        "sample_code_servicebus.py": (30),
+        "session_pool_receive.py": (20),
+        "receive_iterator_queue_async.py": (5),
+        "sample_code_servicebus_async.py": (30),
+        "session_pool_receive_async.py": (20)
+    }
+}
+
+
+# Add your library + sample file if you do not want a particular sample to be run
 IGNORED_SAMPLES = {
     "azure-eventgrid": [
         "__init__.py",
@@ -29,56 +80,25 @@
         "sample_publish_events_to_a_topic_using_sas_credential.py",
         "sample_publish_events_to_a_topic_using_sas_credential_async.py"],
     "azure-eventhub": [
-        "authenticate_with_sas_token.py",
         "connection_to_custom_endpoint_address.py",
         "proxy.py",
-        "receive_batch_with_checkpoint.py",
-        "recv.py",
-        "recv_track_last_enqueued_event_prop.py",
-        "recv_with_checkpoint_by_event_count.py",
-        "recv_with_checkpoint_by_time_interval.py",
-        "recv_with_checkpoint_store.py",
-        "recv_with_custom_starting_position.py",
-        "sample_code_eventhub.py",
-        "authenticate_with_sas_token_async.py",
         "connection_to_custom_endpoint_address_async.py",
         "iot_hub_connection_string_receive_async.py",
-        "proxy_async.py",
-        "receive_batch_with_checkpoint_async.py",
-        "recv_async.py",
-        "recv_track_last_enqueued_event_prop_async.py",
-        "recv_with_checkpoint_by_event_count_async.py",
-        "recv_with_checkpoint_by_time_interval_async.py",
-        "recv_with_checkpoint_store_async.py",
-        "recv_with_custom_starting_position_async.py",
-        "sample_code_eventhub_async.py"
-    ],
-    "azure-eventhub-checkpointstoreblob": [
-        "receive_events_using_checkpoint_store.py",
-        "receive_events_using_checkpoint_store_storage_api_version.py"
-    ],
-    "azure-eventhub-checkpointstoreblob-aio": [
-        "receive_events_using_checkpoint_store_async.py",
-        "receive_events_using_checkpoint_store_storage_api_version_async.py"
+        "proxy_async.py"
     ],
     "azure-servicebus": [
-        "failure_and_recovery.py",
         "mgmt_queue.py",
         "mgmt_rule.py",
         "mgmt_subscription.py",
         "mgmt_topic.py",
         "proxy.py",
         "receive_deferred_message_queue.py",
-        "receive_iterator_queue.py",
-        "session_pool_receive.py",
         "mgmt_queue_async.py",
         "mgmt_rule_async.py",
         "mgmt_subscription_async.py",
         "mgmt_topic_async.py",
         "proxy_async.py",
-        "receive_deferred_message_queue_async.py",
-        "receive_iterator_queue_async.py",
-        "session_pool_receive_async.py"
+        "receive_deferred_message_queue_async.py"
     ],
     "azure-ai-formrecognizer": [
         "sample_recognize_receipts_from_url.py",
@@ -87,45 +107,117 @@
 }
 
 
+def run_check_call_with_timeout(
+    command_array,
+    working_directory,
+    timeout,
+    pass_if_timeout,
+    acceptable_return_codes=[],
+    always_exit=False
+):
+    """This is copied from common_tasks.py with some additions.
+    Don't want to break anyone that's using the original code.
+    """
+    try:
+        logging.info(
+            "Command Array: {0}, Target Working Directory: {1}".format(
+                command_array, working_directory
+            )
+        )
+        check_call(command_array, cwd=working_directory, timeout=timeout)
+    except CalledProcessError as err:
+        if err.returncode not in acceptable_return_codes:
+            logging.error(err)  # , file = sys.stderr
+            if always_exit:
+                exit(1)
+            else:
+                return err
+    except TimeoutExpired as err:
+        if pass_if_timeout:
+            logging.info(
+                "Sample timed out successfully"
+            )
+        else:
+            logging.info(
+                "Fail: Sample timed out"
+            )
+            return err
+
+
+def execute_sample(sample, samples_errors, timed):
+    if isinstance(sample, tuple):
+        sample, timeout, pass_if_timeout = sample
+
+    if sys.version_info < (3, 5) and sample.endswith("_async.py"):
+        return
+
+    logging.info(
+        "Testing {}".format(sample)
+    )
+    command_array = [sys.executable, sample]
+
+    if not timed:
+        errors = run_check_call(command_array, root_dir)
+    else:
+        errors = run_check_call_with_timeout(
+            command_array, root_dir, timeout, pass_if_timeout
+        )
+
+    sample_name = os.path.basename(sample)
+    if errors:
+        samples_errors.append(sample_name)
+        logging.info(
+            "ERROR: {}".format(sample_name)
+        )
+    else:
+        logging.info(
+            "SUCCESS: {}.".format(sample_name)
+        )
+
+
 def run_samples(targeted_package):
     logging.info("running samples for {}".format(targeted_package))
 
     samples_errors = []
     sample_paths = []
+    timed_sample_paths = []
+
     samples_dir_path = os.path.abspath(os.path.join(targeted_package, "samples"))
     package_name = os.path.basename(targeted_package)
+    samples_need_timeout = TIMEOUT_SAMPLES.get(package_name, {})
+
+    # install extra dependencies for samples if needed
+    try:
+        with open(samples_dir_path + "/sample_dev_requirements.txt") as sample_dev_reqs:
+            for dep in sample_dev_reqs.readlines():
+                check_call([sys.executable, '-m', 'pip', 'install', dep])
+    except FileNotFoundError:
+        pass
 
     for path, subdirs, files in os.walk(samples_dir_path):
         for name in files:
-            if fnmatch(name, "*.py") and name not in IGNORED_SAMPLES.get(package_name, []):
+            if fnmatch(name, "*.py") and name in samples_need_timeout:
+                timeout = samples_need_timeout[name]
+                # timeout, pass_if_timeout is True by default if nothing passed in
+                if isinstance(timeout, tuple):
+                    timeout, pass_if_timeout = timeout
+                else:
+                    pass_if_timeout = True
+                timed_sample_paths.append((os.path.abspath(os.path.join(path, name)), timeout, pass_if_timeout))
+            elif fnmatch(name, "*.py") and name not in IGNORED_SAMPLES.get(package_name, []):
                 sample_paths.append(os.path.abspath(os.path.join(path, name)))
 
-    if not sample_paths:
+    if not sample_paths and not timed_sample_paths:
         logging.info(
             "No samples found in {}".format(targeted_package)
         )
         exit(0)
 
     for sample in sample_paths:
-        if sys.version_info < (3, 5) and sample.endswith("_async.py"):
-            continue
-
-        logging.info(
-            "Testing {}".format(sample)
-        )
-        command_array = [sys.executable, sample]
-        errors = run_check_call(command_array, root_dir, always_exit=False)
+        execute_sample(sample, samples_errors, timed=False)
 
-        sample_name = os.path.basename(sample)
-        if errors:
-            samples_errors.append(sample_name)
-            logging.info(
-                "ERROR: {}".format(sample_name)
-            )
-        else:
-            logging.info(
-                "SUCCESS: {}.".format(sample_name)
-            )
+    for sample in timed_sample_paths:
+        execute_sample(sample, samples_errors, timed=True)
 
     if samples_errors:
         logging.error("Sample(s) that ran with errors: {}".format(samples_errors))

sdk/eventhub/azure-eventhub/samples/sample_dev_requirements.txt

@@ -0,0 +1,2 @@
+azure-eventhub-checkpointstoreblob
+azure-eventhub-checkpointstoreblob-aio