diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py index 95ddfec79cf9..a42ff7b456e9 100644 --- a/scripts/devops_tasks/test_run_samples.py +++ b/scripts/devops_tasks/test_run_samples.py @@ -11,6 +11,7 @@ import os import logging from fnmatch import fnmatch +from subprocess import check_call, CalledProcessError, TimeoutExpired from common_tasks import ( run_check_call, process_glob_string, @@ -20,6 +21,56 @@ root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) +""" +Some samples may "run forever" or need to be timed out after a period of time. Add them here in the following format: +TIMEOUT_SAMPLES = { + "": { + "": (, ) + } +} +""" +TIMEOUT_SAMPLES = { + "azure-eventhub": { + "authenticate_with_sas_token.py": (5), + "receive_batch_with_checkpoint.py": (5), + "recv.py": (5), + "recv_track_last_enqueued_event_prop.py": (5), + "recv_with_checkpoint_by_event_count.py": (5), + "recv_with_checkpoint_by_time_interval.py": (5), + "recv_with_checkpoint_store.py": (5), + "recv_with_custom_starting_position.py": (5), + "sample_code_eventhub.py": (10), + "authenticate_with_sas_token_async.py": (5), + "receive_batch_with_checkpoint_async.py": (5), + "recv_async.py": (5), + "recv_track_last_enqueued_event_prop_async.py": (5), + "recv_with_checkpoint_by_event_count_async.py": (5), + "recv_with_checkpoint_by_time_interval_async.py": (5), + "recv_with_checkpoint_store_async.py": (5), + "recv_with_custom_starting_position_async.py": (5), + "sample_code_eventhub_async.py": (10) + }, + "azure-eventhub-checkpointstoreblob": { + "receive_events_using_checkpoint_store.py": (5), + "receive_events_using_checkpoint_store_storage_api_version.py": (5) + }, + "azure-eventhub-checkpointstoreblob-aio": { + "receive_events_using_checkpoint_store_async.py": (5), + "receive_events_using_checkpoint_store_storage_api_version_async.py": (5) + }, + "azure-servicebus": { + "failure_and_recovery.py": (5), + "receive_iterator_queue.py": (5), + "sample_code_servicebus.py": (30), + "session_pool_receive.py": (20), + "receive_iterator_queue_async.py": (5), + "sample_code_servicebus_async.py": (30), + "session_pool_receive_async.py": (20) + } +} + + +# Add your library + sample file if you do not want a particular sample to be run IGNORED_SAMPLES = { "azure-eventgrid": [ "__init__.py", @@ -29,56 +80,25 @@ "sample_publish_events_to_a_topic_using_sas_credential.py", "sample_publish_events_to_a_topic_using_sas_credential_async.py"], "azure-eventhub": [ - "authenticate_with_sas_token.py", "connection_to_custom_endpoint_address.py", "proxy.py", - "receive_batch_with_checkpoint.py", - "recv.py", - "recv_track_last_enqueued_event_prop.py", - "recv_with_checkpoint_by_event_count.py", - "recv_with_checkpoint_by_time_interval.py", - "recv_with_checkpoint_store.py", - "recv_with_custom_starting_position.py", - "sample_code_eventhub.py", - "authenticate_with_sas_token_async.py", "connection_to_custom_endpoint_address_async.py", "iot_hub_connection_string_receive_async.py", - "proxy_async.py", - "receive_batch_with_checkpoint_async.py", - "recv_async.py", - "recv_track_last_enqueued_event_prop_async.py", - "recv_with_checkpoint_by_event_count_async.py", - "recv_with_checkpoint_by_time_interval_async.py", - "recv_with_checkpoint_store_async.py", - "recv_with_custom_starting_position_async.py", - "sample_code_eventhub_async.py" - ], - "azure-eventhub-checkpointstoreblob": [ - "receive_events_using_checkpoint_store.py", - "receive_events_using_checkpoint_store_storage_api_version.py" - ], - "azure-eventhub-checkpointstoreblob-aio": [ - "receive_events_using_checkpoint_store_async.py", - "receive_events_using_checkpoint_store_storage_api_version_async.py" + "proxy_async.py" ], "azure-servicebus": [ - "failure_and_recovery.py", "mgmt_queue.py", "mgmt_rule.py", "mgmt_subscription.py", "mgmt_topic.py", "proxy.py", "receive_deferred_message_queue.py", - "receive_iterator_queue.py", - "session_pool_receive.py", "mgmt_queue_async.py", "mgmt_rule_async.py", "mgmt_subscription_async.py", "mgmt_topic_async.py", "proxy_async.py", - "receive_deferred_message_queue_async.py", - "receive_iterator_queue_async.py", - "session_pool_receive_async.py" + "receive_deferred_message_queue_async.py" ], "azure-ai-formrecognizer": [ "sample_recognize_receipts_from_url.py", @@ -87,45 +107,117 @@ } +def run_check_call_with_timeout( + command_array, + working_directory, + timeout, + pass_if_timeout, + acceptable_return_codes=[], + always_exit=False +): + """This is copied from common_tasks.py with some additions. + Don't want to break anyone that's using the original code. + """ + try: + logging.info( + "Command Array: {0}, Target Working Directory: {1}".format( + command_array, working_directory + ) + ) + check_call(command_array, cwd=working_directory, timeout=timeout) + except CalledProcessError as err: + if err.returncode not in acceptable_return_codes: + logging.error(err) # , file = sys.stderr + if always_exit: + exit(1) + else: + return err + except TimeoutExpired as err: + if pass_if_timeout: + logging.info( + "Sample timed out successfully" + ) + else: + logging.info( + "Fail: Sample timed out" + ) + return err + + +def execute_sample(sample, samples_errors, timed): + if isinstance(sample, tuple): + sample, timeout, pass_if_timeout = sample + + if sys.version_info < (3, 5) and sample.endswith("_async.py"): + return + + logging.info( + "Testing {}".format(sample) + ) + command_array = [sys.executable, sample] + + if not timed: + errors = run_check_call(command_array, root_dir) + else: + errors = run_check_call_with_timeout( + command_array, root_dir, timeout, pass_if_timeout + ) + + sample_name = os.path.basename(sample) + if errors: + samples_errors.append(sample_name) + logging.info( + "ERROR: {}".format(sample_name) + ) + else: + logging.info( + "SUCCESS: {}.".format(sample_name) + ) + + def run_samples(targeted_package): logging.info("running samples for {}".format(targeted_package)) samples_errors = [] sample_paths = [] + timed_sample_paths = [] + samples_dir_path = os.path.abspath(os.path.join(targeted_package, "samples")) package_name = os.path.basename(targeted_package) + samples_need_timeout = TIMEOUT_SAMPLES.get(package_name, {}) + + # install extra dependencies for samples if needed + try: + with open(samples_dir_path + "/sample_dev_requirements.txt") as sample_dev_reqs: + for dep in sample_dev_reqs.readlines(): + check_call([sys.executable, '-m', 'pip', 'install', dep]) + except FileNotFoundError: + pass for path, subdirs, files in os.walk(samples_dir_path): for name in files: - if fnmatch(name, "*.py") and name not in IGNORED_SAMPLES.get(package_name, []): + if fnmatch(name, "*.py") and name in samples_need_timeout: + timeout = samples_need_timeout[name] + # timeout, pass_if_timeout is True by default if nothing passed in + if isinstance(timeout, tuple): + timeout, pass_if_timeout = timeout + else: + pass_if_timeout = True + timed_sample_paths.append((os.path.abspath(os.path.join(path, name)), timeout, pass_if_timeout)) + elif fnmatch(name, "*.py") and name not in IGNORED_SAMPLES.get(package_name, []): sample_paths.append(os.path.abspath(os.path.join(path, name))) - if not sample_paths: + if not sample_paths and not timed_sample_paths: logging.info( "No samples found in {}".format(targeted_package) ) exit(0) for sample in sample_paths: - if sys.version_info < (3, 5) and sample.endswith("_async.py"): - continue - - logging.info( - "Testing {}".format(sample) - ) - command_array = [sys.executable, sample] - errors = run_check_call(command_array, root_dir, always_exit=False) + execute_sample(sample, samples_errors, timed=False) - sample_name = os.path.basename(sample) - if errors: - samples_errors.append(sample_name) - logging.info( - "ERROR: {}".format(sample_name) - ) - else: - logging.info( - "SUCCESS: {}.".format(sample_name) - ) + for sample in timed_sample_paths: + execute_sample(sample, samples_errors, timed=True) if samples_errors: logging.error("Sample(s) that ran with errors: {}".format(samples_errors)) diff --git a/sdk/eventhub/azure-eventhub/samples/sample_dev_requirements.txt b/sdk/eventhub/azure-eventhub/samples/sample_dev_requirements.txt new file mode 100644 index 000000000000..1fe63a3bd091 --- /dev/null +++ b/sdk/eventhub/azure-eventhub/samples/sample_dev_requirements.txt @@ -0,0 +1,2 @@ +azure-eventhub-checkpointstoreblob +azure-eventhub-checkpointstoreblob-aio \ No newline at end of file