Skip to content

Commit

Permalink
add optional sample timeout to tox sample runner (#17139)
Browse files Browse the repository at this point in the history 
* add optional sample timeout to tox sample runner

* updated sample timeout to install sample specific dependencies reqs

* except with FileNotFoundError

* made timeout a required param and pass_if_timeout default to True

Co-authored-by: Swathi Pillalamarri <swathip@microsoft.com>
  • Loading branch information
@kristapratico @swathipil
kristapratico and swathipil authored Mar 11, 2021
1 parent 5243d2e commit 2a8d7c9
Show file tree
Hide file tree
Showing 2 changed files with 147 additions and 53 deletions.
198 changes: 145 additions & 53 deletions scripts/devops_tasks/test_run_samples.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import os
import logging
from fnmatch import fnmatch
from subprocess import check_call, CalledProcessError, TimeoutExpired
from common_tasks import (
run_check_call,
process_glob_string,
Expand All @@ -20,6 +21,56 @@

root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))

"""
Some samples may "run forever" or need to be timed out after a period of time. Add them here in the following format:
TIMEOUT_SAMPLES = {
"<package-name>": {
"<sample_file_name.py>": (<timeout (seconds)>, <pass if timeout? (bool, default: True)>)
}
}
"""
TIMEOUT_SAMPLES = {
"azure-eventhub": {
"authenticate_with_sas_token.py": (5),
"receive_batch_with_checkpoint.py": (5),
"recv.py": (5),
"recv_track_last_enqueued_event_prop.py": (5),
"recv_with_checkpoint_by_event_count.py": (5),
"recv_with_checkpoint_by_time_interval.py": (5),
"recv_with_checkpoint_store.py": (5),
"recv_with_custom_starting_position.py": (5),
"sample_code_eventhub.py": (10),
"authenticate_with_sas_token_async.py": (5),
"receive_batch_with_checkpoint_async.py": (5),
"recv_async.py": (5),
"recv_track_last_enqueued_event_prop_async.py": (5),
"recv_with_checkpoint_by_event_count_async.py": (5),
"recv_with_checkpoint_by_time_interval_async.py": (5),
"recv_with_checkpoint_store_async.py": (5),
"recv_with_custom_starting_position_async.py": (5),
"sample_code_eventhub_async.py": (10)
},
"azure-eventhub-checkpointstoreblob": {
"receive_events_using_checkpoint_store.py": (5),
"receive_events_using_checkpoint_store_storage_api_version.py": (5)
},
"azure-eventhub-checkpointstoreblob-aio": {
"receive_events_using_checkpoint_store_async.py": (5),
"receive_events_using_checkpoint_store_storage_api_version_async.py": (5)
},
"azure-servicebus": {
"failure_and_recovery.py": (5),
"receive_iterator_queue.py": (5),
"sample_code_servicebus.py": (30),
"session_pool_receive.py": (20),
"receive_iterator_queue_async.py": (5),
"sample_code_servicebus_async.py": (30),
"session_pool_receive_async.py": (20)
}
}


# Add your library + sample file if you do not want a particular sample to be run
IGNORED_SAMPLES = {
"azure-eventgrid": [
"__init__.py",
Expand All @@ -29,56 +80,25 @@
"sample_publish_events_to_a_topic_using_sas_credential.py",
"sample_publish_events_to_a_topic_using_sas_credential_async.py"],
"azure-eventhub": [
"authenticate_with_sas_token.py",
"connection_to_custom_endpoint_address.py",
"proxy.py",
"receive_batch_with_checkpoint.py",
"recv.py",
"recv_track_last_enqueued_event_prop.py",
"recv_with_checkpoint_by_event_count.py",
"recv_with_checkpoint_by_time_interval.py",
"recv_with_checkpoint_store.py",
"recv_with_custom_starting_position.py",
"sample_code_eventhub.py",
"authenticate_with_sas_token_async.py",
"connection_to_custom_endpoint_address_async.py",
"iot_hub_connection_string_receive_async.py",
"proxy_async.py",
"receive_batch_with_checkpoint_async.py",
"recv_async.py",
"recv_track_last_enqueued_event_prop_async.py",
"recv_with_checkpoint_by_event_count_async.py",
"recv_with_checkpoint_by_time_interval_async.py",
"recv_with_checkpoint_store_async.py",
"recv_with_custom_starting_position_async.py",
"sample_code_eventhub_async.py"
],
"azure-eventhub-checkpointstoreblob": [
"receive_events_using_checkpoint_store.py",
"receive_events_using_checkpoint_store_storage_api_version.py"
],
"azure-eventhub-checkpointstoreblob-aio": [
"receive_events_using_checkpoint_store_async.py",
"receive_events_using_checkpoint_store_storage_api_version_async.py"
"proxy_async.py"
],
"azure-servicebus": [
"failure_and_recovery.py",
"mgmt_queue.py",
"mgmt_rule.py",
"mgmt_subscription.py",
"mgmt_topic.py",
"proxy.py",
"receive_deferred_message_queue.py",
"receive_iterator_queue.py",
"session_pool_receive.py",
"mgmt_queue_async.py",
"mgmt_rule_async.py",
"mgmt_subscription_async.py",
"mgmt_topic_async.py",
"proxy_async.py",
"receive_deferred_message_queue_async.py",
"receive_iterator_queue_async.py",
"session_pool_receive_async.py"
"receive_deferred_message_queue_async.py"
],
"azure-ai-formrecognizer": [
"sample_recognize_receipts_from_url.py",
Expand All @@ -87,45 +107,117 @@
}


def run_check_call_with_timeout(
command_array,
working_directory,
timeout,
pass_if_timeout,
acceptable_return_codes=[],
always_exit=False
):
"""This is copied from common_tasks.py with some additions.
Don't want to break anyone that's using the original code.
"""
try:
logging.info(
"Command Array: {0}, Target Working Directory: {1}".format(
command_array, working_directory
)
)
check_call(command_array, cwd=working_directory, timeout=timeout)
except CalledProcessError as err:
if err.returncode not in acceptable_return_codes:
logging.error(err) # , file = sys.stderr
if always_exit:
exit(1)
else:
return err
except TimeoutExpired as err:
if pass_if_timeout:
logging.info(
"Sample timed out successfully"
)
else:
logging.info(
"Fail: Sample timed out"
)
return err


def execute_sample(sample, samples_errors, timed):
if isinstance(sample, tuple):
sample, timeout, pass_if_timeout = sample

if sys.version_info < (3, 5) and sample.endswith("_async.py"):
return

logging.info(
"Testing {}".format(sample)
)
command_array = [sys.executable, sample]

if not timed:
errors = run_check_call(command_array, root_dir)
else:
errors = run_check_call_with_timeout(
command_array, root_dir, timeout, pass_if_timeout
)

sample_name = os.path.basename(sample)
if errors:
samples_errors.append(sample_name)
logging.info(
"ERROR: {}".format(sample_name)
)
else:
logging.info(
"SUCCESS: {}.".format(sample_name)
)


def run_samples(targeted_package):
logging.info("running samples for {}".format(targeted_package))

samples_errors = []
sample_paths = []
timed_sample_paths = []

samples_dir_path = os.path.abspath(os.path.join(targeted_package, "samples"))
package_name = os.path.basename(targeted_package)
samples_need_timeout = TIMEOUT_SAMPLES.get(package_name, {})

# install extra dependencies for samples if needed
try:
with open(samples_dir_path + "/sample_dev_requirements.txt") as sample_dev_reqs:
for dep in sample_dev_reqs.readlines():
check_call([sys.executable, '-m', 'pip', 'install', dep])
except FileNotFoundError:
pass

for path, subdirs, files in os.walk(samples_dir_path):
for name in files:
if fnmatch(name, "*.py") and name not in IGNORED_SAMPLES.get(package_name, []):
if fnmatch(name, "*.py") and name in samples_need_timeout:
timeout = samples_need_timeout[name]
# timeout, pass_if_timeout is True by default if nothing passed in
if isinstance(timeout, tuple):
timeout, pass_if_timeout = timeout
else:
pass_if_timeout = True
timed_sample_paths.append((os.path.abspath(os.path.join(path, name)), timeout, pass_if_timeout))
elif fnmatch(name, "*.py") and name not in IGNORED_SAMPLES.get(package_name, []):
sample_paths.append(os.path.abspath(os.path.join(path, name)))

if not sample_paths:
if not sample_paths and not timed_sample_paths:
logging.info(
"No samples found in {}".format(targeted_package)
)
exit(0)

for sample in sample_paths:
if sys.version_info < (3, 5) and sample.endswith("_async.py"):
continue

logging.info(
"Testing {}".format(sample)
)
command_array = [sys.executable, sample]
errors = run_check_call(command_array, root_dir, always_exit=False)
execute_sample(sample, samples_errors, timed=False)

sample_name = os.path.basename(sample)
if errors:
samples_errors.append(sample_name)
logging.info(
"ERROR: {}".format(sample_name)
)
else:
logging.info(
"SUCCESS: {}.".format(sample_name)
)
for sample in timed_sample_paths:
execute_sample(sample, samples_errors, timed=True)

if samples_errors:
logging.error("Sample(s) that ran with errors: {}".format(samples_errors))
Expand Down
2 changes: 2 additions & 0 deletions sdk/eventhub/azure-eventhub/samples/sample_dev_requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
azure-eventhub-checkpointstoreblob
azure-eventhub-checkpointstoreblob-aio

0 comments on commit 2a8d7c9

Please sign in to comment.