Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add retries to Azure Devops Reporter scripts #7399

Merged
merged 2 commits into from
May 18, 2021
Merged

Add retries to Azure Devops Reporter scripts #7399

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
165ef7a
https://github.com/dotnet/arcade/issues/7371 - retry in the case of H…
MattGal May 17, 2021
9e21335
Oops
MattGal May 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 57 additions & 21 deletions ...icrosoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/azure_devops_result_publisher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
import base64
import os
import logging
import time
from typing import Iterable, Mapping, List, Dict, Optional, Tuple
from builtins import str as text
from azure.devops.connection import Connection
from msrest.authentication import BasicTokenAuthentication, BasicAuthentication
from azure.devops.v5_1.test import TestClient
from azure.devops.v5_1.test.models import TestCaseResult, TestAttachmentRequestModel, TestSubResult
from azure.devops.exceptions import AzureDevOpsClientRequestError

from helpers import get_env
from defs import TestResult

log = logging.getLogger(__name__)


class AzureDevOpsTestResultPublisher:
def __init__(self, collection_uri, access_token, team_project, test_run_id=None):
"""
Expand All @@ -38,15 +41,15 @@ def is_data_driven_test(self, r: str) -> bool:
return r.endswith(")")

def get_ddt_base_name(self, r: str) -> str:
return r.split('(',1)[0]
return r.split('(', 1)[0]

def send_attachment(self, test_client, attachment, published_result):
try:
# Python 3 will throw a TypeError exception because b64encode expects bytes
stream=base64.b64encode(text(attachment.text))
stream = base64.b64encode(text(attachment.text))
except TypeError:
# stream has to be a string but b64encode takes and returns bytes on Python 3
stream=base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8")
stream = base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8")

test_client.create_test_result_attachment(
TestAttachmentRequestModel(
Expand All @@ -55,19 +58,51 @@ def send_attachment(self, test_client, attachment, published_result):
), self.team_project, self.test_run_id, published_result.id)

def send_sub_attachment(self, test_client, attachment, published_result, sub_result_id):
stream=base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8")
stream = base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8")

test_client.create_test_sub_result_attachment(
TestAttachmentRequestModel(
file_name=text(attachment.name),
stream=stream,
), self.team_project, self.test_run_id, published_result.id, sub_result_id)

def publish_results(self, test_case_results: Iterable[TestCaseResult], test_result_order: Dict[str, List[str]], results_with_attachments: Mapping[str, TestResult]) -> None:
def publish_results(self, test_case_results: Iterable[TestCaseResult], test_result_order: Dict[str, List[str]],
results_with_attachments: Mapping[str, TestResult]) -> None:
connection = self.get_connection()
test_client = connection.get_client("azure.devops.v5_1.test.TestClient") # type: TestClient

published_results = test_client.add_test_results_to_test_run(list(test_case_results), self.team_project, self.test_run_id) # type: List[TestCaseResult]
tries_left = 10
succeeded = False
test_run_ended = False

while tries_left > 0 and not test_run_ended and not succeeded:
try:
published_results = test_client. \
add_test_results_to_test_run(list(test_case_results),
self.team_project,
self.test_run_id) # type: List[TestCaseResult]
succeeded = True

except AzureDevOpsClientRequestError as ex:
# Odd syntax here is to deal with checking substrings of the list of args in this exception
hit_503 = len([element for element in ex.args if ('invalid status code of 503' in element)]) != 0
test_run_ended = len([element for element in ex.args if ('It may have been deleted' in element)]) != 0
if hit_503:
tries_left -= 1
log.warning("Hit HTTP 503 from Azure DevOps. Will wait three seconds and try again.")
time.sleep(3)
elif test_run_ended: # Not exceptional, don't retry.
tries_left = 0
else:
raise ex

if test_run_ended:
log.info("Test run has ended, skipping attaching results as it would fail.")
return

# !succeeded means a 503 and not succeeding after 10 tries (otherwise we threw already), so give a nice error
if not succeeded:
raise Exception('Failed to report test results to Azure Dev Ops after retrying. Please contact dnceng.')

for published_result in published_results:

Expand All @@ -89,11 +124,12 @@ def publish_results(self, test_case_results: Iterable[TestCaseResult], test_resu
# is true , and uses the order of test names recorded earlier to look-up the attachments.
elif published_result.sub_results is not None:
sub_results_order = test_result_order[published_result.automated_test_name]

# Sanity check
if len(sub_results_order) != len(published_result.sub_results):
log.warning("Returned subresults list length does not match expected. Attachments may not pair correctly.")

log.warning(
"Returned subresults list length does not match expected. Attachments may not pair correctly.")

for (name, sub_result) in zip(sub_results_order, published_result.sub_results):
if name in results_with_attachments:
result = results_with_attachments.get(name)
Expand All @@ -111,25 +147,25 @@ def convert_to_sub_test(r: TestResult) -> Optional[TestSubResult]:
return TestSubResult(
comment=comment,
display_name=text(r.name),
duration_in_ms=r.duration_seconds*1000,
duration_in_ms=r.duration_seconds * 1000,
outcome="Passed"
)
)
if r.result == "Fail":
return TestSubResult(
comment=comment,
display_name=text(r.name),
duration_in_ms=r.duration_seconds*1000,
duration_in_ms=r.duration_seconds * 1000,
outcome="Failed",
stack_trace=text(r.stack_trace) if r.stack_trace is not None else None,
error_message=text(r.failure_message)
)
)
if r.result == "Skip":
return TestSubResult(
comment=comment,
display_name=text(r.name),
duration_in_ms=r.duration_seconds*1000,
duration_in_ms=r.duration_seconds * 1000,
outcome="NotExecuted"
)
)
log.warning("Unexpected result value {} for {}".format(r.result, r.name))
return None

Expand All @@ -141,7 +177,7 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]:
automated_test_type=text(r.kind),
automated_test_storage=self.work_item_name,
priority=1,
duration_in_ms=r.duration_seconds*1000,
duration_in_ms=r.duration_seconds * 1000,
outcome="Passed",
state="Completed",
comment=comment,
Expand All @@ -153,7 +189,7 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]:
automated_test_type=text(r.kind),
automated_test_storage=self.work_item_name,
priority=1,
duration_in_ms=r.duration_seconds*1000,
duration_in_ms=r.duration_seconds * 1000,
outcome="Failed",
state="Completed",
error_message=text(r.failure_message),
Expand All @@ -168,7 +204,7 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]:
automated_test_type=text(r.kind),
automated_test_storage=self.work_item_name,
priority=1,
duration_in_ms=r.duration_seconds*1000,
duration_in_ms=r.duration_seconds * 1000,
outcome="NotExecuted",
state="Completed",
error_message=text(r.skip_reason),
Expand All @@ -178,13 +214,13 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]:
log.warning("Unexpected result value {} for {}".format(r.result, r.name))
return None

unconverted_results = list(results) # type: List[TestResult]
unconverted_results = list(results) # type: List[TestResult]
log.debug("Count of unconverted_results: {0}".format(len(unconverted_results)))

# Find all DDTs, determine parent, and add to dictionary
data_driven_tests = {} # type: Dict[str, TestCaseResult]
non_data_driven_tests = [] # type: List[TestCaseResult]
test_name_ordering = {} # type: Dict[str, List[str]]
non_data_driven_tests = [] # type: List[TestCaseResult]
test_name_ordering = {} # type: Dict[str, List[str]]

for r in unconverted_results:
if r is None:
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ def main():
worker.daemon = True
worker.start()

log.info("Beginning reading of test results.")
log.info("Beginning to read test results...")

# In case the user puts the results in HELIX_WORKITEM_UPLOAD_ROOT for upload, check there too.
all_results = read_results([os.getcwd(),
Expand Down