dotnet#7371 - retry in the case of HTTP 503 (10x, 3 seconds between a…

…ttempts) , and stop trying in the case of "run is deleted"

src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/azure_devops_result_publisher.py

@@ -1,18 +1,21 @@
 import base64
 import os
 import logging
+import time
 from typing import Iterable, Mapping, List, Dict, Optional, Tuple
 from builtins import str as text
 from azure.devops.connection import Connection
 from msrest.authentication import BasicTokenAuthentication, BasicAuthentication
 from azure.devops.v5_1.test import TestClient
 from azure.devops.v5_1.test.models import TestCaseResult, TestAttachmentRequestModel, TestSubResult
+from azure.devops.exceptions import AzureDevOpsClientRequestError
 
 from helpers import get_env
 from defs import TestResult
 
 log = logging.getLogger(__name__)
 
+
 class AzureDevOpsTestResultPublisher:
     def __init__(self, collection_uri, access_token, team_project, test_run_id=None):
         """
@@ -38,15 +41,15 @@ def is_data_driven_test(self, r: str) -> bool:
         return r.endswith(")")
 
     def get_ddt_base_name(self, r: str) -> str:
-        return r.split('(',1)[0]
+        return r.split('(', 1)[0]
 
     def send_attachment(self, test_client, attachment, published_result):
         try:
             # Python 3 will throw a TypeError exception because b64encode expects bytes
-            stream=base64.b64encode(text(attachment.text))
+            stream = base64.b64encode(text(attachment.text))
         except TypeError:
             # stream has to be a string but b64encode takes and returns bytes on Python 3
-            stream=base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8")
+            stream = base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8")
 
         test_client.create_test_result_attachment(
             TestAttachmentRequestModel(
@@ -55,19 +58,50 @@ def send_attachment(self, test_client, attachment, published_result):
             ), self.team_project, self.test_run_id, published_result.id)
 
     def send_sub_attachment(self, test_client, attachment, published_result, sub_result_id):
-        stream=base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8")
+        stream = base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8")
 
         test_client.create_test_sub_result_attachment(
             TestAttachmentRequestModel(
                 file_name=text(attachment.name),
                 stream=stream,
             ), self.team_project, self.test_run_id, published_result.id, sub_result_id)
 
-    def publish_results(self, test_case_results: Iterable[TestCaseResult], test_result_order: Dict[str, List[str]], results_with_attachments: Mapping[str, TestResult]) -> None:
+    def publish_results(self, test_case_results: Iterable[TestCaseResult], test_result_order: Dict[str, List[str]],
+                        results_with_attachments: Mapping[str, TestResult]) -> None:
         connection = self.get_connection()
         test_client = connection.get_client("azure.devops.v5_1.test.TestClient")  # type: TestClient
 
-        published_results = test_client.add_test_results_to_test_run(list(test_case_results), self.team_project, self.test_run_id)  # type: List[TestCaseResult]
+        tries_left = 10
+        succeeded = False
+        test_run_ended = False
+
+        while tries_left > 0 and not test_run_ended and not succeeded:
+            try:
+                published_results = test_client. \
+                    add_test_results_to_test_run(list(test_case_results),
+                                                 self.team_project,
+                                                 self.test_run_id)  # type: List[TestCaseResult]
+
+            except AzureDevOpsClientRequestError as ex:
+                # Odd syntax here is to deal with checking substrings of the list of args in this exception
+                hit_503 = len([element for element in ex.args if ('invalid status code of 503' in element)]) != 0
+                test_run_ended = len([element for element in ex.args if ('It may have been deleted' in element)]) != 0
+                if hit_503:
+                    tries_left -= 1
+                    log.warning("Hit HTTP 503 from Azure DevOps.  Will wait three seconds and try again.")
+                    time.sleep(3)
+                elif test_run_ended:  # Not exceptional, don't retry.
+                    tries_left = 0
+                else:
+                    raise ex
+
+        if test_run_ended:
+            log.info("Test run has ended, skipping attaching results as it would fail.")
+            return
+
+        # !succeeded means a 503 and not succeeding after 10 tries (otherwise we threw already), so give a nice error
+        if not succeeded:
+            raise Exception('Failed to report test results to Azure Dev Ops after retrying.  Please contact dnceng.')
 
         for published_result in published_results:
 
@@ -89,11 +123,12 @@ def publish_results(self, test_case_results: Iterable[TestCaseResult], test_resu
             # is true , and uses the order of test names recorded earlier to look-up the attachments.
             elif published_result.sub_results is not None:
                 sub_results_order = test_result_order[published_result.automated_test_name]
-                
+
                 # Sanity check
                 if len(sub_results_order) != len(published_result.sub_results):
-                    log.warning("Returned subresults list length does not match expected. Attachments may not pair correctly.")
-
+                    log.warning(
+                        "Returned subresults list length does not match expected. Attachments may not pair correctly.")
+
                 for (name, sub_result) in zip(sub_results_order, published_result.sub_results):
                     if name in results_with_attachments:
                         result = results_with_attachments.get(name)
@@ -111,25 +146,25 @@ def convert_to_sub_test(r: TestResult) -> Optional[TestSubResult]:
                 return TestSubResult(
                     comment=comment,
                     display_name=text(r.name),
-                    duration_in_ms=r.duration_seconds*1000,
+                    duration_in_ms=r.duration_seconds * 1000,
                     outcome="Passed"
-                    )
+                )
             if r.result == "Fail":
                 return TestSubResult(
                     comment=comment,
                     display_name=text(r.name),
-                    duration_in_ms=r.duration_seconds*1000,
+                    duration_in_ms=r.duration_seconds * 1000,
                     outcome="Failed",
                     stack_trace=text(r.stack_trace) if r.stack_trace is not None else None,
                     error_message=text(r.failure_message)
-                    )
+                )
             if r.result == "Skip":
                 return TestSubResult(
                     comment=comment,
                     display_name=text(r.name),
-                    duration_in_ms=r.duration_seconds*1000,
+                    duration_in_ms=r.duration_seconds * 1000,
                     outcome="NotExecuted"
-                    )
+                )
             log.warning("Unexpected result value {} for {}".format(r.result, r.name))
             return None
 
@@ -141,7 +176,7 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]:
                     automated_test_type=text(r.kind),
                     automated_test_storage=self.work_item_name,
                     priority=1,
-                    duration_in_ms=r.duration_seconds*1000,
+                    duration_in_ms=r.duration_seconds * 1000,
                     outcome="Passed",
                     state="Completed",
                     comment=comment,
@@ -153,7 +188,7 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]:
                     automated_test_type=text(r.kind),
                     automated_test_storage=self.work_item_name,
                     priority=1,
-                    duration_in_ms=r.duration_seconds*1000,
+                    duration_in_ms=r.duration_seconds * 1000,
                     outcome="Failed",
                     state="Completed",
                     error_message=text(r.failure_message),
@@ -168,7 +203,7 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]:
                     automated_test_type=text(r.kind),
                     automated_test_storage=self.work_item_name,
                     priority=1,
-                    duration_in_ms=r.duration_seconds*1000,
+                    duration_in_ms=r.duration_seconds * 1000,
                     outcome="NotExecuted",
                     state="Completed",
                     error_message=text(r.skip_reason),
@@ -178,13 +213,13 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]:
             log.warning("Unexpected result value {} for {}".format(r.result, r.name))
             return None
 
-        unconverted_results = list(results) # type: List[TestResult]
+        unconverted_results = list(results)  # type: List[TestResult]
         log.debug("Count of unconverted_results: {0}".format(len(unconverted_results)))
 
         # Find all DDTs, determine parent, and add to dictionary
         data_driven_tests = {}  # type: Dict[str, TestCaseResult]
-        non_data_driven_tests = [] # type: List[TestCaseResult]
-        test_name_ordering = {} # type: Dict[str, List[str]]
+        non_data_driven_tests = []  # type: List[TestCaseResult]
+        test_name_ordering = {}  # type: Dict[str, List[str]]
 
         for r in unconverted_results:
             if r is None:

src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py

@@ -93,7 +93,7 @@ def main():
         worker.daemon = True 
         worker.start()
 
-    log.info("Beginning reading of test results.")
+    log.info("Beginning to read test results...")
 
     # In case the user puts the results in HELIX_WORKITEM_UPLOAD_ROOT for upload, check there too.
     all_results = read_results([os.getcwd(),