From ff43194438b1744a098e304e581f0c498608e3c4 Mon Sep 17 00:00:00 2001
From: Alex Perovich <alperovi@microsoft.com>
Date: Wed, 28 Apr 2021 12:52:35 -0700
Subject: [PATCH 1/2] Fail the work item when the azure pipelines upload fails

---
 .../azure-pipelines/AzurePipelines.MonoQueue.targets  |  4 ++--
 .../Sdk/tools/azure-pipelines/reporter/run.bat        |  2 ++
 .../Sdk/tools/azure-pipelines/reporter/run.py         | 11 ++++++++++-
 .../Sdk/tools/azure-pipelines/reporter/run.sh         |  4 +++-
 4 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MonoQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MonoQueue.targets
index 335fac7897f..9dc488962f1 100644
--- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MonoQueue.targets
+++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MonoQueue.targets
@@ -7,11 +7,11 @@
   <PropertyGroup>
     <HelixPostCommands Condition="$(IsPosixShell)">
       $(HelixPostCommands);
-      /bin/sh $HELIX_CORRELATION_PAYLOAD/reporter/run.sh $(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI) $(SYSTEM_TEAMPROJECT) $(TestRunId) $(SYSTEM_ACCESSTOKEN)
+      /bin/sh $HELIX_CORRELATION_PAYLOAD/reporter/run.sh $(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI) $(SYSTEM_TEAMPROJECT) $(TestRunId) $(SYSTEM_ACCESSTOKEN) || exit $?
     </HelixPostCommands>
     <HelixPostCommands Condition="!$(IsPosixShell)">
       $(HelixPostCommands);
-      call %HELIX_CORRELATION_PAYLOAD%\reporter\run.bat $(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI) $(SYSTEM_TEAMPROJECT) $(TestRunId) $(SYSTEM_ACCESSTOKEN)
+      call %HELIX_CORRELATION_PAYLOAD%\reporter\run.bat $(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI) $(SYSTEM_TEAMPROJECT) $(TestRunId) $(SYSTEM_ACCESSTOKEN) || exit /b
     </HelixPostCommands>
   </PropertyGroup>
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.bat b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.bat
index 6522f803bc9..3a3c0f9d1ba 100644
--- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.bat
+++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.bat
@@ -33,6 +33,8 @@ GOTO :retryloop
 
 echo  %date%-%time%
 %ENV_PATH%\Scripts\python.exe -B %~dp0run.py %*
+set _uploaderExitCode=%ERRORLEVEL%
 echo  %date%-%time%
 
 set PYTHONPATH=%_OLD_PYTHONPATH%
+exit /b %_uploaderExitCode%
diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py
index dcfd343dd98..65d749058b2 100644
--- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py
+++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py
@@ -3,13 +3,16 @@
 import traceback
 import logging
 from queue import Queue
-from threading import Thread
+from threading import Thread, Lock
 from typing import Tuple, Optional
 
 from test_results_reader import read_results
 from helpers import batch, get_env
 from azure_devops_result_publisher import AzureDevOpsTestResultPublisher
 
+workerFailedLock = Lock()
+workerFailed = False
+
 class UploadWorker(Thread):
     def __init__(self, queue, idx, collection_uri, team_project, test_run_id, access_token):
         super(UploadWorker, self).__init__()
@@ -40,6 +43,8 @@ def run(self):
                 self.__process(item)
             except:
                 self.__print("got error: {}".format(traceback.format_exc()))
+                with workerFailedLock:
+                    workerFailed = True
             finally:
                 self.queue.task_done()
 
@@ -105,6 +110,10 @@ def main():
     q.join()
 
     log.info("Main thread exiting")
+    
+    with workerFailedLock:
+        if workerFailed:
+            sys.exit(1337)
 
 if __name__ == '__main__':
     main()
diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.sh b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.sh
index 2f082175508..c3f8ee86708 100755
--- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.sh
+++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.sh
@@ -35,6 +35,8 @@ fi
 
 date -u +"%FT%TZ"
 $ENV_PATH/bin/python -B $script_path/run.py "$@"
+export _uploaderExitCode=$?
 date -u +"%FT%TZ"
 
-export PYTHONPATH=$_OLD_PYTHONPATH
\ No newline at end of file
+export PYTHONPATH=$_OLD_PYTHONPATH
+exit $_uploaderExitCode

From dac35206debc994e8b92a82354d57f4c10bca23e Mon Sep 17 00:00:00 2001
From: Alex Perovich <alperovi@microsoft.com>
Date: Wed, 28 Apr 2021 12:56:44 -0700
Subject: [PATCH 2/2] global

---
 .../Sdk/tools/azure-pipelines/reporter/run.py                   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py
index 65d749058b2..bde3200f8f4 100644
--- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py
+++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py
@@ -36,6 +36,7 @@ def __process(self, batch):
         self.__print('uploaded {} results'.format(self.total_uploaded))
 
     def run(self):
+        global workerFailed, workerFailedLock
         self.__print("starting...")
         while True:
             try:
@@ -70,6 +71,7 @@ def process_args() -> Tuple[str, str, str, Optional[str]]:
 
 
 def main():
+    global workerFailed, workerFailedLock
     logging.basicConfig(
         format='%(asctime)s: %(levelname)s: %(thread)d: %(module)s(%(lineno)d): %(funcName)s: %(message)s',
         level=logging.INFO,