OTA E2E test fix

tools/ota_e2e_tests/aws_ota_test/aws_ota_test_case_2_updates_cancel_1st.py

@@ -24,6 +24,7 @@
 
 """
 import time
+import shutil
 from pathlib import Path
 import boto3
 from botocore.exceptions import ClientError
@@ -56,12 +57,18 @@ def run(self):
         self._otaProject.setApplicationVersion(0, 9, 1)
         # Build the OTA image.
         self._otaProject.buildProject()
-        # Create 1st OTA update.
-        otaUpdateId_1 = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
+        # Make a copy of the firmware.
+        firmware = Path(self._otaConfig['ota_firmware_file_path'])
+        first_firmware = Path(self._otaProject._projectRootDir) / firmware.name
+        shutil.copy(firmware, first_firmware)
         # Prepare another image to be updated.
         self._otaProject.setApplicationVersion(0, 9, 2)
         self._otaProject.buildProject()
+        # Create 1st OTA update.
+        self._otaConfig['ota_firmware_file_path'] = str(first_firmware)
+        otaUpdateId_1 = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
         # Create 2nd OTA update.
+        self._otaConfig['ota_firmware_file_path'] = str(firmware)
         otaUpdateId_2 = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
 
         # Wait until the job is in progress.
@@ -74,6 +81,9 @@ def run(self):
         if exec_status == 'QUEUED':
             return OtaTestResult(testName=self._name, result=OtaTestResult.ERROR,
                                  summary='Timeout waiting for OTA job status.')
+        if exec_status == 'SUCCEEDED':
+            return OtaTestResult(testName=self._name, result=OtaTestResult.ERROR,
+                                 summary='OTA update complete too fast before we can cancel the job.')
 
         # Force cancel the first job that's in progress, device should pick up the 2nd update and succeed.
         iot_client = boto3.client('iot')

tools/ota_e2e_tests/aws_ota_test/aws_ota_test_case_cancel_then_update.py

@@ -24,6 +24,7 @@
 
 """
 import time
+import shutil
 from pathlib import Path
 import boto3
 from botocore.exceptions import ClientError
@@ -57,11 +58,16 @@ def run(self):
         self._otaProject.setApplicationVersion(0, 9, 1)
         # Build the OTA image.
         self._otaProject.buildProject()
-        # Start an OTA Update.
-        otaUpdateId = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
+        # Make a copy of the firmware.
+        firmware = Path(self._otaConfig['ota_firmware_file_path'])
+        first_firmware = Path(self._otaProject._projectRootDir) / firmware.name
+        shutil.copy(firmware, first_firmware)
         # Prepare another image to be updated later
         self._otaProject.setApplicationVersion(0, 9, 2)
         self._otaProject.buildProject()
+        # Start first OTA Update.
+        self._otaConfig['ota_firmware_file_path'] = str(first_firmware)
+        otaUpdateId = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
 
         # Wait until the job is in progress.
         thing_name = self._otaAwsAgent._iotThing.thing_name
@@ -73,11 +79,16 @@ def run(self):
         if exec_status == 'QUEUED':
             return OtaTestResult(testName=self._name, result=OtaTestResult.ERROR,
                                  summary='Timeout waiting for OTA job status.')
+        if exec_status == 'SUCCEEDED':
+            return OtaTestResult(testName=self._name, result=OtaTestResult.ERROR,
+                                 summary='OTA update complete too fast before we can cancel the job.')
 
         # Force cancel the job that's in progress.
         iot_client = boto3.client('iot')
         iot_client.cancel_job_execution(jobId=f'AFR_OTA-{otaUpdateId}', thingName=thing_name, force=True)
-        # Do another OTA update, this should succeed.
+
+        # Do another OTA update with second build, this should succeed.
+        self._otaConfig['ota_firmware_file_path'] = str(firmware)
         otaUpdateId = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
 
         return self.getTestResultAfterOtaUpdateCompletion(otaUpdateId)

tools/ota_e2e_tests/aws_ota_test/aws_ota_test_case_disconnect_cancel_update.py

@@ -24,6 +24,7 @@
 
 """
 import time
+import shutil
 from pathlib import Path
 import boto3
 from botocore.exceptions import ClientError
@@ -74,11 +75,16 @@ def run(self):
         self._otaProject.setApplicationVersion(0, 9, 1)
         # Build the OTA image.
         self._otaProject.buildProject()
-        # Start an OTA Update.
-        otaUpdateId = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
+        # Make a copy of the firmware.
+        firmware = Path(self._otaConfig['ota_firmware_file_path'])
+        first_firmware = Path(self._otaProject._projectRootDir) / firmware.name
+        shutil.copy(firmware, first_firmware)
         # Prepare another image to be updated later
         self._otaProject.setApplicationVersion(0, 9, 2)
         self._otaProject.buildProject()
+        # Start first OTA Update.
+        self._otaConfig['ota_firmware_file_path'] = str(first_firmware)
+        otaUpdateId = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
 
         # Wait until the job is in progress.
         thing_name = self._otaAwsAgent._iotThing.thing_name
@@ -90,6 +96,9 @@ def run(self):
         if exec_status == 'QUEUED':
             return OtaTestResult(testName=self._name, result=OtaTestResult.ERROR,
                                  summary='Timeout waiting for OTA job status.')
+        if exec_status == 'SUCCEEDED':
+            return OtaTestResult(testName=self._name, result=OtaTestResult.ERROR,
+                                 summary='OTA update complete too fast before we can cancel the job.')
 
         # Connect to IoT core with same thing name, this should disconnect the device from IoT core.
         # Note: currently there's no way to load the cert/key from memory, this is a limitation from
@@ -114,7 +123,8 @@ def run(self):
             iot_client = boto3.client('iot')
             iot_client.cancel_job_execution(jobId=f'AFR_OTA-{otaUpdateId}', thingName=thing_name, force=True)
 
-            # Do another OTA update, this should succeed.
+            # Do another OTA update with second build, this should succeed.
+            self._otaConfig['ota_firmware_file_path'] = str(firmware)
             otaUpdateId = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
             return self.getTestResultAfterOtaUpdateCompletion(otaUpdateId)
         else:

tools/ota_e2e_tests/aws_ota_test/aws_ota_test_case_factory.py

@@ -26,6 +26,7 @@
 from .aws_ota_test_case_greater_version import OtaTestGreaterVersion
 from .aws_ota_test_case_back_to_back_switch_protocol import OtaTestBackToBackSwitchProtocol
 from .aws_ota_test_case_default_data_protocol import OtaTestDefaultDataProtocol
+from .aws_ota_test_case_2_updates_cancel_1st import OtaTest2UpdatesCancel1st
 from .aws_ota_test_case_disconnect_cancel_update import OtaTestDisconnectCancelUpdate
 from .aws_ota_test_case_disconnect_resume import OtaTestDisconnectResume
 from .aws_ota_test_case_presigned_url_expired import OtaTestPresignedUrlExpired

tools/ota_e2e_tests/aws_ota_test/aws_ota_test_main.py

@@ -30,7 +30,8 @@
 from threading import Thread
 from functools import reduce
 from operator import add
-from junit_xml import TestSuite, TestCase
+
+import junitparser as junit
 from .aws_ota_test_runner import *
 from .aws_ota_test_result import OtaTestResult
 
@@ -154,20 +155,20 @@ def createJunitTestResults(boardToResults, fileName):
         boardToResults(dict[str:obj(OtaTestResult)]): Dictionary of the board name to it's OtaTestResult.
         fileName: The name of the junit test file to create.
     """
-    testSuites = []
+    report = junit.JUnitXml()
     for board in boardToResults.keys():
-        testCases = []
+        group_suite = junit.TestSuite(board + '.OTAEndToEndTests')
         for otaTestResult in boardToResults[board]:
-            testCase = TestCase(otaTestResult.testName, classname=board + '.OTAEndToEndTests')
-            testCases.append(testCase)
+            test_case = junit.TestCase(otaTestResult.testName)
             if otaTestResult.result == OtaTestResult.FAIL:
-                testCases[-1].add_failure_info(message=otaTestResult.summary)
+                test_case.result = junit.Failure(otaTestResult.summary)
             elif otaTestResult.result == OtaTestResult.ERROR:
-                testCases[-1].add_skipped_info(message=otaTestResult.summary)
-        testSuites.append(TestSuite(board, test_cases=testCases, package=board))
+                test_case.result = junit.Skipped(otaTestResult.summary)
+            group_suite.add_testcase(test_case)
+        report.add_testsuite(group_suite)
 
-    with open(fileName, 'w') as f:
-        TestSuite.to_file(f, testSuites)
+    report.update_statistics()
+    report.write(fileName, pretty=True)
 
 def getStageParameters(args):
     stageParams = {}