add more parallelism to nightly builds

* add more parallelism to nightly builds update pulling and retagging for oryx images * change variable name and set pipefail for pullandretag script
microsoft · May 11, 2019 · c2fff09 · c2fff09
1 parent 16b2c67
commit c2fff09
Show file tree

Hide file tree

Showing 12 changed files with 156 additions and 34 deletions.
diff --git a/tests/Oryx.Integration.Tests/DotNetCoreEndToEndTests.cs b/tests/Oryx.Integration.Tests/DotNetCoreEndToEndTests.cs
@@ -13,6 +13,7 @@
 
 namespace Microsoft.Oryx.Integration.Tests
 {
+    [Trait("category", "dotnetcore")]
     public class DotNetCoreEndToEndTests : PlatformEndToEndTestsBase
     {
         private const int HostPort = Constants.DotNetCoreEndToEndTestsPort;

diff --git a/tests/Oryx.Integration.Tests/MySqlIntegrationTests.cs b/tests/Oryx.Integration.Tests/MySqlIntegrationTests.cs
@@ -10,7 +10,7 @@
 
 namespace Microsoft.Oryx.Integration.Tests
 {
-    [Trait("Category", "DB")]
+    [Trait("category", "db")]
     [Trait("db", "mysql")]
     public class MySqlIntegrationTests : DatabaseTestsBase, IClassFixture<Fixtures.MySqlDbContainerFixture>
     {

diff --git a/tests/Oryx.Integration.Tests/NodeEndToEndTests.cs b/tests/Oryx.Integration.Tests/NodeEndToEndTests.cs
@@ -770,7 +770,7 @@ await EndToEndTestHelper.BuildRunAndAssertAppAsync(
                 });
         }
 
-        [Fact]
+        [Fact(Skip = "#824174: Sync the Node Go startup code with the C# 'run-script' code")]
         public async Task Node_CreateReactAppSample_singleImage()
         {
             // Arrange
@@ -856,7 +856,7 @@ await EndToEndTestHelper.BuildRunAndAssertAppAsync(
                 });
         }
 
-        [Fact]
+        [Fact(Skip = "#824174: Sync the Node Go startup code with the C# 'run-script' code")]
         public async Task CanBuildAndRun_NodeExpressApp_UsingSingleImage_AndCustomStartupCommandOnly()
         {
             // Arrange

diff --git a/tests/Oryx.Integration.Tests/PhpEndToEndTests.cs b/tests/Oryx.Integration.Tests/PhpEndToEndTests.cs
@@ -14,6 +14,7 @@
 
 namespace Microsoft.Oryx.Integration.Tests
 {
+    [Trait("category", "php")]
     public class PhpEndToEndTests : PlatformEndToEndTestsBase
     {
         private const int HostPort = Constants.PhpEndToEndTestsPort;

diff --git a/tests/Oryx.Integration.Tests/PostgreSqlIntegrationTests.cs b/tests/Oryx.Integration.Tests/PostgreSqlIntegrationTests.cs
@@ -10,7 +10,7 @@
 
 namespace Microsoft.Oryx.Integration.Tests
 {
-    [Trait("Category", "DB")]
+    [Trait("category", "db")]
     [Trait("db", "postgres")]
     public class PostgreSqlIntegrationTests : DatabaseTestsBase, IClassFixture<Fixtures.PostgreSqlDbContainerFixture>
     {

diff --git a/tests/Oryx.Integration.Tests/PythonEndToEndTests.cs b/tests/Oryx.Integration.Tests/PythonEndToEndTests.cs
@@ -13,6 +13,7 @@
 
 namespace Microsoft.Oryx.Integration.Tests
 {
+    [Trait("category", "python")]
     public class PythonEndToEndTests : PlatformEndToEndTestsBase
     {
         private const int HostPort = Constants.PythonEndToEndTestsPort;

diff --git a/tests/Oryx.Integration.Tests/SqlServerIntegrationTests.cs b/tests/Oryx.Integration.Tests/SqlServerIntegrationTests.cs
@@ -14,7 +14,7 @@ namespace Microsoft.Oryx.Integration.Tests
     //// Commenting out the tests here as the test fixture is still run and we want to avoid that as stopping the
     //// SQL Server container on the build agent fails sometimes requiring a reboot of the agent machine.
 
-    //[Trait("Category", "DB")]
+    //[Trait("category", "db")]
     //[Trait("db", "sqlserver")]
     //public class SqlServerIntegrationTests : DatabaseTestsBase, IClassFixture<Fixtures.SqlServerDbContainerFixture>
     //{

diff --git a/vsts/pipelines/_buildParallel.yml b/vsts/pipelines/_buildParallel.yml
@@ -85,19 +85,11 @@ steps:
   condition: and(succeeded(), eq(variables['TestIntegration'], 'true'))
 
 - task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0
-  displayName: 'Pull and Retag recently built oryx build image'
+  displayName: 'Pull and Retag recently built oryx build and runtime images'
   inputs:
     type: FilePath
     scriptPath: ./vsts/scripts/pullandretag.sh
-    args: $(System.ArtifactsDirectory)/drop/images/build-images-acr.txt
-  condition: and(succeeded(), eq(variables['TestIntegration'], 'true'))
-
-- task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0
-  displayName: 'Pull and Retag recently built oryx runtime images'
-  inputs:
-    type: FilePath
-    scriptPath: ./vsts/scripts/pullandretag.sh
-    args: $(System.ArtifactsDirectory)/drop/images/runtime-images-acr.txt
+    args: $(System.ArtifactsDirectory)/drop/images/build-images-acr.txt $(System.ArtifactsDirectory)/drop/images/runtime-images-acr.txt
   condition: and(succeeded(), eq(variables['TestIntegration'], 'true'))
 
 - task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0

diff --git a/vsts/pipelines/ci.yml b/vsts/pipelines/ci.yml
@@ -42,4 +42,4 @@ jobs:
     displayName: 'Set variables'
   - template: _buildTemplate.yml
 
-trigger: none
+trigger: none
diff --git a/vsts/pipelines/nightly.yml b/vsts/pipelines/nightly.yml
@@ -48,8 +48,8 @@ jobs:
     displayName: 'Set variables'
   - template: _buildParallel.yml
 
-- job: Job_IntegrationTests
-  displayName: Test Integration
+- job: Job_PythonIntegrationTests
+  displayName: Run Python Integration Tests
   dependsOn: 
     - Job_BuildImage
     - Job_RuntimeImages
@@ -62,7 +62,74 @@ jobs:
       echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
       echo "##vso[task.setvariable variable=TestBuildImages;]false"
       echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]Category!=DB"
+      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python"
+      echo "##vso[task.setvariable variable=TestIntegration;]true"
+      echo "##vso[task.setvariable variable=PushBuildImages;]false"
+      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
+      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
+    displayName: 'Set variables'
+  - template: _buildParallel.yml
+
+- job: Job_DotNetCoreIntegrationTests
+  displayName: Run DotNetCore Integration Tests
+  dependsOn: 
+    - Job_BuildImage
+    - Job_RuntimeImages
+  pool:
+    name: OryxLinux
+  steps:
+  - script: |
+      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
+      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestBuildImages;]false"
+      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore"
+      echo "##vso[task.setvariable variable=TestIntegration;]true"
+      echo "##vso[task.setvariable variable=PushBuildImages;]false"
+      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
+      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
+    displayName: 'Set variables'
+  - template: _buildParallel.yml
+
+- job: Job_NodeIntegrationTests
+  displayName: Run NodeJs Integration Tests
+  dependsOn: 
+    - Job_BuildImage
+    - Job_RuntimeImages
+  pool:
+    name: OryxLinux
+  timeoutInMinutes: 150
+  steps:
+  - script: |
+      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
+      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestBuildImages;]false"
+      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node"
+      echo "##vso[task.setvariable variable=TestIntegration;]true"
+      echo "##vso[task.setvariable variable=PushBuildImages;]false"
+      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
+      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
+    displayName: 'Set variables'
+  - template: _buildParallel.yml
+
+- job: Job_PhpIntegrationTests
+  displayName: Run Php Integration Tests
+  dependsOn: 
+    - Job_BuildImage
+    - Job_RuntimeImages
+  pool:
+    name: OryxLinux
+  steps:
+  - script: |
+      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
+      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestBuildImages;]false"
+      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php"
       echo "##vso[task.setvariable variable=TestIntegration;]true"
       echo "##vso[task.setvariable variable=PushBuildImages;]false"
       echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
@@ -72,7 +139,7 @@ jobs:
   - template: _buildParallel.yml
 
 - job: Job_DbIntegrationTests
-  displayName: Test Database Integration
+  displayName: Run Database Integration Tests
   dependsOn: 
     - Job_BuildImage
     - Job_RuntimeImages
@@ -84,7 +151,7 @@ jobs:
       echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
       echo "##vso[task.setvariable variable=TestBuildImages;]false"
       echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]Category=DB"
+      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=db"
       echo "##vso[task.setvariable variable=TestIntegration;]true"
       echo "##vso[task.setvariable variable=PushBuildImages;]false"
       echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
@@ -93,4 +160,23 @@ jobs:
     displayName: 'Set variables'
   - template: _buildParallel.yml
 
+- job: Job_ENDIntegrationTests
+  displayName: End of Integration Tests
+  dependsOn: 
+    - Job_BuildImage
+    - Job_RuntimeImages
+    - Job_PythonIntegrationTests
+    - Job_PhpIntegrationTests
+    - Job_NodeIntegrationTests
+    - Job_DotNetCoreIntegrationTests
+    - Job_DbIntegrationTests
+  pool:
+    name: OryxLinux
+  steps:
+  - task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0
+    displayName: 'End of Integration Tests'
+    inputs:
+      type: InlineScript
+      script: 'echo "Integration tests ended"'
+
 trigger: none
diff --git a/vsts/scripts/dockerCleanup.sh b/vsts/scripts/dockerCleanup.sh
@@ -7,6 +7,10 @@
 declare -r BUILD_NUMBER="$BUILD_BUILDNUMBER"
 declare -r DOCKER_SYSTEM_PRUNE="${ORYX_DOCKER_SYSTEM_PRUNE:-false}"
 
+echo
+echo "Printing all running containers and stopped containers"
+echo
+docker ps -a 
 echo
 echo "Kill all running containers and delete all stopped containers"
 echo

diff --git a/vsts/scripts/pullandretag.sh b/vsts/scripts/pullandretag.sh
@@ -5,23 +5,60 @@
 # --------------------------------------------------------------------------------------------
 
 set -euo pipefail
+# $1 > buildimage-acr.txt
+# $2 > runtime-images-acr.txt
+declare imagefilter="oryxdevmcr.azurecr.io/public/oryx"
 
-while read sourceImage; do
+echo "Build image filter is set"
+while read buildImage; do
   # Always use specific build number based tag and then use the same tag to create a 'latest' tag and push it
-  if [[ $sourceImage != *:latest ]]; then
-    echo "Pulling the source image $sourceImage ..."
-    docker pull "$sourceImage" | sed 's/^/     /'
+  if [[ $buildImage != *:latest ]]; then
+	echo "Pulling the build image $buildImage ..."
+	docker pull "$buildImage" | sed 's/^/     /'
 
-    # Trim the build number tag and append the '':latest' to end of it
-    newtag="${sourceImage%:*}:latest"
+	# Trim the build number tag and append the '':latest' to end of it
+	newtag="${buildImage%:*}:latest"
 
-    # Replace the ACR registry repository name with a name that the tests know about
-    newtag=$(echo "$newtag" | sed 's,oryxdevmcr.azurecr.io/public/oryx,oryxdevms,g')
+	# Replace the ACR registry repository name with a name that the tests know about
+	newtag=$(echo "$newtag" | sed 's,oryxdevmcr.azurecr.io/public/oryx,oryxdevms,g')
 
-    echo
-    echo "Tagging the source image with tag $newtag ..."
-    docker tag "$sourceImage" "$newtag" | sed 's/^/     /'
-    echo
-    echo -------------------------------------------------------------------------------
+	echo
+	echo "Tagging the source image with tag $newtag ..."
+	docker tag "$buildImage" "$newtag" | sed 's/^/     /'
+	echo
+	echo -------------------------------------------------------------------------------
   fi
 done <"$1"
+
+# Extract language string from string (e.g extract 'python' from 'category=python')
+if [ -n "$TESTINTEGRATIONCASEFILTER" ];then
+	# For DB tests we want all the runtime images to be present at thae agent machine
+	if [[ "$TESTINTEGRATIONCASEFILTER" != *db* ]];then
+		imagefilter=$(echo $TESTINTEGRATIONCASEFILTER | cut -d'=' -f 2)
+	fi
+fi
+
+# Always convert filter for runtime images to lower case
+echo "Runtime image filter is set for "$imagefilter
+
+while read sourceImage; do
+  # Always use specific build number based tag and then use the same tag to create a 'latest' tag and push it
+  if [[ "$sourceImage" != *:latest ]]; then
+	if [[ "$sourceImage" == *"$imagefilter"* ]]; then
+		echo "Pulling the runtime image $sourceImage ..."
+		docker pull "$sourceImage" | sed 's/^/     /'
+
+		# Trim the build number tag and append the '':latest' to end of it
+		newtag="${sourceImage%:*}:latest"
+
+		# Replace the ACR registry repository name with a name that the tests know about
+		newtag=$(echo "$newtag" | sed 's,oryxdevmcr.azurecr.io/public/oryx,oryxdevms,g')
+
+		echo
+		echo "Tagging the source image with tag $newtag ..."
+		docker tag "$sourceImage" "$newtag" | sed 's/^/     /'
+		echo
+		echo -------------------------------------------------------------------------------
+	fi
+  fi
+done <"$2"