converting ci builds to parallel (#161)

* initial experiment * updating with few fixes to make ci builds parallel update updating few fixes to ci build update remove trigger remove trigger initial experiment update ci def remove condition from ci udpate dependency update dependency implementing parallel ci build fix ci fix signing issue fix yaml troubleshoot signtype troubleshoot signtype delete _nightly.yml reorganising templates and build definitions remove file writing steps for successful integration test update download artifact step * add debug echos * fix casing for signtype variable * add some debug echos * update dockerfile and build script * updating signing process * fix signing issue
microsoft · Jun 3, 2019 · 07558da · 07558da
1 parent d2d9625
commit 07558da
Show file tree

Hide file tree

Showing 11 changed files with 207 additions and 216 deletions.
diff --git a/build/build-buildimages.sh b/build/build-buildimages.sh
@@ -13,17 +13,18 @@ source $REPO_DIR/build/__variables.sh
 
 cd "$BUILD_IMAGES_BUILD_CONTEXT_DIR"
 
-declare BUILDSCRIPT_SOURCE="buildscriptbuilder"
 declare BUILD_SIGNED=""
 
+echo "SignType is: "$SIGNTYPE
+
 # Check to see if the build is by scheduled ORYX-CI or other azure devops build
-if [ "$SignType" == "real" ] || [ "$SignType" == "Real" ]
+if [ "$SIGNTYPE" == "real" ] || [ "$SIGNTYPE" == "Real" ]
 then
 # "SignType" will be real only for builds by scheduled and/or manual builds  of ORYX-CI
-    BUILDSCRIPT_SOURCE="copybuildscriptbinaries"
 	BUILD_SIGNED="true"
+	ls -l $BUILD_IMAGES_BUILD_CONTEXT_DIR
 else
-# locally we need to fake "binaries" directory to get a successful "copybuildscriptbinaries" build stage
+	# locally we need to fake "binaries" directory to get a successful "copybuildscriptbinaries" build stage
     mkdir -p $BUILD_IMAGES_BUILD_CONTEXT_DIR/binaries
 fi
 
@@ -57,14 +58,11 @@ BuildAndTagStage node-install
 BuildAndTagStage dotnet-install
 BuildAndTagStage python
 BuildAndTagStage buildscriptbuilder
-BuildAndTagStage copybuildscriptbinaries
-BuildAndTagStage buildscriptbinaries
 
 builtImageTag="$DOCKER_BUILD_IMAGES_REPO:latest"
 docker build -t $builtImageTag \
 	--build-arg AI_KEY=$APPLICATION_INSIGHTS_INSTRUMENTATION_KEY \
 	--build-arg AGENTBUILD=$BUILD_SIGNED \
-	--build-arg BUILDSCRIPT_SOURCE=$BUILDSCRIPT_SOURCE \
 	$ctxArgs -f "$BUILD_IMAGES_DOCKERFILE" .
 
 echo

diff --git a/images/build/Dockerfile b/images/build/Dockerfile
@@ -1,5 +1,4 @@
 # Start declaration of Build-Arg to determine where the image is getting built (devops agents or local)
-ARG BUILDSCRIPT_SOURCE=buildscriptbuilder
 ARG AGENTBUILD
 FROM buildpack-deps:stretch AS main
 # End declaration of Build-Arg to determine where the image is getting built (devops agents or local)
@@ -306,6 +305,9 @@ COPY src/BuildScriptGeneratorCli /usr/oryx/src/BuildScriptGeneratorCli
 COPY src/Common /usr/oryx/src/Common
 COPY build/FinalPublicKey.snk usr/oryx/build/
 COPY src/CommonFiles /usr/oryx/src/CommonFiles
+# This statement copies signed oryx binaries from during agent build.
+# For local/dev contents of blank/empty directory named binaries are getting copied
+COPY binaries /opt/buildscriptgen/
 WORKDIR /usr/oryx/src
 ARG GIT_COMMIT=unspecified
 ARG AGENTBUILD=${AGENTBUILD}
@@ -315,21 +317,6 @@ ENV BUILD_NUMBER=${BUILD_NUMBER}
 RUN if [ -z "$AGENTBUILD" ]; then \
         dotnet publish -r linux-x64 -o /opt/buildscriptgen/ -c Release BuildScriptGeneratorCli/BuildScriptGeneratorCli.csproj; \
     fi
-
-# This stage is only when building in devops agents
-FROM main AS copybuildscriptbinaries
-COPY binaries /opt/buildscriptgen/
-
-# This stage copies oryx binaries from specific build stage, for local/dev it copies
-# from "buildscriptbuilder" and for devops it copies from "copybuildscriptbinaries"
-# https://github.com/moby/moby/issues/34482
-FROM ${BUILDSCRIPT_SOURCE} AS buildscriptbinaries
-ARG BUILDSCRIPT_SOURCE
-WORKDIR /usr/oryx/src
-ARG GIT_COMMIT=unspecified
-ARG BUILD_NUMBER=unspecified
-ENV GIT_COMMIT=${GIT_COMMIT}
-ENV BUILD_NUMBER=${BUILD_NUMBER}
 RUN chmod a+x /opt/buildscriptgen/GenerateBuildScript
 
 FROM python AS final
@@ -368,7 +355,7 @@ RUN ln -s /opt/php/5.6 /opt/php/5 \
 
 # Build script generator content, currently docker doesn't support variables in --from
 # so we are building an extra stage to copy binaries from correct build stage
-COPY --from=buildscriptbinaries /opt/buildscriptgen/ /opt/buildscriptgen/
+COPY --from=buildscriptbuilder /opt/buildscriptgen/ /opt/buildscriptgen/
 RUN ln -s /opt/buildscriptgen/GenerateBuildScript /usr/local/bin/oryx
 
 # Bake Application Insights key from pipeline variable into final image

diff --git a/vsts/pipelines/ci.yml b/vsts/pipelines/ci.yml
@@ -1,17 +1,16 @@
 resources:
 - repo: self
 jobs:
-- job: Job_1
+- job: Job_Security
   displayName: Security
   condition: succeeded()
   pool:
     name: Hosted VS2017
   steps:
-  - template: _securityChecks.yml
+  - template: templates/_securityChecks.yml
 
-- job: Job_2
+- job: Job_SignBinaries
   displayName: Build and Sign Oryx Binaries
-  condition: succeeded()
   pool:
     name: VSEng-MicroBuildVS2017
     demands:
@@ -20,26 +19,54 @@ jobs:
   variables:
     SignType: 'test'
   steps:
-  - template: _signbinary.yml
+  - template: templates/_signbinary.yml
 
-- job: Job_3
-  displayName: Build and Publish Oryx Images
-  dependsOn: Job_2
+- job: Job_BuildImage
+  displayName: Build and Test Build Image
+  dependsOn: Job_SignBinaries
   condition: succeeded()
-  timeoutInMinutes: 250
   pool:
     name: OryxLinux
+  variables:
+    SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ]  # map in the signtype variable
+
   steps:
   - script: |
-        echo "##vso[task.setvariable variable=BuildBuildImages;]true"
-        echo "##vso[task.setvariable variable=TestBuildImages;]true"
-        echo "##vso[task.setvariable variable=BuildRuntimeImages;]true"
-        echo "##vso[task.setvariable variable=TestRuntimeImages;]true"
-        echo "##vso[task.setvariable variable=TestIntegration;]true"
-        echo "##vso[task.setvariable variable=PushBuildImages;]true"
-        echo "##vso[task.setvariable variable=PushRuntimeImages;]true"
-        echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
+      echo "##vso[task.setvariable variable=BuildBuildImages;]true"
+      echo "##vso[task.setvariable variable=TestBuildImages;]true"
+      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=PushBuildImages;]true"
+      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
     displayName: 'Set variables'
-  - template: _buildTemplate.yml
+      
+  - template: templates/_buildParallel.yml
+    parameters:
+      pushToDockerHub: 'true'
+
+- job: Job_RuntimeImages
+  displayName: Build and Test Runtime Images
+  dependsOn: Job_SignBinaries
+  condition: succeeded()
+  pool:
+    name: OryxLinux
+  # Building runtime images can take a long time due our PHP images
+  timeoutInMinutes: 100
+  steps:
+  - script: |
+      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
+      echo "##vso[task.setvariable variable=TestBuildImages;]false"
+      echo "##vso[task.setvariable variable=BuildRuntimeImages;]true"
+      echo "##vso[task.setvariable variable=TestRuntimeImages;]true"
+      echo "##vso[task.setvariable variable=PushRuntimeImages;]true"
+      echo "##vso[task.setvariable variable=PushBuildImages;]false"
+      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
+    displayName: 'Set variables'
+  - template: templates/_buildParallel.yml
+    parameters:
+      pushToDockerHub: 'true'
+
+- template: templates/_integration.yml
 
 trigger: none
diff --git a/vsts/pipelines/nightly.yml b/vsts/pipelines/nightly.yml
@@ -8,11 +8,10 @@ jobs:
   pool:
     name: Hosted VS2017
   steps:
-  - template: _securityChecks.yml
+  - template: templates/_securityChecks.yml
 
 - job: Job_BuildImage
   displayName: Build and Test Build Image
-  condition: succeeded()
   pool:
     name: OryxLinux
   steps:
@@ -23,18 +22,19 @@ jobs:
       echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
       echo "##vso[task.setvariable variable=PushBuildImages;]true"
       echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
       echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
     displayName: 'Set variables'
-  - template: _buildParallel.yml
+  - template: templates/_buildParallel.yml
+    parameters:
+      pushToDockerHub: 'false'
 
 - job: Job_RuntimeImages
   displayName: Build and Test Runtime Images
   condition: succeeded()
   pool:
     name: OryxLinux
   # Building runtime images can take a long time due our PHP images
-  timeoutInMinutes: 150
+  timeoutInMinutes: 100
   steps:
   - script: |
       echo "##vso[task.setvariable variable=BuildBuildImages;]false"
@@ -43,140 +43,12 @@ jobs:
       echo "##vso[task.setvariable variable=TestRuntimeImages;]true"
       echo "##vso[task.setvariable variable=PushRuntimeImages;]true"
       echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
       echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
     displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_PythonIntegrationTests
-  displayName: Run Python Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  timeoutInMinutes: 150
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_DotNetCoreIntegrationTests
-  displayName: Run DotNetCore Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_NodeIntegrationTests
-  displayName: Run NodeJs Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  timeoutInMinutes: 150
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_PhpIntegrationTests
-  displayName: Run Php Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_DbIntegrationTests
-  displayName: Run Database Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=db"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
+  - template: templates/_buildParallel.yml
+    parameters:
+      pushToDockerHub: 'false'
 
-- job: Job_ENDIntegrationTests
-  displayName: End of Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-    - Job_PythonIntegrationTests
-    - Job_PhpIntegrationTests
-    - Job_NodeIntegrationTests
-    - Job_DotNetCoreIntegrationTests
-    - Job_DbIntegrationTests
-  pool:
-    name: OryxLinux
-  steps:
-  - task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0
-    displayName: 'End of Integration Tests'
-    inputs:
-      type: InlineScript
-      script: 'echo "Integration tests ended"'
+- template: templates/_integration.yml
 
 trigger: none
diff --git a/vsts/pipelines/_buildTemplate.yml → vsts/pipelines/obsolete/_buildTemplate.yml b/vsts/pipelines/_buildTemplate.yml → vsts/pipelines/obsolete/_buildTemplate.yml