converting ci builds to parallel

build/build-buildimages.sh

@@ -13,17 +13,18 @@ source $REPO_DIR/build/__variables.sh
 
 cd "$BUILD_IMAGES_BUILD_CONTEXT_DIR"
 
-declare BUILDSCRIPT_SOURCE="buildscriptbuilder"
 declare BUILD_SIGNED=""
 
+echo "SignType is: "$SIGNTYPE
+
 # Check to see if the build is by scheduled ORYX-CI or other azure devops build
-if [ "$SignType" == "real" ] || [ "$SignType" == "Real" ]
+if [ "$SIGNTYPE" == "real" ] || [ "$SIGNTYPE" == "Real" ]
 then
 # "SignType" will be real only for builds by scheduled and/or manual builds  of ORYX-CI
-    BUILDSCRIPT_SOURCE="copybuildscriptbinaries"
 	BUILD_SIGNED="true"
+	ls -l $BUILD_IMAGES_BUILD_CONTEXT_DIR
 else
-# locally we need to fake "binaries" directory to get a successful "copybuildscriptbinaries" build stage
+	# locally we need to fake "binaries" directory to get a successful "copybuildscriptbinaries" build stage
     mkdir -p $BUILD_IMAGES_BUILD_CONTEXT_DIR/binaries
 fi
 
@@ -57,14 +58,11 @@ BuildAndTagStage node-install
 BuildAndTagStage dotnet-install
 BuildAndTagStage python
 BuildAndTagStage buildscriptbuilder
-BuildAndTagStage copybuildscriptbinaries
-BuildAndTagStage buildscriptbinaries
 
 builtImageTag="$DOCKER_BUILD_IMAGES_REPO:latest"
 docker build -t $builtImageTag \
 	--build-arg AI_KEY=$APPLICATION_INSIGHTS_INSTRUMENTATION_KEY \
 	--build-arg AGENTBUILD=$BUILD_SIGNED \
-	--build-arg BUILDSCRIPT_SOURCE=$BUILDSCRIPT_SOURCE \
 	$ctxArgs -f "$BUILD_IMAGES_DOCKERFILE" .
 
 echo

images/build/Dockerfile

@@ -1,5 +1,4 @@
 # Start declaration of Build-Arg to determine where the image is getting built (devops agents or local)
-ARG BUILDSCRIPT_SOURCE=buildscriptbuilder
 ARG AGENTBUILD
 FROM buildpack-deps:stretch AS main
 # End declaration of Build-Arg to determine where the image is getting built (devops agents or local)
@@ -257,6 +256,9 @@ COPY src/BuildScriptGeneratorCli /usr/oryx/src/BuildScriptGeneratorCli
 COPY src/Common /usr/oryx/src/Common
 COPY build/FinalPublicKey.snk usr/oryx/build/
 COPY src/CommonFiles /usr/oryx/src/CommonFiles
+# This statement copies signed oryx binaries from during agent build.
+# For local/dev contents of blank/empty directory named binaries are getting copied
+COPY binaries /opt/buildscriptgen/
 WORKDIR /usr/oryx/src
 ARG GIT_COMMIT=unspecified
 ARG AGENTBUILD=${AGENTBUILD}
@@ -266,21 +268,6 @@ ENV BUILD_NUMBER=${BUILD_NUMBER}
 RUN if [ -z "$AGENTBUILD" ]; then \
         dotnet publish -r linux-x64 -o /opt/buildscriptgen/ -c Release BuildScriptGeneratorCli/BuildScriptGeneratorCli.csproj; \
     fi
-
-# This stage is only when building in devops agents
-FROM main AS copybuildscriptbinaries
-COPY binaries /opt/buildscriptgen/
-
-# This stage copies oryx binaries from specific build stage, for local/dev it copies
-# from "buildscriptbuilder" and for devops it copies from "copybuildscriptbinaries"
-# https://github.com/moby/moby/issues/34482
-FROM ${BUILDSCRIPT_SOURCE} AS buildscriptbinaries
-ARG BUILDSCRIPT_SOURCE
-WORKDIR /usr/oryx/src
-ARG GIT_COMMIT=unspecified
-ARG BUILD_NUMBER=unspecified
-ENV GIT_COMMIT=${GIT_COMMIT}
-ENV BUILD_NUMBER=${BUILD_NUMBER}
 RUN chmod a+x /opt/buildscriptgen/GenerateBuildScript
 
 FROM python AS final
@@ -319,7 +306,7 @@ RUN ln -s /opt/php/5.6 /opt/php/5 \
 
 # Build script generator content, currently docker doesn't support variables in --from
 # so we are building an extra stage to copy binaries from correct build stage
-COPY --from=buildscriptbinaries /opt/buildscriptgen/ /opt/buildscriptgen/
+COPY --from=buildscriptbuilder /opt/buildscriptgen/ /opt/buildscriptgen/
 RUN ln -s /opt/buildscriptgen/GenerateBuildScript /usr/local/bin/oryx
 
 # Bake Application Insights key from pipeline variable into final image

vsts/pipelines/ci.yml

@@ -1,17 +1,16 @@
 resources:
 - repo: self
 jobs:
-- job: Job_1
+- job: Job_Security
   displayName: Security
   condition: succeeded()
   pool:
     name: Hosted VS2017
   steps:
-  - template: _securityChecks.yml
+  - template: templates/_securityChecks.yml
 
-- job: Job_2
+- job: Job_SignBinaries
   displayName: Build and Sign Oryx Binaries
-  condition: succeeded()
   pool:
     name: VSEng-MicroBuildVS2017
     demands:
@@ -20,26 +19,54 @@ jobs:
   variables:
     SignType: 'test'
   steps:
-  - template: _signbinary.yml
+  - template: templates/_signbinary.yml
 
-- job: Job_3
-  displayName: Build and Publish Oryx Images
-  dependsOn: Job_2
+- job: Job_BuildImage
+  displayName: Build and Test Build Image
+  dependsOn: Job_SignBinaries
   condition: succeeded()
-  timeoutInMinutes: 250
   pool:
     name: OryxLinux
+  variables:
+    SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ]  # map in the signtype variable
+
   steps:
   - script: |
-        echo "##vso[task.setvariable variable=BuildBuildImages;]true"
-        echo "##vso[task.setvariable variable=TestBuildImages;]true"
-        echo "##vso[task.setvariable variable=BuildRuntimeImages;]true"
-        echo "##vso[task.setvariable variable=TestRuntimeImages;]true"
-        echo "##vso[task.setvariable variable=TestIntegration;]true"
-        echo "##vso[task.setvariable variable=PushBuildImages;]true"
-        echo "##vso[task.setvariable variable=PushRuntimeImages;]true"
-        echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
+      echo "##vso[task.setvariable variable=BuildBuildImages;]true"
+      echo "##vso[task.setvariable variable=TestBuildImages;]true"
+      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=PushBuildImages;]true"
+      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
     displayName: 'Set variables'
-  - template: _buildTemplate.yml
+
+  - template: templates/_buildParallel.yml
+    parameters:
+      pushToDockerHub: 'true'
+
+- job: Job_RuntimeImages
+  displayName: Build and Test Runtime Images
+  dependsOn: Job_SignBinaries
+  condition: succeeded()
+  pool:
+    name: OryxLinux
+  # Building runtime images can take a long time due our PHP images
+  timeoutInMinutes: 100
+  steps:
+  - script: |
+      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
+      echo "##vso[task.setvariable variable=TestBuildImages;]false"
+      echo "##vso[task.setvariable variable=BuildRuntimeImages;]true"
+      echo "##vso[task.setvariable variable=TestRuntimeImages;]true"
+      echo "##vso[task.setvariable variable=PushRuntimeImages;]true"
+      echo "##vso[task.setvariable variable=PushBuildImages;]false"
+      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
+    displayName: 'Set variables'
+  - template: templates/_buildParallel.yml
+    parameters:
+      pushToDockerHub: 'true'
+
+- template: templates/_integration.yml
 
 trigger: none

vsts/pipelines/nightly.yml

@@ -8,11 +8,10 @@ jobs:
   pool:
     name: Hosted VS2017
   steps:
-  - template: _securityChecks.yml
+  - template: templates/_securityChecks.yml
 
 - job: Job_BuildImage
   displayName: Build and Test Build Image
-  condition: succeeded()
   pool:
     name: OryxLinux
   steps:
@@ -23,18 +22,19 @@ jobs:
       echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
       echo "##vso[task.setvariable variable=PushBuildImages;]true"
       echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
       echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
     displayName: 'Set variables'
-  - template: _buildParallel.yml
+  - template: templates/_buildParallel.yml
+    parameters:
+      pushToDockerHub: 'false'
 
 - job: Job_RuntimeImages
   displayName: Build and Test Runtime Images
   condition: succeeded()
   pool:
     name: OryxLinux
   # Building runtime images can take a long time due our PHP images
-  timeoutInMinutes: 150
+  timeoutInMinutes: 100
   steps:
   - script: |
       echo "##vso[task.setvariable variable=BuildBuildImages;]false"
@@ -43,140 +43,12 @@ jobs:
       echo "##vso[task.setvariable variable=TestRuntimeImages;]true"
       echo "##vso[task.setvariable variable=PushRuntimeImages;]true"
       echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
       echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
     displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_PythonIntegrationTests
-  displayName: Run Python Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  timeoutInMinutes: 150
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_DotNetCoreIntegrationTests
-  displayName: Run DotNetCore Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_NodeIntegrationTests
-  displayName: Run NodeJs Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  timeoutInMinutes: 150
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_PhpIntegrationTests
-  displayName: Run Php Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_DbIntegrationTests
-  displayName: Run Database Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=db"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]false"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
+  - template: templates/_buildParallel.yml
+    parameters:
+      pushToDockerHub: 'false'
 
-- job: Job_ENDIntegrationTests
-  displayName: End of Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-    - Job_PythonIntegrationTests
-    - Job_PhpIntegrationTests
-    - Job_NodeIntegrationTests
-    - Job_DotNetCoreIntegrationTests
-    - Job_DbIntegrationTests
-  pool:
-    name: OryxLinux
-  steps:
-  - task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0
-    displayName: 'End of Integration Tests'
-    inputs:
-      type: InlineScript
-      script: 'echo "Integration tests ended"'
+- template: templates/_integration.yml
 
 trigger: none