updating with few fixes to make ci builds parallel

update updating few fixes to ci build update remove trigger remove trigger initial experiment update ci def remove condition from ci udpate dependency update dependency implementing parallel ci build fix ci fix signing issue fix yaml troubleshoot signtype troubleshoot signtype delete _nightly.yml reorganising templates and build definitions remove file writing steps for successful integration test update download artifact step
microsoft · May 21, 2019 · 05b8637 · 05b8637
1 parent ac67f45
commit 05b8637
Show file tree

Hide file tree

Showing 11 changed files with 174 additions and 312 deletions.
diff --git a/build/build-buildimages.sh b/build/build-buildimages.sh
@@ -16,6 +16,8 @@ cd "$BUILD_IMAGES_BUILD_CONTEXT_DIR"
 declare BUILDSCRIPT_SOURCE="buildscriptbuilder"
 declare BUILD_SIGNED=""
 
+echo "SignType is: "$SignType
+
 # Check to see if the build is by scheduled ORYX-CI or other azure devops build
 if [ "$SignType" == "real" ] || [ "$SignType" == "Real" ]
 then

diff --git a/vsts/pipelines/buildimage-bases.yml b/vsts/pipelines/buildimage-bases.yml
@@ -5,35 +5,35 @@ variables:
 
 jobs:
 
-- template: _buildimage-bases.job-template.yml
+- template: templates/_buildimage-bases.job-template.yml
   parameters:
     displayName: Build and push Node base images for runtime
     scriptPath: ./build/build-runtimeimages-bases.sh
     imageDir: node
     artifactsFileName: node-runtimeimage-bases.txt
 
-- template: _buildimage-bases.job-template.yml
+- template: templates/_buildimage-bases.job-template.yml
   parameters:
     displayName: Build and push Python base images for build
     scriptPath: ./build/build-buildimage-bases.sh
     imageDir: python
     artifactsFileName: python-buildimage-bases.txt
 
-- template: _buildimage-bases.job-template.yml
+- template: templates/_buildimage-bases.job-template.yml
   parameters:
     displayName: Build and push PHP base images for build
     scriptPath: ./build/build-buildimage-bases.sh
     imageDir: php
     artifactsFileName: php-buildimage-bases.txt
 
-- template: _buildimage-bases.job-template.yml
+- template: templates/_buildimage-bases.job-template.yml
   parameters:
     displayName: Build and push PHP base images for runtime
     scriptPath: ./build/build-runtimeimages-bases.sh
     imageDir: php
     artifactsFileName: php-runtimeimage-bases.txt
 
-- template: _buildimage-bases.job-template.yml
+- template: templates/_buildimage-bases.job-template.yml
   parameters:
     displayName: Build and push Yarn Cache base image
     scriptPath: ./build/build-buildimage-bases.sh

diff --git a/vsts/pipelines/ci.yml b/vsts/pipelines/ci.yml
@@ -1,17 +1,16 @@
 resources:
 - repo: self
 jobs:
-- job: Job_1
+- job: Job_Security
   displayName: Security
   condition: succeeded()
   pool:
     name: Hosted VS2017
   steps:
-  - template: _securityChecks.yml
+  - template: templates/_securityChecks.yml
 
-- job: Job_2
+- job: Job_SignBinaries
   displayName: Build and Sign Oryx Binaries
-  condition: succeeded()
   pool:
     name: VSEng-MicroBuildVS2017
     demands:
@@ -20,44 +19,51 @@ jobs:
   variables:
     SignType: 'test'
   steps:
-  - template: _signbinary.yml
+  - template: templates/_signbinary.yml
 
 - job: Job_BuildImage
   displayName: Build and Test Build Image
-  dependsOn: Job_2
+  dependsOn: Job_SignBinaries
   condition: succeeded()
   pool:
     name: OryxLinux
-  steps:
-  - task: DownloadPipelineArtifact@0
-    displayName: 'Download Pipeline Artifact'
-    inputs:
-      pipelineId: 'Oryx-CI'
-      artifactName: 'drop-$(Build.BuildNumber)'
-      targetPath: '$(Build.SourcesDirectory)'
-    condition: >
-     and(eq(variables['Build.DefinitionName'], 'Oryx-CI'), in(variables['Build.Reason'], 'Schedule', 'Manual'),
-     or(startsWith(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'],'refs/heads/patch/' )))
+  variables:
+    SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ]  # map in the signtype variable
 
+  steps:
   - script: |
+      echo "Sign Type is"
+      echo $(SignType)
       echo "##vso[task.setvariable variable=BuildBuildImages;]true"
       echo "##vso[task.setvariable variable=TestBuildImages;]true"
       echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
       echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
       echo "##vso[task.setvariable variable=PushBuildImages;]true"
       echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]true"
       echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
     displayName: 'Set variables'
-  - template: _buildParallel.yml
+    
+  - task: DownloadPipelineArtifact@0
+    displayName: 'Download Pipeline Artifact'
+    inputs:
+      pipelineId: 'Oryx-CI'
+      artifactName: 'drop-$(Build.BuildNumber)'
+      targetPath: '$(Build.SourcesDirectory)'
+    condition: >
+     and(eq(variables['Build.DefinitionName'], 'Oryx-CI'), in(variables['Build.Reason'], 'Schedule', 'Manual'),
+     or(startsWith(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'],'refs/heads/patch/' )))
+   
+  - template: templates/_buildParallel.yml
+    parameters:
+      pushToDockerHub: 'true'
 
 - job: Job_RuntimeImages
   displayName: Build and Test Runtime Images
   condition: succeeded()
   pool:
     name: OryxLinux
   # Building runtime images can take a long time due our PHP images
-  timeoutInMinutes: 150
+  timeoutInMinutes: 100
   steps:
   - script: |
       echo "##vso[task.setvariable variable=BuildBuildImages;]false"
@@ -66,139 +72,12 @@ jobs:
       echo "##vso[task.setvariable variable=TestRuntimeImages;]true"
       echo "##vso[task.setvariable variable=PushRuntimeImages;]true"
       echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]true"
       echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true"
     displayName: 'Set variables'
-  - template: _buildParallel.yml
+  - template: templates/_buildParallel.yml
+    parameters:
+      pushToDockerHub: 'true'
 
-- job: Job_PythonIntegrationTests
-  displayName: Run Python Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  timeoutInMinutes: 150
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]true"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
+- template: templates/_integration.yml
 
-- job: Job_DotNetCoreIntegrationTests
-  displayName: Run DotNetCore Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]true"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_NodeIntegrationTests
-  displayName: Run NodeJs Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  timeoutInMinutes: 150
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]true"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_PhpIntegrationTests
-  displayName: Run Php Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]true"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_DbIntegrationTests
-  displayName: Run Database Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-  pool:
-    name: OryxLinux
-  steps:
-  - script: |
-      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
-      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestBuildImages;]false"
-      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=db"
-      echo "##vso[task.setvariable variable=TestIntegration;]true"
-      echo "##vso[task.setvariable variable=PushBuildImages;]false"
-      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
-      echo "##vso[task.setvariable variable=PushToDockerHub;]true"
-      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
-    displayName: 'Set variables'
-  - template: _buildParallel.yml
-
-- job: Job_ENDIntegrationTests
-  displayName: End of Integration Tests
-  dependsOn: 
-    - Job_BuildImage
-    - Job_RuntimeImages
-    - Job_PythonIntegrationTests
-    - Job_PhpIntegrationTests
-    - Job_NodeIntegrationTests
-    - Job_DotNetCoreIntegrationTests
-    - Job_DbIntegrationTests
-  pool:
-    name: OryxLinux
-  steps:
-  - task: ms-devlabs.utilitytasks.task-Shellpp.Shell++@0
-    displayName: 'End of Integration Tests'
-    inputs:
-      type: InlineScript
-      script: 'echo "Integration tests ended"'
 trigger: none