Cache nuget packages

Cache nuget packags in libraries build TOREVERT: Just build windows/x64 libraries Use Cache_Key variable No cache key PreBuildSteps fix echo statement Set cacheHitVar Add variables space space 2 no need of another variables Print CACHE_RESTORED Cache based on commit hash fix space Remove condition from upload Remove one more condition minor change Try upload artifact step misc changes in libraries Use caching for Coreclr builds Print commit hashes print dir Point to staging folder Skip zipping if using cached Add debugging logic add | Comment coreclr product for now useCached1,2,3 Change git log command upload-cached-artifact and checking about CACHE_RESTORED get-commit-hash.cmd Another CACHE_RESTORED try fix minor issue script fix use no-pager Use git log in yml Use CACHE_RESTORED in template files directly Echo git commands Temp disable building command Add -- Use python script to get commits fix the cache key add condition at one place Disable commit cachekey Move the conditions properly: Build coreclr with cache Use DAC_CACHE_RESTORED Run for all OS/platforms include arch in cache Includearch on libraries Remove failing test keys Just do windows x64/x86 Cache build log for libraries UsePipelineCache Remove the condition for DAC_CACHE_RESTORED caching fix the coreclr sample key Make space change Fix the missing ) Fix the missing ) variable UsePipelineCache Use variables.UsePipelineCache Fix the missing ) Fix variables.UsePipelineCache Add - before name/value Use cache variable names Just build libraries x64 Print build reason echo %BUILD_REASON% Change to condition Enable windows x86 libraries: fix publish logs enable libraries for all platforms use actual cache key Enable for coreclr all platforms Misc. changes + cleanup Enable Linux for debugging Use *.* Use full path Full expand Expand cache key path Cache libraries/coreclr for all platforms shorten the coreclr cache key for testing just run for win/x86 fix the cross dac problem fix the parameters removing ) Skip restore internal tools Cache build logs Reintroduce python script for git hash fix build error add logging to see why things are failing wrap in quotes python inline fix alignment: Add back print fix the yml files Run git fetch Better history fetching Enable linux x64 to check the errors convert from deepen to depth use access token replace dceng as well: fix typo Remove print tool retry logic Add git log fix exit_on_fail print git version no pager for initial log fix error Enable Linux build enable for all platforms Do not use retry logic create crossDacBuild Change from md to mkdir enable all platforms Use mkdir -p Use label proper label do not cache/restore for DAC Reenable for all platforms Use updated commit_hash file Enable libraries as well Rename some variables and comments Dummy change Revert "Dummy change" This reverts commit 645b20b7fe53e1f3787e42bee7add9895acdbc1e. Flip the condition to fix Mono builds Print source_directory Use os/subob/archtype/ Dummy change Fix the os arch names
dotnet · Dec 7, 2021 · 3c71025 · 3c71025
1 parent f388909
commit 3c71025
Show file tree

Hide file tree

Showing 9 changed files with 322 additions and 23 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# .NET Runtime
+# .NET Runtime 
 [![Build Status](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/runtime/runtime?branchName=main)](https://dnceng.visualstudio.com/public/_build/latest?definitionId=686&branchName=main)
 [![Help Wanted](https://img.shields.io/github/issues/dotnet/runtime/up-for-grabs?style=flat-square&color=%232EA043&label=help%20wanted)](https://github.com/dotnet/runtime/issues?q=is%3Aissue+is%3Aopen+label%3A%22up-for-grabs%22)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dotnet/runtime)

diff --git a/eng/pipelines/common/restore-internal-tools.yml b/eng/pipelines/common/restore-internal-tools.yml
@@ -3,6 +3,7 @@ steps:
     inputs:
       nuGetServiceConnections: 'dotnet-core-internal-tooling'
       forceReinstallCredentialProvider: true
+    condition: ne(variables.BUILD_CACHE_RESTORED, 'true')
 
   - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt)
             -ci
@@ -12,4 +13,4 @@ steps:
             /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/RestoreInternal.binlog
             /v:normal
     displayName: Restore internal tools
-    condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], 'true'))
+    condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], 'true'), ne(variables.BUILD_CACHE_RESTORED, 'true'))
diff --git a/eng/pipelines/common/upload-artifact-step.yml b/eng/pipelines/common/upload-artifact-step.yml
@@ -8,7 +8,7 @@ parameters:
   displayName: ''
 
 steps:
-  # Zip Artifact
+  # Zip Artifact only if not cached
   - task: ArchiveFiles@2
     displayName: 'Zip ${{ parameters.displayName }}'
     inputs:
@@ -17,6 +17,7 @@ steps:
       archiveType:       ${{ parameters.archiveType }}
       tarCompression:    ${{ parameters.tarCompression }}
       includeRootFolder: ${{ parameters.includeRootFolder }}
+    condition: ne(variables.BUILD_CACHE_RESTORED, 'true')
 
   - task: PublishBuildArtifacts@1
     displayName: 'Publish ${{ parameters.displayName }}'

diff --git a/eng/pipelines/coreclr/exploratory.yml b/eng/pipelines/coreclr/exploratory.yml
@@ -29,26 +29,26 @@ jobs:
     - windows_x64
     - windows_x86
     - windows_arm64
-    - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
+    # - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
     jobParameters:
       testGroup: outerloop
 
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/jit-exploratory-job.yml
-    buildConfig: checked
-    platforms:
-    # Linux tests are built on the OSX machines.
-    # - OSX_x64
-    - Linux_arm
-    - Linux_arm64
-    - Linux_x64
-    - windows_x64
-    - windows_x86
-    - windows_arm64
-    helixQueueGroup: ci
-    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
-    jobParameters:
-      testGroup: outerloop
-      liveLibrariesBuildConfig: Release
+# - template: /eng/pipelines/common/platform-matrix.yml
+#   parameters:
+#     jobTemplate: /eng/pipelines/coreclr/templates/jit-exploratory-job.yml
+#     buildConfig: checked
+#     platforms:
+#     # Linux tests are built on the OSX machines.
+#     # - OSX_x64
+#     - Linux_arm
+#     - Linux_arm64
+#     - Linux_x64
+#     - windows_x64
+#     - windows_x86
+#     - windows_arm64
+#     helixQueueGroup: ci
+#     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+#     jobParameters:
+#       testGroup: outerloop
+#       liveLibrariesBuildConfig: Release
       toolName: ${{ variables.toolName }}
diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml
@@ -152,21 +152,102 @@ jobs:
       - name: clrRuntimePortableBuildArg
         value: '-portablebuild=false'
 
+    - name: CommitsOutputFile
+      value: '$(Build.SourcesDirectory)/coreclr.commit_hash'
+
+    - name: GetCommitScript
+      value: '$(Build.SourcesDirectory)/src/coreclr/scripts/get_commit_hash.py'
+
+    # Currently only support pipeline caching for PRs and manual triggered runs
+    - ${{ if or(in(variables['Build.Reason'], 'PullRequest'), in(variables['Build.Reason'], 'Manual')) }}:
+      - name: UsePipelineCache
+        value: true
+
+    # Otherwise set all the cache-specific variables to 'false' which will ensure to build product.
+    - ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), notin(variables['Build.Reason'], 'Manual')) }}:
+      - name: UsePipelineCache
+        value: false
+      - name: BUILD_CACHE_RESTORED
+        value: false
+      - name: DAC_BUILD_CACHE_RESTORED
+        value: false
+      - name: BUILD_LOGS_RESTORED
+        value: false
+
+    - ${{ if eq(parameters.osGroup, 'windows') }}:
+      - name: PythonScript
+        value: 'py -3'
+      - name: PipScript
+        value: 'py -3 -m pip'
+
+    - ${{ if ne(parameters.osGroup, 'windows') }}:
+      - name: PythonScript
+        value: 'python3'
+      - name: PipScript
+        value: 'pip3'
+
     - ${{ parameters.variables }}
 
     steps:
 
+    # Get the commit hashes of interesting folder for caching
+    - script: |
+        $(PythonScript) $(GetCommitScript) -run_type coreclr -output $(CommitsOutputFile)
+      displayName: Record commit hashes of coreclr
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+        REPO_URI: $(Build.Repository.Uri)
+      condition: eq(variables.UsePipelineCache, 'true')
+
+    # Cache tasks
+    - task: Cache@2
+      displayName: Cache coreclr build
+      inputs:
+        key: '" coreclr-build" | "${{ parameters.runtimeVariant }}" | "${{ parameters.osGroup }}" | "${{ parameters.osSubgroup }}" | "${{ parameters.archType }}" | "${{ parameters.buildConfig }}" | $(CommitsOutputFile)'
+        path: $(Build.StagingDirectory)
+        cacheHitVar: BUILD_CACHE_RESTORED
+      condition: eq(variables.UsePipelineCache, 'true')
+
+    - ${{ if and(in(parameters.osGroup, 'windows', 'Linux'), ne(parameters.archType, 'x86'), ne(parameters.compilerName, 'gcc'), ne(parameters.testGroup, 'clrTools'), eq(parameters.pgoType, '')) }}:
+      - task: Cache@2
+        displayName: Cache cross DAC build
+        inputs:
+          key: '" crossDAC-build" | "${{ parameters.runtimeVariant }}" | "${{ parameters.osGroup }}" | "${{ parameters.osSubgroup }}" | "${{ parameters.archType }}" | "${{ parameters.buildConfig }}" | $(CommitsOutputFile)'
+          path: $(crossDacArtifactPath)
+          cacheHitVar: DAC_BUILD_CACHE_RESTORED
+        condition: eq(variables.UsePipelineCache, 'true')
+
+    - task: Cache@2
+      displayName: Cache coreclr build logs
+      inputs:
+        key: '"coreclr-logs" | "${{ parameters.runtimeVariant }}" | "${{ parameters.osGroup }}" | "${{ parameters.osSubgroup }}" | "${{ parameters.archType }}" | "${{ parameters.buildConfig }}" | $(CommitsOutputFile)'
+        path: $(Build.SourcesDirectory)/artifacts/log/
+        cacheHitVar: BUILD_LOGS_RESTORED
+        condition: eq(variables.UsePipelineCache, 'true')
+
+    - script: |
+        echo Cache restored
+      displayName: Cache Hit
+      condition: eq(variables.BUILD_CACHE_RESTORED, 'true')
+
+    - script: |
+        echo Cache not restored
+      displayName: Cache miss
+      condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
+
     # Install native dependencies
     # Linux builds use docker images with dependencies preinstalled,
     # and FreeBSD builds use a build agent with dependencies
     # preinstalled, so we only need this step for OSX and Windows.
     - ${{ if eq(parameters.osGroup, 'OSX') }}:
       - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) ${{ parameters.archType }} azDO
         displayName: Install native dependencies
+        condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
     - ${{ if eq(parameters.osGroup, 'windows') }}:
       # Necessary to install python
       - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
         displayName: Install native dependencies
+        condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
 
     # Install internal tools on official builds
     # Since our internal tools are behind an authenticated feed,
@@ -195,14 +276,18 @@ jobs:
     # Build/Generate native prerequisites
     - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.nativeprereqs $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci /bl:$(Build.SourcesDirectory)artifacts/log/$(buildConfig)/CoreCLRNativePrereqs.binlog
       displayName: Build and generate native prerequisites
+      condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
 
     # Build CoreCLR Runtime
     - ${{ if ne(parameters.osGroup, 'windows') }}:
       - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(clrRuntimePortableBuildArg) $(CoreClrPgoDataArg)
         displayName: Build CoreCLR Runtime
+        condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
+
     - ${{ if eq(parameters.osGroup, 'windows') }}:
       - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg)
         displayName: Build CoreCLR Runtime
+        condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
 
     - ${{ if in(parameters.osGroup, 'OSX', 'iOS', 'tvOS') }}:
       - script: |
@@ -214,9 +299,12 @@ jobs:
     - ${{ if or(ne(parameters.osGroup, 'Linux'), ne(parameters.archType, 'x86')) }}:
       - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.tools+clr.packages+clr.paltestlist $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci
         displayName: Build managed product components and packages
+        condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
+
     - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.archType, 'x86')) }}:
       - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci
         displayName: Build managed product components and packages
+        condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
 
     # Run CoreCLR Tools unit tests
     - ${{ if eq(parameters.testGroup, 'clrTools') }}:
@@ -227,6 +315,7 @@ jobs:
     - ${{ if and(ne(parameters.isOfficialBuild, true), ne(parameters.disableClrTest, true)) }}:
       - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(osArg) $(priorityArg) $(compilerArg)
         displayName: Build native test components
+        condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
 
     # Build libs.native, host.native and mono with gcc
     - ${{ if eq(parameters.compilerName, 'gcc') }}:

diff --git a/eng/pipelines/coreclr/templates/crossdac-build.yml b/eng/pipelines/coreclr/templates/crossdac-build.yml
@@ -8,9 +8,11 @@ steps:
   - ${{ if and(eq(parameters.osGroup, 'windows'), notin(parameters.archType, 'x86')) }}:
     - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -linuxdac -ninja $(officialBuildIdArg)
       displayName: Build Cross OS Linux DAC for Windows
+      condition: ne(variables.DAC_BUILD_CACHE_RESTORED, 'true')
 
     - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -alpinedac -ninja $(officialBuildIdArg)
       displayName: Build Cross OS Linux-musl DAC for Windows
+      condition: ne(variables.DAC_BUILD_CACHE_RESTORED, 'true')
 
     - task: CopyFiles@2
       displayName: Gather CrossDac Artifacts (Linux)
@@ -20,6 +22,7 @@ steps:
           **/*
           !**/sharedFramework/**/*
         TargetFolder: $(buildLinuxDacStagingPath)
+      condition: ne(variables.DAC_BUILD_CACHE_RESTORED, 'true')
 
     - task: CopyFiles@2
       displayName: Gather CrossDac Artifacts (Linux_musl)
@@ -29,6 +32,7 @@ steps:
           **/*
           !**/sharedFramework/**/*
         TargetFolder: '$(buildMuslDacStagingPath)'
+      condition: ne(variables.DAC_BUILD_CACHE_RESTORED, 'true')
 
   - ${{ if eq(parameters.osGroup, 'Linux') }}:
     - task: CopyFiles@2
@@ -37,6 +41,7 @@ steps:
         SourceFolder: $(coreClrProductRootFolderPath)
         Contents: libcoreclr.so
         TargetFolder: '$(crossDacArtifactPath)/${{ parameters.osGroup }}${{ parameters.osSubgroup }}.$(archType).$(buildConfigUpper)/$(crossDacHostArch)'
+      condition: ne(variables.DAC_BUILD_CACHE_RESTORED, 'true')
 
   # Make the assets available in a single container for the packaging job.
   - task: PublishBuildArtifacts@1

diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml
@@ -81,9 +81,77 @@ jobs:
         - ${{ if eq(parameters.useHelix, true) }}:
           - _additionalBuildArguments: /p:ArchiveTests=true
 
+        # Currently only support pipeline caching for PRs and manual triggered runs
+        - ${{ if or(in(variables['Build.Reason'], 'PullRequest'), in(variables['Build.Reason'], 'Manual')) }}:
+          - name: UsePipelineCache
+            value: true
+
+        - name: CommitsOutputFile
+          value: '$(Build.SourcesDirectory)/libraries.commit_hash'
+        - name: GetCommitScript
+          value: '$(Build.SourcesDirectory)/src/coreclr/scripts/get_commit_hash.py'
+
+        # Otherwise set all the cache-specific variables to 'false' which will ensure to build product.
+        - ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), notin(variables['Build.Reason'], 'Manual')) }}:
+          - name: UsePipelineCache
+            value: false
+          - name: BUILD_CACHE_RESTORED
+            value: false
+          - name: BUILD_LOGS_RESTORED
+            value: false
+
+        - ${{ if eq(parameters.osGroup, 'windows') }}:
+          - name: PythonScript
+            value: 'py -3'
+          - name: PipScript
+            value: 'py -3 -m pip'
+
+        - ${{ if ne(parameters.osGroup, 'windows') }}:
+          - name: PythonScript
+            value: 'python3'
+          - name: PipScript
+            value: 'pip3'
+
         - ${{ parameters.variables }}
 
       steps:
+
+        # Get the commit hashes of interesting folder for caching
+        - script: |
+            $(PythonScript) $(GetCommitScript) -run_type libraries -output $(CommitsOutputFile)
+          displayName: Record commit hashes of libraries
+          env:
+            SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+            REPO_URI: $(Build.Repository.Uri)
+          condition: eq(variables.UsePipelineCache, 'true')
+
+        #  Cache tasks
+        - task: Cache@2
+          displayName: Cache libraries build
+          inputs:
+            key: '"libraries-build" | "${{ parameters.runtimeVariant }}" | "${{ parameters.osGroup }}" | "${{ parameters.osSubgroup }}" | "${{ parameters.archType }}" | "${{ parameters.buildConfig }}" | $(CommitsOutputFile)'
+            path: $(Build.StagingDirectory)
+            cacheHitVar: BUILD_CACHE_RESTORED
+          condition: eq(variables.UsePipelineCache, 'true')
+
+        - task: Cache@2
+          displayName: Cache libraries build logs
+          inputs:
+            key: '"libraries-logs" | "${{ parameters.runtimeVariant }}" | "${{ parameters.osGroup }}" | "${{ parameters.osSubgroup }}" | "${{ parameters.archType }}" | "${{ parameters.buildConfig }}" | $(CommitsOutputFile)'
+            path: $(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)
+            cacheHitVar: BUILD_LOGS_RESTORED
+            condition: eq(variables.UsePipelineCache, 'true')
+
+        - script: |
+            echo Cache restored
+          displayName: Cache Hit
+          condition: eq(variables.BUILD_CACHE_RESTORED, 'true')
+
+        - script: |
+            echo Cache not restored
+          displayName: Cache Miss
+          condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
+
         - ${{ if eq(parameters.isOfficialBuild, true) }}:
           - template: /eng/pipelines/common/restore-internal-tools.yml
 
@@ -102,6 +170,7 @@ jobs:
                 $(_buildArguments)
                 $(_additionalBuildArguments)
           displayName: Restore and Build Product
+          condition: eq(variables.BUILD_CACHE_RESTORED, 'false')
 
         - ${{ if in(parameters.osGroup, 'OSX', 'iOS', 'tvOS') }}:
           - script: |

diff --git a/eng/pipelines/libraries/prepare-for-bin-publish.yml b/eng/pipelines/libraries/prepare-for-bin-publish.yml
@@ -4,6 +4,9 @@ parameters:
   isOfficialBuild: ''
 
 steps:
+
+
+
   - ${{ if ne(parameters.isOfficialBuild, true) }}:
     - task: CopyFiles@2
       displayName: Prepare bin folders to publish (unofficial build)
@@ -14,6 +17,7 @@ steps:
           ref/**
           runtime/**
           testhost/**
+      condition: ne(variables.BUILD_CACHE_RESTORED, 'true')
 
   - task: CopyFiles@2
     displayName: Prepare bin folders to publish
@@ -24,10 +28,11 @@ steps:
         docs/**
         microsoft.netcore.app.*/**
         native/**
+    condition: ne(variables.BUILD_CACHE_RESTORED, 'true')
 
   - task: CopyFiles@2
     displayName: Prepare artifacts packages folder to publish
     inputs:
       sourceFolder: $(Build.SourcesDirectory)/artifacts/packages
       targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/packages
-    condition: and(succeeded(), eq(variables['_librariesBuildProducedPackages'], true))
+    condition: and(and(succeeded(), eq(variables['_librariesBuildProducedPackages'], true)), ne(variables.BUILD_CACHE_RESTORED, 'true'))