Skip to content

Commit

Permalink
Merge pull request #1882 from actions/enhance-blob-client
Browse files Browse the repository at this point in the history 
Enhance blob client resilience & performance
  • Loading branch information
@Link-
Link- authored Dec 2, 2024
2 parents 9cc30cb + c02c929 commit a10e209
Show file tree
Hide file tree
Showing 11 changed files with 514 additions and 256 deletions.
43 changes: 36 additions & 7 deletions packages/cache/__tests__/options.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ const downloadConcurrency = 8
const timeoutInMs = 30000
const segmentTimeoutInMs = 600000
const lookupOnly = false
const uploadConcurrency = 4
const uploadChunkSize = 32 * 1024 * 1024

test('getDownloadOptions sets defaults', async () => {
const actualOptions = getDownloadOptions()
Expand Down Expand Up @@ -43,25 +41,56 @@ test('getDownloadOptions overrides all settings', async () => {
})

test('getUploadOptions sets defaults', async () => {
const expectedOptions: UploadOptions = {
uploadConcurrency: 4,
uploadChunkSize: 32 * 1024 * 1024,
useAzureSdk: false
}
const actualOptions = getUploadOptions()

expect(actualOptions).toEqual({
uploadConcurrency,
uploadChunkSize
})
expect(actualOptions).toEqual(expectedOptions)
})

test('getUploadOptions overrides all settings', async () => {
const expectedOptions: UploadOptions = {
uploadConcurrency: 2,
uploadChunkSize: 16 * 1024 * 1024
uploadChunkSize: 16 * 1024 * 1024,
useAzureSdk: true
}

const actualOptions = getUploadOptions(expectedOptions)

expect(actualOptions).toEqual(expectedOptions)
})

test('env variables override all getUploadOptions settings', async () => {
const expectedOptions: UploadOptions = {
uploadConcurrency: 16,
uploadChunkSize: 64 * 1024 * 1024,
useAzureSdk: true
}

process.env.CACHE_UPLOAD_CONCURRENCY = '16'
process.env.CACHE_UPLOAD_CHUNK_SIZE = '64'

const actualOptions = getUploadOptions(expectedOptions)
expect(actualOptions).toEqual(expectedOptions)
})

test('env variables override all getUploadOptions settings but do not exceed caps', async () => {
const expectedOptions: UploadOptions = {
uploadConcurrency: 32,
uploadChunkSize: 128 * 1024 * 1024,
useAzureSdk: true
}

process.env.CACHE_UPLOAD_CONCURRENCY = '64'
process.env.CACHE_UPLOAD_CHUNK_SIZE = '256'

const actualOptions = getUploadOptions(expectedOptions)
expect(actualOptions).toEqual(expectedOptions)
})

test('getDownloadOptions overrides download timeout minutes', async () => {
const expectedOptions: DownloadOptions = {
useAzureSdk: false,
Expand Down
75 changes: 24 additions & 51 deletions packages/cache/__tests__/restoreCacheV2.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ import * as path from 'path'
import * as tar from '../src/internal/tar'
import * as config from '../src/internal/config'
import * as cacheUtils from '../src/internal/cacheUtils'
import * as downloadCacheModule from '../src/internal/blob/download-cache'
import * as cacheHttpClient from '../src/internal/cacheHttpClient'
import {restoreCache} from '../src/cache'
import {CacheFilename, CompressionMethod} from '../src/internal/constants'
import {CacheServiceClientJSON} from '../src/generated/results/api/v1/cache.twirp'
import {BlobDownloadResponseParsed} from '@azure/storage-blob'
import {DownloadOptions} from '../src/options'

jest.mock('../src/internal/cacheHttpClient')
jest.mock('../src/internal/cacheUtils')
Expand Down Expand Up @@ -142,6 +142,7 @@ test('restore with gzip compressed cache found', async () => {
const signedDownloadUrl = 'https://blob-storage.local?signed=true'
const cacheVersion =
'd90f107aaeb22920dba0c637a23c37b5bc497b4dfa3b07fe3f79bf88a273c11b'
const options = {useAzureSdk: true} as DownloadOptions

const getCacheVersionMock = jest.spyOn(cacheUtils, 'getCacheVersion')
getCacheVersionMock.mockReturnValue(cacheVersion)
Expand Down Expand Up @@ -169,17 +170,7 @@ test('restore with gzip compressed cache found', async () => {
})

const archivePath = path.join(tempPath, CacheFilename.Gzip)
const downloadCacheFileMock = jest.spyOn(
downloadCacheModule,
'downloadCacheFile'
)
downloadCacheFileMock.mockReturnValue(
Promise.resolve({
_response: {
status: 200
}
} as BlobDownloadResponseParsed)
)
const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')

const fileSize = 142
const getArchiveFileSizeInBytesMock = jest
Expand All @@ -189,7 +180,7 @@ test('restore with gzip compressed cache found', async () => {
const extractTarMock = jest.spyOn(tar, 'extractTar')
const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile')

const cacheKey = await restoreCache(paths, key)
const cacheKey = await restoreCache(paths, key, [], options)

expect(cacheKey).toBe(key)
expect(getCacheVersionMock).toHaveBeenCalledWith(
Expand All @@ -203,9 +194,10 @@ test('restore with gzip compressed cache found', async () => {
version: cacheVersion
})
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
expect(downloadCacheFileMock).toHaveBeenCalledWith(
expect(downloadCacheMock).toHaveBeenCalledWith(
signedDownloadUrl,
archivePath
archivePath,
options
)
expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath)
expect(logInfoMock).toHaveBeenCalledWith(`Cache Size: ~0 MB (142 B)`)
Expand All @@ -226,6 +218,7 @@ test('restore with zstd compressed cache found', async () => {
const signedDownloadUrl = 'https://blob-storage.local?signed=true'
const cacheVersion =
'8e2e96a184cb0cd6b48285b176c06a418f3d7fce14c29d9886fd1bb4f05c513d'
const options = {useAzureSdk: true} as DownloadOptions

const getCacheVersionMock = jest.spyOn(cacheUtils, 'getCacheVersion')
getCacheVersionMock.mockReturnValue(cacheVersion)
Expand Down Expand Up @@ -253,17 +246,7 @@ test('restore with zstd compressed cache found', async () => {
})

const archivePath = path.join(tempPath, CacheFilename.Zstd)
const downloadCacheFileMock = jest.spyOn(
downloadCacheModule,
'downloadCacheFile'
)
downloadCacheFileMock.mockReturnValue(
Promise.resolve({
_response: {
status: 200
}
} as BlobDownloadResponseParsed)
)
const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')

const fileSize = 62915000
const getArchiveFileSizeInBytesMock = jest
Expand All @@ -273,7 +256,7 @@ test('restore with zstd compressed cache found', async () => {
const extractTarMock = jest.spyOn(tar, 'extractTar')
const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile')

const cacheKey = await restoreCache(paths, key)
const cacheKey = await restoreCache(paths, key, [], options)

expect(cacheKey).toBe(key)
expect(getCacheVersionMock).toHaveBeenCalledWith(
Expand All @@ -287,9 +270,10 @@ test('restore with zstd compressed cache found', async () => {
version: cacheVersion
})
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
expect(downloadCacheFileMock).toHaveBeenCalledWith(
expect(downloadCacheMock).toHaveBeenCalledWith(
signedDownloadUrl,
archivePath
archivePath,
options
)
expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath)
expect(logInfoMock).toHaveBeenCalledWith(`Cache Size: ~60 MB (62915000 B)`)
Expand All @@ -311,6 +295,7 @@ test('restore with cache found for restore key', async () => {
const signedDownloadUrl = 'https://blob-storage.local?signed=true'
const cacheVersion =
'b8b58e9bd7b1e8f83d9f05c7e06ea865ba44a0330e07a14db74ac74386677bed'
const options = {useAzureSdk: true} as DownloadOptions

const getCacheVersionMock = jest.spyOn(cacheUtils, 'getCacheVersion')
getCacheVersionMock.mockReturnValue(cacheVersion)
Expand Down Expand Up @@ -338,17 +323,7 @@ test('restore with cache found for restore key', async () => {
})

const archivePath = path.join(tempPath, CacheFilename.Gzip)
const downloadCacheFileMock = jest.spyOn(
downloadCacheModule,
'downloadCacheFile'
)
downloadCacheFileMock.mockReturnValue(
Promise.resolve({
_response: {
status: 200
}
} as BlobDownloadResponseParsed)
)
const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')

const fileSize = 142
const getArchiveFileSizeInBytesMock = jest
Expand All @@ -358,7 +333,7 @@ test('restore with cache found for restore key', async () => {
const extractTarMock = jest.spyOn(tar, 'extractTar')
const unlinkFileMock = jest.spyOn(cacheUtils, 'unlinkFile')

const cacheKey = await restoreCache(paths, key, restoreKeys)
const cacheKey = await restoreCache(paths, key, restoreKeys, options)

expect(cacheKey).toBe(restoreKeys[0])
expect(getCacheVersionMock).toHaveBeenCalledWith(
Expand All @@ -372,9 +347,10 @@ test('restore with cache found for restore key', async () => {
version: cacheVersion
})
expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
expect(downloadCacheFileMock).toHaveBeenCalledWith(
expect(downloadCacheMock).toHaveBeenCalledWith(
signedDownloadUrl,
archivePath
archivePath,
options
)
expect(getArchiveFileSizeInBytesMock).toHaveBeenCalledWith(archivePath)
expect(logInfoMock).toHaveBeenCalledWith(`Cache Size: ~0 MB (142 B)`)
Expand All @@ -388,14 +364,14 @@ test('restore with cache found for restore key', async () => {
expect(compressionMethodMock).toHaveBeenCalledTimes(1)
})

test('restore with dry run', async () => {
test('restore with lookup only enabled', async () => {
const paths = ['node_modules']
const key = 'node-test'
const options = {lookupOnly: true}
const compressionMethod = CompressionMethod.Gzip
const signedDownloadUrl = 'https://blob-storage.local?signed=true'
const cacheVersion =
'd90f107aaeb22920dba0c637a23c37b5bc497b4dfa3b07fe3f79bf88a273c11b'
const options = {lookupOnly: true, useAzureSdk: true} as DownloadOptions

const getCacheVersionMock = jest.spyOn(cacheUtils, 'getCacheVersion')
getCacheVersionMock.mockReturnValue(cacheVersion)
Expand All @@ -416,10 +392,7 @@ test('restore with dry run', async () => {
)

const createTempDirectoryMock = jest.spyOn(cacheUtils, 'createTempDirectory')
const downloadCacheFileMock = jest.spyOn(
downloadCacheModule,
'downloadCacheFile'
)
const downloadCacheMock = jest.spyOn(cacheHttpClient, 'downloadCache')

const cacheKey = await restoreCache(paths, key, undefined, options)

Expand All @@ -438,5 +411,5 @@ test('restore with dry run', async () => {

// creating a tempDir and downloading the cache are skipped
expect(createTempDirectoryMock).toHaveBeenCalledTimes(0)
expect(downloadCacheFileMock).toHaveBeenCalledTimes(0)
expect(downloadCacheMock).toHaveBeenCalledTimes(0)
})
14 changes: 12 additions & 2 deletions packages/cache/__tests__/saveCache.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,12 @@ test('save with server error should fail', async () => {
compression
)
expect(saveCacheMock).toHaveBeenCalledTimes(1)
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archiveFile, undefined)
expect(saveCacheMock).toHaveBeenCalledWith(
cacheId,
archiveFile,
'',
undefined
)
expect(getCompressionMock).toHaveBeenCalledTimes(1)
})

Expand Down Expand Up @@ -315,7 +320,12 @@ test('save with valid inputs uploads a cache', async () => {
compression
)
expect(saveCacheMock).toHaveBeenCalledTimes(1)
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archiveFile, undefined)
expect(saveCacheMock).toHaveBeenCalledWith(
cacheId,
archiveFile,
'',
undefined
)
expect(getCompressionMock).toHaveBeenCalledTimes(1)
})

Expand Down
Loading

0 comments on commit a10e209

Please sign in to comment.