From 177f905c33cc47af25b059bab8d3209780d7bbff Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Tue, 26 Mar 2024 17:21:41 +0000 Subject: [PATCH 1/3] feat: add ability to create a File object from URL --- src/file.ts | 34 +++++++++++++++++++++++++++++++++- test/file.ts | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/file.ts b/src/file.ts index 1df546b93..30b2df6c8 100644 --- a/src/file.ts +++ b/src/file.ts @@ -324,7 +324,7 @@ export const STORAGE_POST_POLICY_BASE_URL = 'https://storage.googleapis.com'; /** * @private */ -const GS_URL_REGEXP = /^gs:\/\/([a-z0-9_.-]+)\/(.+)$/; +const GS_URL_REGEXP = /^gs:\/\/([a-z0-9_.-]+)\/(.+)$/g; export interface FileOptions { crc32cGenerator?: CRC32CValidatorGenerator; @@ -479,6 +479,9 @@ export class RequestError extends Error { } const SEVEN_DAYS = 7 * 24 * 60 * 60; +const GS_URL_REGEX = /(gs):\/\/([a-z0-9_.-]+)\/(.+)/g; +const HTTPS_PUBLIC_URL_REGEX = + /(https):\/\/(storage\.googleapis\.com)\/([a-z0-9_.-]+)\/(.+)/g; export enum FileExceptionMessages { EXPIRATION_TIME_NA = 'An expiration time is not available.', @@ -2358,6 +2361,35 @@ class File extends ServiceObject { return this; } + /** + * Gets a reference to a Cloud Storage {@link File} file from the provided URL in string format. + * @param {string} publicUrlOrGsUrl the URL as a string. Must be of the format gs://bucket/file + * or https://storage.googleapis.com/bucket/file. + * @param {Storage} storageInstance an instance of a Storage object. + * @param {FileOptions} [options] Configuration options + * @returns {File} + */ + static from( + publicUrlOrGsUrl: string, + storageInstance: Storage, + options?: FileOptions + ): File { + const gsMatches = [...publicUrlOrGsUrl.matchAll(GS_URL_REGEX)]; + const httpsMatches = [...publicUrlOrGsUrl.matchAll(HTTPS_PUBLIC_URL_REGEX)]; + + if (gsMatches.length > 0) { + const bucket = new Bucket(storageInstance, gsMatches[0][1]); + return new File(bucket, gsMatches[0][2], options); + } else if (httpsMatches.length > 0) { + const bucket = new Bucket(storageInstance, httpsMatches[0][2]); + return new File(bucket, httpsMatches[0][3], options); + } else { + throw new Error( + 'URL string must be of format gs://bucket/file or https://storage.googleapis.com/bucket/file' + ); + } + } + get(options?: GetFileOptions): Promise>; get(callback: InstanceResponseCallback): void; get(options: GetFileOptions, callback: InstanceResponseCallback): void; diff --git a/test/file.ts b/test/file.ts index 73de841dd..d0cc0c2ba 100644 --- a/test/file.ts +++ b/test/file.ts @@ -5168,4 +5168,49 @@ describe('File', () => { file.setUserProject(userProject); }); }); + + describe('from', () => { + it('should create a File object from a gs:// formatted URL', () => { + const gsUrl = 'gs://mybucket/myfile'; + const result = File.from(gsUrl, STORAGE); + + assert(result); + assert(result.bucket.name, 'mybucket'); + assert(result.name, 'myfile'); + }); + + it('should create a File object from a gs:// formatted URL including a folder', () => { + const gsUrl = 'gs://mybucket/myfolder/myfile'; + const result = File.from(gsUrl, STORAGE); + + assert(result); + assert(result.bucket.name, 'mybucket'); + assert(result.name, 'myfolder/myfile'); + }); + + it('should create a File object from a https:// formatted URL', () => { + const httpsUrl = 'https://storage.googleapis.com/mybucket/myfile'; + const result = File.from(httpsUrl, STORAGE); + + assert(result); + assert(result.bucket.name, 'mybucket'); + assert(result.name, 'myfile'); + }); + + it('should create a File object from a https:// formatted URL including a folder', () => { + const httpsUrl = + 'https://storage.googleapis.com/mybucket/myfolder/myfile'; + const result = File.from(httpsUrl, STORAGE); + + assert(result); + assert(result.bucket.name, 'mybucket'); + assert(result.name, 'myfolder/myfile'); + }); + + it('should throw an error when invoked with an incorrectly formatted URL', () => { + const invalidUrl = 'https://storage.com/mybucket/myfile'; + + assert.throws(() => File.from(invalidUrl, STORAGE)); + }); + }); }); From 412f0e5f31e763fa7fa9caf4d46bfb4c8655b9ad Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Tue, 26 Mar 2024 17:24:32 +0000 Subject: [PATCH 2/3] fixes --- src/file.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/file.ts b/src/file.ts index 30b2df6c8..0a5acf2bc 100644 --- a/src/file.ts +++ b/src/file.ts @@ -324,7 +324,7 @@ export const STORAGE_POST_POLICY_BASE_URL = 'https://storage.googleapis.com'; /** * @private */ -const GS_URL_REGEXP = /^gs:\/\/([a-z0-9_.-]+)\/(.+)$/g; +const GS_URL_REGEXP = /^gs:\/\/([a-z0-9_.-]+)\/(.+)$/; export interface FileOptions { crc32cGenerator?: CRC32CValidatorGenerator; @@ -479,7 +479,7 @@ export class RequestError extends Error { } const SEVEN_DAYS = 7 * 24 * 60 * 60; -const GS_URL_REGEX = /(gs):\/\/([a-z0-9_.-]+)\/(.+)/g; +const GS_UTIL_URL_REGEX = /(gs):\/\/([a-z0-9_.-]+)\/(.+)/g; const HTTPS_PUBLIC_URL_REGEX = /(https):\/\/(storage\.googleapis\.com)\/([a-z0-9_.-]+)\/(.+)/g; @@ -2374,7 +2374,7 @@ class File extends ServiceObject { storageInstance: Storage, options?: FileOptions ): File { - const gsMatches = [...publicUrlOrGsUrl.matchAll(GS_URL_REGEX)]; + const gsMatches = [...publicUrlOrGsUrl.matchAll(GS_UTIL_URL_REGEX)]; const httpsMatches = [...publicUrlOrGsUrl.matchAll(HTTPS_PUBLIC_URL_REGEX)]; if (gsMatches.length > 0) { From c5b395e0173261351927cbb6f56fa04448dffde8 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Tue, 26 Mar 2024 17:30:11 +0000 Subject: [PATCH 3/3] linter fixes --- src/nodejs-common/service.ts | 5 ++--- src/resumable-upload.ts | 5 ++--- src/transfer-manager.ts | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/nodejs-common/service.ts b/src/nodejs-common/service.ts index 0a3111667..8156cd176 100644 --- a/src/nodejs-common/service.ts +++ b/src/nodejs-common/service.ts @@ -271,9 +271,8 @@ export class Service { }; if (reqOpts[GCCL_GCS_CMD_KEY]) { - reqOpts.headers[ - 'x-goog-api-client' - ] += ` gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`; + reqOpts.headers['x-goog-api-client'] += + ` gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`; } if (reqOpts.shouldReturnStream) { diff --git a/src/resumable-upload.ts b/src/resumable-upload.ts index 049e20c43..5a1f71ccc 100644 --- a/src/resumable-upload.ts +++ b/src/resumable-upload.ts @@ -933,9 +933,8 @@ export class Upload extends Writable { // `Content-Length` for multiple chunk uploads is the size of the chunk, // not the overall object headers['Content-Length'] = bytesToUpload; - headers[ - 'Content-Range' - ] = `bytes ${this.offset}-${endingByte}/${totalObjectSize}`; + headers['Content-Range'] = + `bytes ${this.offset}-${endingByte}/${totalObjectSize}`; } else { headers['Content-Range'] = `bytes ${this.offset}-*/${this.contentLength}`; } diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 37230389f..8b361b688 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -223,9 +223,8 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { // Prepend command feature to value, if not already there if (!value.includes(GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED)) { - headers[ - key - ] = `${value} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`; + headers[key] = + `${value} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`; } } else if (key.toLocaleLowerCase().trim() === 'user-agent') { userAgentFound = true;