@uppy/aws-s3-multipart: fix Golden Retriever integration (#4526)

transloadit · Jun 29, 2023 · d1697b0 · d1697b0
1 parent 35b4b0e
commit d1697b0
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 7 deletions.
diff --git a/packages/@uppy/aws-s3-multipart/src/MultipartUploader.js b/packages/@uppy/aws-s3-multipart/src/MultipartUploader.js
@@ -60,9 +60,12 @@ class MultipartUploader {
   /** @type {() => void} */
   #onSuccess
 
-  /** @type {typeof import('../types/index').AwsS3MultipartOptions["shouldUseMultipart"]} */
+  /** @type {import('../types/index').AwsS3MultipartOptions["shouldUseMultipart"]} */
   #shouldUseMultipart
 
+  /** @type {boolean} */
+  #isRestoring
+
   #onReject = (err) => (err?.cause === pausingUploadReason ? null : this.#onError(err))
 
   #maxMultipartParts = 10_000
@@ -83,6 +86,11 @@ class MultipartUploader {
     this.#onError = this.options.onError
     this.#shouldUseMultipart = this.options.shouldUseMultipart
 
+    // When we are restoring an upload, we already have an uploadId. Otherwise
+    // we need to call `createMultipartUpload` to get an `uploadId`.
+    // Non-multipart uploads are not restorable.
+    this.#isRestoring = 'uploadId' in options
+
     this.#initChunks()
   }
 
@@ -108,12 +116,12 @@ class MultipartUploader {
       }
       this.#chunks = Array(arraySize)
 
-      for (let i = 0, j = 0; i < fileSize; i += chunkSize, j++) {
-        const end = Math.min(fileSize, i + chunkSize)
+      for (let offset = 0, j = 0; offset < fileSize; offset += chunkSize, j++) {
+        const end = Math.min(fileSize, offset + chunkSize)
 
         // Defer data fetching/slicing until we actually need the data, because it's slow if we have a lot of files
         const getData = () => {
-          const i2 = i
+          const i2 = offset
           return this.#data.slice(i2, end)
         }
 
@@ -123,6 +131,14 @@ class MultipartUploader {
           onComplete: this.#onPartComplete(j),
           shouldUseMultipart,
         }
+        if (this.#isRestoring) {
+          const size = offset + chunkSize > fileSize ? fileSize - offset : chunkSize
+          // setAsUploaded is called by listPart, to keep up-to-date the
+          // quantity of data that is left to actually upload.
+          this.#chunks[j].setAsUploaded = () => {
+            this.#chunkState[j].uploaded = size
+          }
+        }
       }
     } else {
       this.#chunks = [{
@@ -181,6 +197,9 @@ class MultipartUploader {
       if (!this.#abortController.signal.aborted) this.#abortController.abort(pausingUploadReason)
       this.#abortController = new AbortController()
       this.#resumeUpload()
+    } else if (this.#isRestoring) {
+      this.options.companionComm.restoreUploadFile(this.#file, { uploadId: this.options.uploadId, key: this.options.key })
+      this.#resumeUpload()
     } else {
       this.#createUpload()
     }

diff --git a/packages/@uppy/aws-s3-multipart/src/index.js b/packages/@uppy/aws-s3-multipart/src/index.js
@@ -264,6 +264,10 @@ class HTTPCommunicationQueue {
     }
   }
 
+  restoreUploadFile (file, uploadIdAndKey) {
+    this.#cache.set(file.data, uploadIdAndKey)
+  }
+
   async resumeUploadFile (file, chunks, signal) {
     throwIfAborted(signal)
     if (chunks.length === 1 && !chunks[0].shouldUseMultipart) {
@@ -278,9 +282,11 @@ class HTTPCommunicationQueue {
         .map((chunk, i) => {
           const partNumber = i + 1
           const alreadyUploadedInfo = alreadyUploadedParts.find(({ PartNumber }) => PartNumber === partNumber)
-          return alreadyUploadedInfo == null
-            ? this.uploadChunk(file, partNumber, chunk, signal)
-            : { PartNumber: partNumber, ETag: alreadyUploadedInfo.ETag }
+          if (alreadyUploadedInfo == null) {
+            return this.uploadChunk(file, partNumber, chunk, signal)
+          }
+          chunk.setAsUploaded?.()
+          return { PartNumber: partNumber, ETag: alreadyUploadedInfo.ETag }
         }),
     )
     throwIfAborted(signal)

diff --git a/packages/@uppy/aws-s3-multipart/types/chunk.d.ts b/packages/@uppy/aws-s3-multipart/types/chunk.d.ts
@@ -3,4 +3,5 @@ export interface Chunk {
     onProgress: (ev: ProgressEvent) => void
     onComplete: (etag: string) => void
     shouldUseMultipart: boolean
+    setAsUploaded?: () => void
 }