@uppy/provider-views: fix race condition when adding folders

packages/@uppy/core/src/Uppy.js

@@ -458,12 +458,9 @@ class Uppy {
     const fileName = getFileName(fileType, fileDescriptor)
     const fileExtension = getFileNameAndExtension(fileName).extension
     const isRemote = Boolean(fileDescriptor.isRemote)
-    const fileID = generateFileID({
-      ...fileDescriptor,
-      type: fileType,
-    })
+    const id = generateFileId2(fileDescriptor)
 
-    if (this.checkIfFileAlreadyExists(fileID)) {
+    if (this.checkIfFileAlreadyExists(id)) {
       const error = new RestrictionError(this.i18n('noDuplicates', { fileName }))
       this.#informAndEmit(error, fileDescriptor)
       throw error
@@ -478,7 +475,7 @@ class Uppy {
 
     let newFile = {
       source: fileDescriptor.source || '',
-      id: fileID,
+      id,
       name: fileName,
       extension: fileExtension || '',
       meta: {
@@ -1573,4 +1570,14 @@ class Uppy {
   }
 }
 
+// todo come up with a better name or remove the need for two different generators
+export function generateFileId2 (file) {
+  const fileType = getFileType(file)
+
+  return generateFileID({
+    ...file,
+    type: fileType,
+  })
+}
+
 export default Uppy

packages/@uppy/provider-views/package.json

@@ -22,6 +22,7 @@
   "dependencies": {
     "@uppy/utils": "workspace:^",
     "classnames": "^2.2.6",
+    "p-map": "^5.5.0",
     "preact": "^10.5.13"
   },
   "peerDependencies": {

packages/@uppy/provider-views/src/ProviderView/ProviderView.jsx

@@ -1,4 +1,7 @@
 import { h } from 'preact'
+import pMap from 'p-map'
+
+import { generateFileId2 } from '@uppy/core/src/Uppy.js'
 
 import AuthView from './AuthView.jsx'
 import Header from './Header.jsx'
@@ -58,7 +61,6 @@ export default class ProviderView extends View {
     this.logout = this.logout.bind(this)
     this.handleAuth = this.handleAuth.bind(this)
     this.handleScroll = this.handleScroll.bind(this)
-    this.listAllFiles = this.listAllFiles.bind(this)
     this.donePicking = this.donePicking.bind(this)
 
     // Visual
@@ -100,29 +102,31 @@ export default class ProviderView extends View {
    * @param  {string} id Folder id
    * @returns {Promise}   Folders/files in folder
    */
-  getFolder (id, name) {
-    return this.sharedHandler.loaderWrapper(
-      this.provider.list(id),
-      (res) => {
-        const folders = []
-        const files = []
-        let updatedDirectories
-
-        const state = this.plugin.getPluginState()
-        const index = state.directories.findIndex((dir) => id === dir.id)
-
-        if (index !== -1) {
-          updatedDirectories = state.directories.slice(0, index + 1)
-        } else {
-          updatedDirectories = state.directories.concat([{ id, title: name }])
-        }
+  async getFolder (id, name) {
+    this.setLoading(true)
+    try {
+      const res = await this.provider.list(id)
+      const folders = []
+      const files = []
+      let updatedDirectories
+
+      const state = this.plugin.getPluginState()
+      const index = state.directories.findIndex((dir) => id === dir.id)
+
+      if (index !== -1) {
+        updatedDirectories = state.directories.slice(0, index + 1)
+      } else {
+        updatedDirectories = state.directories.concat([{ id, title: name }])
+      }
 
-        this.username = res.username || this.username
-        this.#updateFilesAndFolders(res, files, folders)
-        this.plugin.setPluginState({ directories: updatedDirectories, filterInput: '' })
-      },
-      this.handleError,
-    )
+      this.username = res.username || this.username
+      this.#updateFilesAndFolders(res, files, folders)
+      this.plugin.setPluginState({ directories: updatedDirectories, filterInput: '' })
+    } catch (err) {
+      this.handleError(err)
+    } finally {
+      this.setLoading(false)
+    }
   }
 
   /**
@@ -167,74 +171,6 @@ export default class ProviderView extends View {
     this.plugin.setPluginState({ ...state, filterInput: e ? e.target.value : '' })
   }
 
-  /**
-   * Adds all files found inside of specified folder.
-   *
-   * Uses separated state while folder contents are being fetched and
-   * mantains list of selected folders, which are separated from files.
-   */
-  addFolder (folder) {
-    const folderId = this.providerFileToId(folder)
-    const folders = { ...this.plugin.getPluginState().selectedFolders }
-
-    if (folderId in folders && folders[folderId].loading) {
-      return
-    }
-
-    folders[folderId] = { loading: true, files: [] }
-
-    this.plugin.setPluginState({ selectedFolders: { ...folders } })
-
-    // eslint-disable-next-line consistent-return
-    return this.listAllFiles(folder.requestPath).then((files) => {
-      let count = 0
-
-      // If the same folder is added again, we don't want to send
-      // X amount of duplicate file notifications, we want to say
-      // the folder was already added. This checks if all files are duplicate,
-      // if that's the case, we don't add the files.
-      files.forEach(file => {
-        const id = this.providerFileToId(file)
-        if (!this.plugin.uppy.checkIfFileAlreadyExists(id)) {
-          count++
-        }
-      })
-
-      if (count > 0) {
-        files.forEach((file) => this.addFile(file))
-      }
-
-      const ids = files.map(this.providerFileToId)
-
-      folders[folderId] = {
-        loading: false,
-        files: ids,
-      }
-      this.plugin.setPluginState({ selectedFolders: folders, filterInput: '' })
-
-      let message
-
-      if (count === 0) {
-        message = this.plugin.uppy.i18n('folderAlreadyAdded', {
-          folder: folder.name,
-        })
-      } else if (files.length) {
-        message = this.plugin.uppy.i18n('folderAdded', {
-          smart_count: count, folder: folder.name,
-        })
-      } else {
-        message = this.plugin.uppy.i18n('emptyFolderAdded')
-      }
-
-      this.plugin.uppy.info(message)
-    }).catch((e) => {
-      const selectedFolders = { ...this.plugin.getPluginState().selectedFolders }
-      delete selectedFolders[folderId]
-      this.plugin.setPluginState({ selectedFolders })
-      this.handleError(e)
-    })
-  }
-
   async handleAuth () {
     await this.provider.ensurePreAuth()
 
@@ -296,35 +232,89 @@ export default class ProviderView extends View {
     }
   }
 
-  async listAllFiles (path, files = null) {
-    files = files || [] // eslint-disable-line no-param-reassign
-    const res = await this.provider.list(path)
-    res.items.forEach((item) => {
-      if (!item.isFolder) {
-        files.push(item)
-      } else {
-        this.addFolder(item)
-      }
-    })
-    const moreFiles = res.nextPagePath
-    if (moreFiles) {
-      return this.listAllFiles(moreFiles, files)
+  async recursivelyListAllFiles (path, files = []) {
+    let curPath = path
+
+    // need to repeat the list call until there are no more pages
+    while (curPath) {
+      const res = await this.provider.list(curPath)
+
+      // limit concurrency (because what if we have a folder with 1000 folders inside!)
+      // todo this might still lead to a memory run-off
+      await pMap(res.items, async (item) => {
+        if (item.isFolder) {
+          // recursively call self for folder
+          await this.recursivelyListAllFiles(item.requestPath, files)
+        } else {
+          files.push(item)
+        }
+      }, { concurrency: 10 })
+
+      curPath = res.nextPagePath
     }
+
     return files
   }
 
-  donePicking () {
-    const { currentSelection } = this.plugin.getPluginState()
-    const promises = currentSelection.map((file) => {
-      if (file.isFolder) {
-        return this.addFolder(file)
+  async donePicking () {
+    this.setLoading(true)
+    try {
+      const { currentSelection } = this.plugin.getPluginState()
+
+      const allFiles = []
+      const foldersAdded = []
+
+      // eslint-disable-next-line no-unreachable-loop
+      for (const file of currentSelection) {
+        if (file.isFolder) {
+          const folder = file
+          const filesInFolder = await this.recursivelyListAllFiles(folder.requestPath)
+
+          // If the same folder is added again, we don't want to send
+          // X amount of duplicate file notifications, we want to say
+          // the folder was already added. This checks if all files are duplicate,
+          // if that's the case, we don't add the files.
+          const newFilesInFolder = filesInFolder.filter((fileInFolder) => {
+            const tagFile = this.getTagFile(fileInFolder)
+            const id = generateFileId2(tagFile)
+            return !this.plugin.uppy.checkIfFileAlreadyExists(id)
+          })
+
+          allFiles.push(...newFilesInFolder)
+
+          foldersAdded.push({ numFiles: filesInFolder.length, numNewFiles: newFilesInFolder.length, name: folder.name })
+        } else {
+          allFiles.push(file)
+        }
+      }
+
+      allFiles.forEach((file) => this.addFile(file))
+
+      this.plugin.setPluginState({ filterInput: '' })
+
+      for (const { name, numFiles, numNewFiles } of foldersAdded) {
+        let message
+        if (numFiles === 0) {
+          message = this.plugin.uppy.i18n('emptyFolderAdded')
+        } else if (numNewFiles === 0) {
+          message = this.plugin.uppy.i18n('folderAlreadyAdded', {
+            folder: name,
+          })
+        } else {
+          message = this.plugin.uppy.i18n('folderAdded', {
+            smart_count: numNewFiles, folder: name,
+          })
+        }
+
+        this.plugin.uppy.info(message)
       }
-      return this.addFile(file)
-    })
 
-    this.sharedHandler.loaderWrapper(Promise.all(promises), () => {
       this.clearSelection()
-    }, () => {})
+    } catch (err) {
+      this.handleError(err)
+    } finally {
+      this.setLoading(false)
+    }
   }
 
   render (state, viewOptions = {}) {
@@ -336,7 +326,7 @@ export default class ProviderView extends View {
 
     const targetViewOptions = { ...this.opts, ...viewOptions }
     const { files, folders, filterInput, loading, currentSelection } = this.plugin.getPluginState()
-    const { isChecked, toggleCheckbox, recordShiftKeyPress, filterItems } = this.sharedHandler
+    const { isChecked, toggleCheckbox, recordShiftKeyPress, filterItems } = this
     const hasInput = filterInput !== ''
     const headerProps = {
       showBreadcrumbs: targetViewOptions.showBreadcrumbs,
@@ -359,11 +349,9 @@ export default class ProviderView extends View {
       username: this.username,
       getNextFolder: this.getNextFolder,
       getFolder: this.getFolder,
-      filterItems: this.sharedHandler.filterItems,
       filterQuery: this.filterQuery,
       logout: this.logout,
       handleScroll: this.handleScroll,
-      listAllFiles: this.listAllFiles,
       done: this.donePicking,
       cancel: this.cancelPicking,
       headerComponent: Header(headerProps),

packages/@uppy/provider-views/src/SearchProviderView/SearchProviderView.jsx

@@ -79,20 +79,22 @@ export default class SearchProviderView extends View {
     })
   }
 
-  search (query) {
+  async search (query) {
     const { searchTerm } = this.plugin.getPluginState()
     if (query && query === searchTerm) {
       // no need to search again as this is the same as the previous search
-      return undefined
+      return
     }
 
-    return this.sharedHandler.loaderWrapper(
-      this.provider.search(query),
-      (res) => {
-        this.#updateFilesAndInputMode(res, [])
-      },
-      this.handleError,
-    )
+    this.setLoading(true)
+    try {
+      const res = await this.provider.search(query)
+      this.#updateFilesAndInputMode(res, [])
+    } catch (err) {
+      this.handleError(err)
+    } finally {
+      this.setLoading(false)
+    }
   }
 
   triggerSearchInput () {
@@ -120,11 +122,8 @@ export default class SearchProviderView extends View {
 
   donePicking () {
     const { currentSelection } = this.plugin.getPluginState()
-    const promises = currentSelection.map((file) => this.addFile(file))
-
-    this.sharedHandler.loaderWrapper(Promise.all(promises), () => {
-      this.clearSelection()
-    }, () => {})
+    currentSelection.forEach((file) => this.addFile(file))
+    this.clearSelection()
   }
 
   render (state, viewOptions = {}) {
@@ -136,7 +135,7 @@ export default class SearchProviderView extends View {
 
     const targetViewOptions = { ...this.opts, ...viewOptions }
     const { files, folders, filterInput, loading, currentSelection } = this.plugin.getPluginState()
-    const { isChecked, toggleCheckbox, filterItems } = this.sharedHandler
+    const { isChecked, toggleCheckbox, filterItems } = this
     const hasInput = filterInput !== ''
 
     const browserProps = {