From 64aa0221f1b2b3ea64765446264bc796f879a870 Mon Sep 17 00:00:00 2001 From: Minh Nguyen Cong Date: Tue, 30 Jan 2024 15:00:36 +0100 Subject: [PATCH 1/5] feat: Support overwrite/skip folder download --- src/commands/folders/download.js | 113 +++++++++++++----- test/commands/folders.test.js | 189 +++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+), 29 deletions(-) diff --git a/src/commands/folders/download.js b/src/commands/folders/download.js index b3fc609b..3f25edf5 100644 --- a/src/commands/folders/download.js +++ b/src/commands/folders/download.js @@ -22,13 +22,15 @@ const utils = require('../../util'); * @private */ function saveFileToDisk(folderPath, file, stream) { - let output; try { output = fs.createWriteStream(path.join(folderPath, file.path)); stream.pipe(output); } catch (ex) { - throw new BoxCLIError(`Error downloading file ${file.id} to ${file.path}`, ex); + throw new BoxCLIError( + `Error downloading file ${file.id} to ${file.path}`, + ex + ); } /* eslint-disable promise/avoid-new */ @@ -44,11 +46,17 @@ class FoldersDownloadCommand extends BoxCommand { async run() { const { flags, args } = this.parse(FoldersDownloadCommand); - this.maxDepth = flags.hasOwnProperty('depth') && flags.depth >= 0 ? flags.depth : Number.POSITIVE_INFINITY; + this.outputPath = null; + this.maxRecurDepth = + flags.hasOwnProperty('depth') && flags.depth >= 0 + ? flags.depth + : Number.POSITIVE_INFINITY; + this.overwrite = flags.overwrite; + this.maxDepth = flags['max-depth']; - let outputPath; let id = args.id; let outputFinalized = Promise.resolve(); + let rootItemPath = null; let destinationPath; if (flags.destination) { @@ -72,26 +80,31 @@ class FoldersDownloadCommand extends BoxCommand { let spinner = ora('Starting download').start(); if (flags.zip) { - let fileName = `folders-download-${id}-${dateTime.format(new Date(), 'YYYY-MM-DDTHH_mm_ss_SSS')}.zip`; - outputPath = path.join(destinationPath, fileName); - outputFinalized = this._setupZip(outputPath); + let fileName = `folders-download-${id}-${dateTime.format( + new Date(), + 'YYYY-MM-DDTHH_mm_ss_SSS' + )}.zip`; + this.outputPath = path.join(destinationPath, fileName); + outputFinalized = this._setupZip(this.outputPath); } try { + this.outputPath = destinationPath; for await (let item of this._getItems(id, '')) { if (item.type === 'folder' && !this.zip) { - // Set output path to the top-level folder, which is the first item in the generator - outputPath = outputPath || path.join(destinationPath, item.path); + rootItemPath = rootItemPath || item.path; spinner.text = `Creating folder ${item.id} at ${item.path}`; try { await mkdirp(path.join(destinationPath, item.path)); } catch (ex) { - throw new BoxCLIError(`Folder ${item.path} could not be created`, ex); + throw new BoxCLIError( + `Folder ${item.path} could not be created`, + ex + ); } } else if (item.type === 'file') { - spinner.text = `Downloading file ${item.id} to ${item.path}`; let stream = await this.client.files.getReadStream(item.id); @@ -112,7 +125,9 @@ class FoldersDownloadCommand extends BoxCommand { this.zip.finalize(); } await outputFinalized; - spinner.succeed(`Downloaded folder ${id} to ${outputPath}`); + spinner.succeed( + `Downloaded folder ${id} to ${path.join(this.outputPath, rootItemPath)}` + ); } /** @@ -124,7 +139,6 @@ class FoldersDownloadCommand extends BoxCommand { * @private */ async* _getItems(folderId, folderPath) { - let folder = await this.client.folders.get(folderId); folderPath = path.join(folderPath, folder.name); @@ -137,19 +151,46 @@ class FoldersDownloadCommand extends BoxCommand { let folderItems = folder.item_collection.entries; if (folder.item_collection.total_count > folderItems.length) { - let iterator = await this.client.folders.getItems(folderId, { usemarker: true, fields: 'type,id,name' }); + let iterator = await this.client.folders.getItems(folderId, { + usemarker: true, + fields: 'type,id,name', + }); folderItems = { [Symbol.asyncIterator]: () => iterator }; } for await (let item of folderItems) { - if (item.type === 'folder' && folderPath.split(path.sep).length <= this.maxDepth) { - yield* this._getItems(item.id, folderPath); + if ( + item.type === 'folder' && + folderPath.split(path.sep).length <= this.maxRecurDepth + ) { + // We only recurse this folder by one of the following conditions: + // 1. The overwrite flag is true. We will download all files and folders. + // 2. The maxDepth flag is set to 'max'. We will go through all folders at any depth. + // 3. The folder does not exist. We will go through all folders at any depth and download all files. + /* eslint-disable no-sync */ + if ( + this.overwrite || + this.maxDepth === 'max' || + !fs.existsSync(path.join(this.outputPath, folderPath, item.name)) + ) { + /* eslint-enable no-sync */ + yield* this._getItems(item.id, folderPath); + } } else if (item.type === 'file') { - yield { - type: 'file', - id: item.id, - name: item.name, - path: path.join(folderPath, item.name), - }; + // We only download file if overwrite is true or the file does not exist. + // Skip downloading if overwrite is false and the file exists. + /* eslint-disable no-sync */ + if ( + this.overwrite || + !fs.existsSync(path.join(this.outputPath, folderPath, item.name)) + ) { + /* eslint-enable no-sync */ + yield { + type: 'file', + id: item.id, + name: item.name, + path: path.join(folderPath, item.name), + }; + } } } } @@ -163,20 +204,22 @@ class FoldersDownloadCommand extends BoxCommand { * @private */ _setupZip(destinationPath) { - // Set up archive stream this.zip = archiver('zip', { - zlib: { level: 9 } // Use the best available compression + zlib: { level: 9 }, // Use the best available compression }); let output; try { output = fs.createWriteStream(destinationPath); } catch (ex) { - throw new BoxCLIError(`Could not write to destination path ${destinationPath}`, ex); + throw new BoxCLIError( + `Could not write to destination path ${destinationPath}`, + ex + ); } - this.zip.on('error', err => { + this.zip.on('error', (err) => { throw new BoxCLIError('Error writing to zip file', err); }); @@ -199,7 +242,7 @@ FoldersDownloadCommand.flags = { ...BoxCommand.flags, destination: flags.string({ description: 'The destination folder to download the Box folder into', - parse: utils.parsePath + parse: utils.parsePath, }), zip: flags.boolean({ description: 'Download the folder into a single .zip archive', @@ -209,7 +252,19 @@ FoldersDownloadCommand.flags = { 'Number of levels deep to recurse when downloading the folder tree', }), 'create-path': flags.boolean({ - description: 'Recursively creates a path to a directory if it does not exist', + description: + 'Recursively creates a path to a directory if it does not exist', + allowNo: true, + default: true, + }), + 'max-depth': flags.enum({ + description: + 'Maximum depth to verify if files and folders conflict, only used with --no-overwrite', + options: ['root', 'max'], + default: 'max', + }), + overwrite: flags.boolean({ + description: '[default: true] Overwrite the folder if it already exists.', allowNo: true, default: true, }), @@ -221,7 +276,7 @@ FoldersDownloadCommand.args = [ required: true, hidden: false, description: 'ID of the folder to download', - } + }, ]; module.exports = FoldersDownloadCommand; diff --git a/test/commands/folders.test.js b/test/commands/folders.test.js index ab77190d..0f039fc9 100644 --- a/test/commands/folders.test.js +++ b/test/commands/folders.test.js @@ -1773,6 +1773,195 @@ describe('Folders', () => { assert.deepEqual(actualContents, expectedContents); assert.equal(ctx.stdout, ''); }); + + test + .nock(TEST_API_ROOT, api => api + .get(`/2.0/folders/${folderID}`) + .reply(200, getFolderFixture) + .get('/2.0/folders/22222') + .reply(200, getSubfolderFixture) + .get('/2.0/files/77777/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/77777` }) + .get('/2.0/files/44444/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/44444` }) + .get('/2.0/files/55555/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/55555` }) + ) + .nock(TEST_DOWNLOAD_ROOT, api => api + .get('/44444') + .reply(200, expectedContents['file 1.txt']) + .get('/55555') + .reply(200, expectedContents['file 2.txt']) + .get('/77777') + .reply(200, expectedContents.subfolder['subfolder file 1.txt']) + ) + .do(() => { + /* eslint-disable no-sync */ + const folderPath = path.join(destination, folderName); + if (fs.existsSync(destination)) { + fs.removeSync(destination); + } + fs.mkdirSync(destination); + fs.mkdirSync(folderPath); + fs.writeFileSync(path.join(folderPath, 'file 1.txt'), 'test123'); + /* eslint-enable no-sync */ + }) + .stdout() + .stderr() + .command([ + 'folders:download', + folderID, + `--destination=${destination}`, + '--no-color', + '--token=test', + ]) + .it('should download and overwrite existing folder', async(ctx) => { + let folderPath = path.join(destination, folderName); + let actualContents = getDirectoryContents(folderPath); + await fs.remove(destination); + + assert.deepEqual(actualContents, expectedContents); + assert.equal(ctx.stdout, ''); + }); + + test + .nock(TEST_API_ROOT, api => api + .get(`/2.0/folders/${folderID}`) + .reply(200, getFolderFixture) + .get('/2.0/folders/22222') + .reply(200, getSubfolderFixture) + .get('/2.0/files/77777/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/77777` }) + .get('/2.0/files/55555/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/55555` }) + ) + .nock(TEST_DOWNLOAD_ROOT, api => api + .get('/55555') + .reply(200, expectedContents['file 2.txt']) + .get('/77777') + .reply(200, expectedContents.subfolder['subfolder file 1.txt']) + ) + .do(() => { + /* eslint-disable no-sync */ + const folderPath = path.join(destination, folderName); + if (fs.existsSync(destination)) { + fs.removeSync(destination); + } + fs.mkdirSync(destination); + fs.mkdirSync(folderPath); + fs.writeFileSync(path.join(folderPath, 'file 1.txt'), 'test123'); + /* eslint-enable no-sync */ + }) + .stdout() + .stderr() + .command([ + 'folders:download', + folderID, + `--destination=${destination}`, + '--no-color', + '--no-overwrite', + '--token=test', + ]) + .it('should not overwrite existing folder when --no-overwrite flag is passed', async(ctx) => { + let folderPath = path.join(destination, folderName); + let actualContents = getDirectoryContents(folderPath); + await fs.remove(destination); + assert.equal(actualContents['file 1.txt'], 'test123'); + assert.equal(ctx.stdout, ''); + }); + + test + .nock(TEST_API_ROOT, api => api + .get(`/2.0/folders/${folderID}`) + .reply(200, getFolderFixture) + .get('/2.0/files/55555/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/55555` }) + ) + .nock(TEST_DOWNLOAD_ROOT, api => api + .get('/55555') + .reply(200, expectedContents['file 2.txt']) + ) + .do(() => { + /* eslint-disable no-sync */ + const folderPath = path.join(destination, folderName); + if (fs.existsSync(destination)) { + fs.removeSync(destination); + } + fs.mkdirSync(destination); + fs.mkdirSync(folderPath); + fs.mkdirSync(path.join(folderPath, 'subfolder')); + fs.writeFileSync(path.join(folderPath, 'file 1.txt'), 'test123'); + /* eslint-enable no-sync */ + }) + .stdout() + .stderr() + .command([ + 'folders:download', + folderID, + `--destination=${destination}`, + '--no-color', + '--no-overwrite', + '--max-depth=root', + '--token=test', + ]) + .it('should not overwrite existing file and folder in root folder when --no-overwrite and --max-depth=root flag is passed', async(ctx) => { + let folderPath = path.join(destination, folderName); + let actualContents = getDirectoryContents(folderPath); + await fs.remove(destination); + assert.equal(actualContents['file 1.txt'], 'test123'); + assert.equal(Object.keys(actualContents.subfolder).length, 0); + assert.equal(ctx.stdout, ''); + }); + + test + .nock(TEST_API_ROOT, api => api + .get(`/2.0/folders/${folderID}`) + .reply(200, getFolderFixture) + .get('/2.0/folders/22222') + .reply(200, getSubfolderFixture) + .get('/2.0/files/77777/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/77777` }) + .get('/2.0/files/55555/content') + .reply(302, '', { Location: `${TEST_DOWNLOAD_ROOT}/55555` }) + ) + .nock(TEST_DOWNLOAD_ROOT, api => api + .get('/55555') + .reply(200, expectedContents['file 2.txt']) + .get('/77777') + .reply(200, expectedContents.subfolder['subfolder file 1.txt']) + ) + .do(() => { + /* eslint-disable no-sync */ + const folderPath = path.join(destination, folderName); + if (fs.existsSync(destination)) { + fs.removeSync(destination); + } + fs.mkdirSync(destination); + fs.mkdirSync(folderPath); + fs.mkdirSync(path.join(folderPath, 'subfolder')); + fs.writeFileSync(path.join(folderPath, 'file 1.txt'), 'test123'); + /* eslint-enable no-sync */ + }) + .stdout() + .stderr() + .command([ + 'folders:download', + folderID, + `--destination=${destination}`, + '--no-color', + '--no-overwrite', + '--max-depth=max', + '--token=test', + ]) + .it('should not overwrite existing file and folder in folder recursively when --no-overwrite and --max-depth=max flag is passed', async(ctx) => { + let folderPath = path.join(destination, folderName); + let actualContents = getDirectoryContents(folderPath); + await fs.remove(destination); + assert.equal(actualContents['file 1.txt'], 'test123'); + assert.equal(Object.keys(actualContents.subfolder).length, 1); + assert.equal(ctx.stdout, ''); + }); + }); describe('folders:locks', () => { From 77bf65bf2984f4179b13d92982b71aef4313f935 Mon Sep 17 00:00:00 2001 From: Minh Nguyen Cong Date: Tue, 30 Jan 2024 15:06:49 +0100 Subject: [PATCH 2/5] Update docs --- src/commands/folders/download.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/folders/download.js b/src/commands/folders/download.js index 3f25edf5..1d69a54d 100644 --- a/src/commands/folders/download.js +++ b/src/commands/folders/download.js @@ -259,7 +259,7 @@ FoldersDownloadCommand.flags = { }), 'max-depth': flags.enum({ description: - 'Maximum depth to verify if files and folders conflict, only used with --no-overwrite', + 'Maximum depth to verify if files and folders are already exists, only used with --no-overwrite', options: ['root', 'max'], default: 'max', }), From 61a65ddceb2c3f69bb9f3ee942c0bbeb1c5abe76 Mon Sep 17 00:00:00 2001 From: Minh Nguyen Cong Date: Mon, 5 Feb 2024 17:09:29 +0100 Subject: [PATCH 3/5] Update logic --- src/commands/folders/download.js | 64 ++++++++++++++++++-------------- test/commands/folders.test.js | 6 +-- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/commands/folders/download.js b/src/commands/folders/download.js index 1d69a54d..4a0d8ff9 100644 --- a/src/commands/folders/download.js +++ b/src/commands/folders/download.js @@ -47,12 +47,11 @@ class FoldersDownloadCommand extends BoxCommand { const { flags, args } = this.parse(FoldersDownloadCommand); this.outputPath = null; - this.maxRecurDepth = + this.maxDepth = flags.hasOwnProperty('depth') && flags.depth >= 0 ? flags.depth : Number.POSITIVE_INFINITY; this.overwrite = flags.overwrite; - this.maxDepth = flags['max-depth']; let id = args.id; let outputFinalized = Promise.resolve(); @@ -77,15 +76,16 @@ class FoldersDownloadCommand extends BoxCommand { } /* eslint-enable no-sync */ - let spinner = ora('Starting download').start(); + this.spinner = ora('Starting download').start(); if (flags.zip) { + this.overwrite = true; let fileName = `folders-download-${id}-${dateTime.format( new Date(), 'YYYY-MM-DDTHH_mm_ss_SSS' )}.zip`; - this.outputPath = path.join(destinationPath, fileName); - outputFinalized = this._setupZip(this.outputPath); + rootItemPath = path.join(destinationPath, fileName); + outputFinalized = this._setupZip(rootItemPath); } try { @@ -95,7 +95,7 @@ class FoldersDownloadCommand extends BoxCommand { // Set output path to the top-level folder, which is the first item in the generator rootItemPath = rootItemPath || item.path; - spinner.text = `Creating folder ${item.id} at ${item.path}`; + this.spinnerLog(`Creating folder ${item.id} at ${item.path}`); try { await mkdirp(path.join(destinationPath, item.path)); } catch (ex) { @@ -105,7 +105,7 @@ class FoldersDownloadCommand extends BoxCommand { ); } } else if (item.type === 'file') { - spinner.text = `Downloading file ${item.id} to ${item.path}`; + this.spinnerLog(`Downloading file ${item.id} to ${item.path}`); let stream = await this.client.files.getReadStream(item.id); if (this.zip) { @@ -117,7 +117,7 @@ class FoldersDownloadCommand extends BoxCommand { } } } catch (err) { - spinner.stop(); + this.spinner.stop(); throw err; } @@ -125,11 +125,21 @@ class FoldersDownloadCommand extends BoxCommand { this.zip.finalize(); } await outputFinalized; - spinner.succeed( - `Downloaded folder ${id} to ${path.join(this.outputPath, rootItemPath)}` + this.spinner.succeed( + `${this.bufferLog || ''}\nDownloaded folder ${id} to ${path.join( + this.outputPath, + rootItemPath + )}`.trim() ); } + spinnerLog(message, preserveText = false) { + this.spinner.text = `${this.bufferLog || ''}\n${message}`.trim(); + if (preserveText) { + this.bufferLog = this.spinner.text; + } + } + /** * Generator for items in the given folder. Yields items starting with the top-level folder itself. * @param {string} folderId The ID of the folder to generate items for @@ -138,7 +148,7 @@ class FoldersDownloadCommand extends BoxCommand { * @returns {void} * @private */ - async* _getItems(folderId, folderPath) { + async *_getItems(folderId, folderPath) { let folder = await this.client.folders.get(folderId); folderPath = path.join(folderPath, folder.name); @@ -158,22 +168,23 @@ class FoldersDownloadCommand extends BoxCommand { folderItems = { [Symbol.asyncIterator]: () => iterator }; } for await (let item of folderItems) { - if ( - item.type === 'folder' && - folderPath.split(path.sep).length <= this.maxRecurDepth - ) { + if (item.type === 'folder') { // We only recurse this folder by one of the following conditions: - // 1. The overwrite flag is true. We will download all files and folders. - // 2. The maxDepth flag is set to 'max'. We will go through all folders at any depth. - // 3. The folder does not exist. We will go through all folders at any depth and download all files. + // 1. The overwrite flag is true. We will download all files and folders within the provided depth (overwite). + // 2. The folder does not exist. We will download all files and folders within the provided depth. + // 3. The folder exists and overwrite is false, we only download files and folders not existing, within the provided depth. /* eslint-disable no-sync */ if ( - this.overwrite || - this.maxDepth === 'max' || - !fs.existsSync(path.join(this.outputPath, folderPath, item.name)) + folderPath.split(path.sep).length <= this.maxDepth ) { /* eslint-enable no-sync */ yield* this._getItems(item.id, folderPath); + } else { + // If the folder exists and overwrite is false, we skip the folder. + this.spinnerLog( + `Skipping folder ${item.name} (${item.id}) at ${folderPath} because reached max depth of ${this.maxDepth}`, + true + ); } } else if (item.type === 'file') { // We only download file if overwrite is true or the file does not exist. @@ -190,6 +201,11 @@ class FoldersDownloadCommand extends BoxCommand { name: item.name, path: path.join(folderPath, item.name), }; + } else { + this.spinnerLog( + `Skipping file ${item.name} (${item.id}) at ${folderPath} because it already exists and overwrite is disabled`, + true + ); } } } @@ -257,12 +273,6 @@ FoldersDownloadCommand.flags = { allowNo: true, default: true, }), - 'max-depth': flags.enum({ - description: - 'Maximum depth to verify if files and folders are already exists, only used with --no-overwrite', - options: ['root', 'max'], - default: 'max', - }), overwrite: flags.boolean({ description: '[default: true] Overwrite the folder if it already exists.', allowNo: true, diff --git a/test/commands/folders.test.js b/test/commands/folders.test.js index 0f039fc9..080257ae 100644 --- a/test/commands/folders.test.js +++ b/test/commands/folders.test.js @@ -1901,7 +1901,7 @@ describe('Folders', () => { `--destination=${destination}`, '--no-color', '--no-overwrite', - '--max-depth=root', + '--depth=0', '--token=test', ]) .it('should not overwrite existing file and folder in root folder when --no-overwrite and --max-depth=root flag is passed', async(ctx) => { @@ -1950,10 +1950,10 @@ describe('Folders', () => { `--destination=${destination}`, '--no-color', '--no-overwrite', - '--max-depth=max', + '--depth=10', '--token=test', ]) - .it('should not overwrite existing file and folder in folder recursively when --no-overwrite and --max-depth=max flag is passed', async(ctx) => { + .it('should not overwrite existing file and folder in folder recursively when --no-overwrite and --depth=10 flag is passed', async(ctx) => { let folderPath = path.join(destination, folderName); let actualContents = getDirectoryContents(folderPath); await fs.remove(destination); From 44c1ac15aedd875880613954d4bbf8d98ab2f68a Mon Sep 17 00:00:00 2001 From: Minh Nguyen Cong Date: Mon, 5 Feb 2024 17:16:08 +0100 Subject: [PATCH 4/5] Update logic --- src/commands/folders/download.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/folders/download.js b/src/commands/folders/download.js index 4a0d8ff9..fb3a0fb5 100644 --- a/src/commands/folders/download.js +++ b/src/commands/folders/download.js @@ -84,8 +84,8 @@ class FoldersDownloadCommand extends BoxCommand { new Date(), 'YYYY-MM-DDTHH_mm_ss_SSS' )}.zip`; - rootItemPath = path.join(destinationPath, fileName); - outputFinalized = this._setupZip(rootItemPath); + rootItemPath = fileName; + outputFinalized = this._setupZip(path.join(destinationPath, fileName)); } try { @@ -148,7 +148,7 @@ class FoldersDownloadCommand extends BoxCommand { * @returns {void} * @private */ - async *_getItems(folderId, folderPath) { + async* _getItems(folderId, folderPath) { let folder = await this.client.folders.get(folderId); folderPath = path.join(folderPath, folder.name); From 731c2dec00e756c3de068d6298d6d79441fde915 Mon Sep 17 00:00:00 2001 From: Minh Nguyen Cong Date: Tue, 6 Feb 2024 11:57:37 +0100 Subject: [PATCH 5/5] Update folders.test.js --- test/commands/folders.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/commands/folders.test.js b/test/commands/folders.test.js index 080257ae..6cbed4e8 100644 --- a/test/commands/folders.test.js +++ b/test/commands/folders.test.js @@ -1904,7 +1904,7 @@ describe('Folders', () => { '--depth=0', '--token=test', ]) - .it('should not overwrite existing file and folder in root folder when --no-overwrite and --max-depth=root flag is passed', async(ctx) => { + .it('should not overwrite existing file and folder in root folder when --no-overwrite and --depth=0 flag is passed', async(ctx) => { let folderPath = path.join(destination, folderName); let actualContents = getDirectoryContents(folderPath); await fs.remove(destination);