Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Support overwrite/skip folder download #516

Merged
merged 6 commits into from
Feb 6, 2024
Merged

feat: Support overwrite/skip folder download #516

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
64aa022
feat: Support overwrite/skip folder download
congminh1254 Jan 30, 2024
77bf65b
Update docs
congminh1254 Jan 30, 2024
61a65dd
Update logic
congminh1254 Feb 5, 2024
44c1ac1
Update logic
congminh1254 Feb 5, 2024
ab1937a
Merge branch 'main' into support-overwrite-skip
congminh1254 Feb 5, 2024
731c2de
Update folders.test.js
congminh1254 Feb 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 98 additions & 33 deletions src/commands/folders/download.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,15 @@ const utils = require('../../util');
* @private
*/
function saveFileToDisk(folderPath, file, stream) {

let output;
try {
output = fs.createWriteStream(path.join(folderPath, file.path));
stream.pipe(output);
} catch (ex) {
throw new BoxCLIError(`Error downloading file ${file.id} to ${file.path}`, ex);
throw new BoxCLIError(
`Error downloading file ${file.id} to ${file.path}`,
ex
);
}

/* eslint-disable promise/avoid-new */
Expand All @@ -44,11 +46,16 @@ class FoldersDownloadCommand extends BoxCommand {
async run() {
const { flags, args } = this.parse(FoldersDownloadCommand);

this.maxDepth = flags.hasOwnProperty('depth') && flags.depth >= 0 ? flags.depth : Number.POSITIVE_INFINITY;
this.outputPath = null;
this.maxDepth =
flags.hasOwnProperty('depth') && flags.depth >= 0
? flags.depth
: Number.POSITIVE_INFINITY;
this.overwrite = flags.overwrite;

let outputPath;
let id = args.id;
let outputFinalized = Promise.resolve();
let rootItemPath = null;

let destinationPath;
if (flags.destination) {
Expand All @@ -69,30 +76,36 @@ class FoldersDownloadCommand extends BoxCommand {
}
/* eslint-enable no-sync */

let spinner = ora('Starting download').start();
this.spinner = ora('Starting download').start();

if (flags.zip) {
let fileName = `folders-download-${id}-${dateTime.format(new Date(), 'YYYY-MM-DDTHH_mm_ss_SSS')}.zip`;
outputPath = path.join(destinationPath, fileName);
outputFinalized = this._setupZip(outputPath);
this.overwrite = true;
let fileName = `folders-download-${id}-${dateTime.format(
new Date(),
'YYYY-MM-DDTHH_mm_ss_SSS'
)}.zip`;
rootItemPath = fileName;
outputFinalized = this._setupZip(path.join(destinationPath, fileName));
}

try {
this.outputPath = destinationPath;
for await (let item of this._getItems(id, '')) {
if (item.type === 'folder' && !this.zip) {

// Set output path to the top-level folder, which is the first item in the generator
outputPath = outputPath || path.join(destinationPath, item.path);
rootItemPath = rootItemPath || item.path;

spinner.text = `Creating folder ${item.id} at ${item.path}`;
this.spinnerLog(`Creating folder ${item.id} at ${item.path}`);
try {
await mkdirp(path.join(destinationPath, item.path));
} catch (ex) {
throw new BoxCLIError(`Folder ${item.path} could not be created`, ex);
throw new BoxCLIError(
`Folder ${item.path} could not be created`,
ex
);
}
} else if (item.type === 'file') {

spinner.text = `Downloading file ${item.id} to ${item.path}`;
this.spinnerLog(`Downloading file ${item.id} to ${item.path}`);
let stream = await this.client.files.getReadStream(item.id);

if (this.zip) {
Expand All @@ -104,15 +117,27 @@ class FoldersDownloadCommand extends BoxCommand {
}
}
} catch (err) {
spinner.stop();
this.spinner.stop();
throw err;
}

if (this.zip) {
this.zip.finalize();
}
await outputFinalized;
spinner.succeed(`Downloaded folder ${id} to ${outputPath}`);
this.spinner.succeed(
`${this.bufferLog || ''}\nDownloaded folder ${id} to ${path.join(
this.outputPath,
rootItemPath
)}`.trim()
);
}

spinnerLog(message, preserveText = false) {
this.spinner.text = `${this.bufferLog || ''}\n${message}`.trim();
if (preserveText) {
this.bufferLog = this.spinner.text;
}
}

/**
Expand All @@ -124,7 +149,6 @@ class FoldersDownloadCommand extends BoxCommand {
* @private
*/
async* _getItems(folderId, folderPath) {

let folder = await this.client.folders.get(folderId);
folderPath = path.join(folderPath, folder.name);

Expand All @@ -137,19 +161,52 @@ class FoldersDownloadCommand extends BoxCommand {

let folderItems = folder.item_collection.entries;
if (folder.item_collection.total_count > folderItems.length) {
let iterator = await this.client.folders.getItems(folderId, { usemarker: true, fields: 'type,id,name' });
let iterator = await this.client.folders.getItems(folderId, {
usemarker: true,
fields: 'type,id,name',
});
folderItems = { [Symbol.asyncIterator]: () => iterator };
}
for await (let item of folderItems) {
if (item.type === 'folder' && folderPath.split(path.sep).length <= this.maxDepth) {
yield* this._getItems(item.id, folderPath);
if (item.type === 'folder') {
// We only recurse this folder by one of the following conditions:
// 1. The overwrite flag is true. We will download all files and folders within the provided depth (overwite).
// 2. The folder does not exist. We will download all files and folders within the provided depth.
// 3. The folder exists and overwrite is false, we only download files and folders not existing, within the provided depth.
/* eslint-disable no-sync */
if (
folderPath.split(path.sep).length <= this.maxDepth
) {
/* eslint-enable no-sync */
yield* this._getItems(item.id, folderPath);
} else {
// If the folder exists and overwrite is false, we skip the folder.
this.spinnerLog(
`Skipping folder ${item.name} (${item.id}) at ${folderPath} because reached max depth of ${this.maxDepth}`,
true
);
}
} else if (item.type === 'file') {
yield {
type: 'file',
id: item.id,
name: item.name,
path: path.join(folderPath, item.name),
};
// We only download file if overwrite is true or the file does not exist.
// Skip downloading if overwrite is false and the file exists.
/* eslint-disable no-sync */
if (
this.overwrite ||
!fs.existsSync(path.join(this.outputPath, folderPath, item.name))
) {
/* eslint-enable no-sync */
yield {
congminh1254 marked this conversation as resolved.
Show resolved Hide resolved
type: 'file',
id: item.id,
name: item.name,
path: path.join(folderPath, item.name),
};
} else {
this.spinnerLog(
`Skipping file ${item.name} (${item.id}) at ${folderPath} because it already exists and overwrite is disabled`,
true
);
}
}
}
}
Expand All @@ -163,20 +220,22 @@ class FoldersDownloadCommand extends BoxCommand {
* @private
*/
_setupZip(destinationPath) {

// Set up archive stream
this.zip = archiver('zip', {
zlib: { level: 9 } // Use the best available compression
zlib: { level: 9 }, // Use the best available compression
});

let output;
try {
output = fs.createWriteStream(destinationPath);
} catch (ex) {
throw new BoxCLIError(`Could not write to destination path ${destinationPath}`, ex);
throw new BoxCLIError(
`Could not write to destination path ${destinationPath}`,
ex
);
}

this.zip.on('error', err => {
this.zip.on('error', (err) => {
throw new BoxCLIError('Error writing to zip file', err);
});

Expand All @@ -199,7 +258,7 @@ FoldersDownloadCommand.flags = {
...BoxCommand.flags,
destination: flags.string({
description: 'The destination folder to download the Box folder into',
parse: utils.parsePath
parse: utils.parsePath,
}),
zip: flags.boolean({
congminh1254 marked this conversation as resolved.
Show resolved Hide resolved
description: 'Download the folder into a single .zip archive',
Expand All @@ -209,7 +268,13 @@ FoldersDownloadCommand.flags = {
'Number of levels deep to recurse when downloading the folder tree',
}),
'create-path': flags.boolean({
description: 'Recursively creates a path to a directory if it does not exist',
description:
'Recursively creates a path to a directory if it does not exist',
allowNo: true,
default: true,
}),
overwrite: flags.boolean({
description: '[default: true] Overwrite the folder if it already exists.',
allowNo: true,
default: true,
}),
Expand All @@ -221,7 +286,7 @@ FoldersDownloadCommand.args = [
required: true,
hidden: false,
description: 'ID of the folder to download',
}
},
];

module.exports = FoldersDownloadCommand;
Loading
Loading