From 8afd59d21ea5c2648c407d595a3246958522ae0f Mon Sep 17 00:00:00 2001 From: Uni Sayo Date: Tue, 16 Apr 2019 16:10:28 +0000 Subject: [PATCH] improve download, allow download of big files, works on ff and chrome This PR improves the download by letting the native browser handle it instead of fetching it in the background. This also adds a unique link which expires in 1 minute. Tested on chrome 73.0 and firefox 60.5 Signed-off-by: Uni Sayo --- packages/filesystem/package.json | 2 - .../browser/download/file-download-service.ts | 93 ++++----- .../download/file-download-backend-module.ts | 5 +- .../src/node/download/file-download-cache.ts | 88 +++++++++ .../node/download/file-download-endpoint.ts | 5 + .../node/download/file-download-handler.ts | 179 ++++++++++++++---- .../src/browser/workspace-commands.ts | 1 + 7 files changed, 280 insertions(+), 93 deletions(-) create mode 100644 packages/filesystem/src/node/download/file-download-cache.ts diff --git a/packages/filesystem/package.json b/packages/filesystem/package.json index b71f7cfe05131..124d684328f67 100644 --- a/packages/filesystem/package.json +++ b/packages/filesystem/package.json @@ -8,7 +8,6 @@ "@types/body-parser": "^1.17.0", "@types/formidable": "^1.0.31", "@types/fs-extra": "^4.0.2", - "@types/mime-types": "^2.1.0", "@types/rimraf": "^2.0.2", "@types/tar-fs": "^1.16.1", "@types/touch": "0.0.1", @@ -19,7 +18,6 @@ "formidable": "^1.2.1", "fs-extra": "^4.0.2", "http-status-codes": "^1.3.0", - "mime-types": "^2.1.18", "minimatch": "^3.0.4", "mv": "^2.1.1", "rimraf": "^2.6.2", diff --git a/packages/filesystem/src/browser/download/file-download-service.ts b/packages/filesystem/src/browser/download/file-download-service.ts index 9f057c905ca38..7bec11ffc29d3 100644 --- a/packages/filesystem/src/browser/download/file-download-service.ts +++ b/packages/filesystem/src/browser/download/file-download-service.ts @@ -19,19 +19,16 @@ import URI from '@theia/core/lib/common/uri'; import { cancelled } from '@theia/core/lib/common/cancellation'; import { ILogger } from '@theia/core/lib/common/logger'; import { Endpoint } from '@theia/core/lib/browser/endpoint'; -import { StatusBar, StatusBarAlignment } from '@theia/core/lib/browser/status-bar'; import { FileSystem } from '../../common/filesystem'; import { FileDownloadData } from '../../common/download/file-download-data'; import { Deferred } from '@theia/core/lib/common/promise-util'; import { MessageService } from '@theia/core/lib/common/message-service'; +import { addClipboardListener } from '@theia/core/lib/browser/widgets'; @injectable() export class FileDownloadService { - protected static PREPARING_DOWNLOAD_ID = 'theia-preparing-download'; - protected anchor: HTMLAnchorElement | undefined; - protected downloadQueue: number[] = []; protected downloadCounter: number = 0; @inject(ILogger) @@ -40,8 +37,6 @@ export class FileDownloadService { @inject(FileSystem) protected readonly fileSystem: FileSystem; - @inject(StatusBar) - protected readonly statusBar: StatusBar; @inject(MessageService) protected readonly messageService: MessageService; @@ -147,54 +142,68 @@ export class FileDownloadService { return this.deferredUpload.promise; } + protected handleCopy(event: ClipboardEvent, downloadUrl: string) { + if (downloadUrl) { + event.clipboardData.setData('text/plain', downloadUrl); + event.preventDefault(); + this.messageService.info('Download link copied!'); + } + } + + async cancelDownload(id: string) { + await fetch(`${this.endpoint()}/download/?id=${id}&cancel=true`); + } + async download(uris: URI[]): Promise { + let cancel = false; if (uris.length === 0) { return; } - let downloadId: number | undefined; try { - downloadId = this.downloadCounter++; - if (this.downloadQueue.length === 0) { - await this.statusBar.setElement(FileDownloadService.PREPARING_DOWNLOAD_ID, { - alignment: StatusBarAlignment.RIGHT, - text: '$(spinner~spin) Preparing download...', - tooltip: 'Preparing download...', - priority: 1 - }); + const [progress, response] = await Promise.all([ + this.messageService.showProgress({ + text: 'Preparing download link...', options: { cancelable: true } + }, () => { cancel = true; }), + fetch(this.request(uris)) + ]); + const jsonResponse = await response.json(); + if (cancel) { + this.cancelDownload(jsonResponse.id); + return; } - this.downloadQueue.push(downloadId); - const response = await fetch(this.request(uris)); - await this.statusBar.removeElement(FileDownloadService.PREPARING_DOWNLOAD_ID); - const title = await this.title(response, uris); const { status, statusText } = response; if (status === 200) { - await this.forceDownload(response, decodeURIComponent(title)); + progress.cancel(); + const downloadUrl = `${this.endpoint()}/download/?id=${jsonResponse.id}`; + this.messageService.info(downloadUrl, 'Download', 'Copy Download Link').then(action => { + if (action === 'Download') { + this.forceDownload(jsonResponse.id, decodeURIComponent(jsonResponse.name)); + this.messageService.info('Download started!'); + } else if (action === 'Copy Download Link') { + if (document.documentElement) { + addClipboardListener(document.documentElement, 'copy', e => this.handleCopy(e, downloadUrl)); + document.execCommand('copy'); + } + } else { + this.cancelDownload(jsonResponse.id); + } + }); } else { throw new Error(`Received unexpected status code: ${status}. [${statusText}]`); } } catch (e) { this.logger.error(`Error occurred when downloading: ${uris.map(u => u.toString(true))}.`, e); - } finally { - if (downloadId !== undefined) { - const indexOf = this.downloadQueue.indexOf(downloadId); - if (indexOf !== -1) { - this.downloadQueue.splice(indexOf, 1); - } - if (this.downloadQueue.length === 0) { - this.statusBar.removeElement(FileDownloadService.PREPARING_DOWNLOAD_ID); - } - } } } - protected async forceDownload(response: Response, title: string): Promise { + protected async forceDownload(id: string, title: string): Promise { let url: string | undefined; try { - const blob = await response.blob(); - url = URL.createObjectURL(blob); if (this.anchor === undefined) { this.anchor = document.createElement('a'); } + const endpoint = this.endpoint(); + url = `${endpoint}/download/?id=${id}`; this.anchor.href = url; this.anchor.style.display = 'none'; this.anchor.download = title; @@ -211,24 +220,6 @@ export class FileDownloadService { } } - protected async title(response: Response, uris: URI[]): Promise { - let title = (response.headers.get('Content-Disposition') || '').split('attachment; filename=').pop(); - if (title) { - return title; - } - // tslint:disable-next-line:whitespace - const [uri,] = uris; - if (uris.length === 1) { - const stat = await this.fileSystem.getFileStat(uri.toString()); - if (stat === undefined) { - throw new Error(`Unexpected error occurred when downloading file. Files does not exist. URI: ${uri.toString(true)}.`); - } - title = uri.path.base; - return stat.isDirectory ? `${title}.tar` : title; - } - return `${uri.parent.path.name}.tar`; - } - protected request(uris: URI[]): Request { const url = this.url(uris); const init = this.requestInit(uris); diff --git a/packages/filesystem/src/node/download/file-download-backend-module.ts b/packages/filesystem/src/node/download/file-download-backend-module.ts index 109f6a0b500f9..c5e893dd3f8ff 100644 --- a/packages/filesystem/src/node/download/file-download-backend-module.ts +++ b/packages/filesystem/src/node/download/file-download-backend-module.ts @@ -17,13 +17,16 @@ import { ContainerModule } from 'inversify'; import { BackendApplicationContribution } from '@theia/core/lib/node/backend-application'; import { FileDownloadEndpoint } from './file-download-endpoint'; -import { FileDownloadHandler, SingleFileDownloadHandler, MultiFileDownloadHandler } from './file-download-handler'; +import { FileDownloadHandler, SingleFileDownloadHandler, MultiFileDownloadHandler, DownloadLinkHandler } from './file-download-handler'; import { DirectoryArchiver } from './directory-archiver'; +import { FileDownloadCache } from './file-download-cache'; export default new ContainerModule(bind => { bind(FileDownloadEndpoint).toSelf().inSingletonScope(); bind(BackendApplicationContribution).toService(FileDownloadEndpoint); + bind(FileDownloadCache).toSelf().inSingletonScope(); bind(FileDownloadHandler).to(SingleFileDownloadHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.SINGLE); bind(FileDownloadHandler).to(MultiFileDownloadHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.MULTI); + bind(FileDownloadHandler).to(DownloadLinkHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.DOWNLOAD_LINK); bind(DirectoryArchiver).toSelf().inSingletonScope(); }); diff --git a/packages/filesystem/src/node/download/file-download-cache.ts b/packages/filesystem/src/node/download/file-download-cache.ts new file mode 100644 index 0000000000000..09591f8aaf83b --- /dev/null +++ b/packages/filesystem/src/node/download/file-download-cache.ts @@ -0,0 +1,88 @@ +/******************************************************************************** + * Copyright (C) 2019 Bitsler and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ +import { injectable, inject } from 'inversify'; +import { ILogger } from '@theia/core/lib/common/logger'; +import * as rimraf from 'rimraf'; + +export interface DownloadStorageItem { + file: string; + root?: string; + size: number; + remove: boolean; + expire?: number; +} + +@injectable() +export class FileDownloadCache { + + @inject(ILogger) + protected readonly logger: ILogger; + protected readonly downloads = new Map(); + protected readonly expireTimeInMinutes: number = 1; + + addDownload(id: string, downloadInfo: DownloadStorageItem): void { + downloadInfo.file = encodeURIComponent(downloadInfo.file); + if (downloadInfo.root) { + downloadInfo.root = encodeURIComponent(downloadInfo.root); + } + // expires in 1 minute enough for parallel connections to be connected. + downloadInfo.expire = Date.now() + (this.expireTimeInMinutes * 600000); + this.downloads.set(id, downloadInfo); + } + + getDownload(id: string): DownloadStorageItem | undefined { + this.expireDownloads(); + const downloadInfo = this.downloads.get(id); + if (downloadInfo) { + + downloadInfo.file = decodeURIComponent(downloadInfo.file); + if (downloadInfo.root) { + downloadInfo.root = decodeURIComponent(downloadInfo.root); + } + } + return downloadInfo; + } + + deleteDownload(id: string): void { + const downloadInfo = this.downloads.get(id); + if (downloadInfo && downloadInfo.remove) { + this.deleteRecursively(downloadInfo.root || downloadInfo.file); + } + this.downloads.delete(id); + } + + values(): { [key: string]: DownloadStorageItem } { + this.expireDownloads(); + return [...this.downloads.entries()].reduce((downloads, [key, value]) => ({ ...downloads, [key]: value }), {}); + } + + protected deleteRecursively(pathToDelete: string): void { + rimraf(pathToDelete, error => { + if (error) { + this.logger.warn(`An error occurred while deleting the temporary data from the disk. Cannot clean up: ${pathToDelete}.`, error); + } + }); + } + + protected expireDownloads(): void { + const time = Date.now(); + for (const [id, download] of this.downloads.entries()) { + if (download.expire && download.expire <= time) { + this.deleteDownload(id); + } + } + } +} diff --git a/packages/filesystem/src/node/download/file-download-endpoint.ts b/packages/filesystem/src/node/download/file-download-endpoint.ts index be81f6306df09..afb7b0eed337d 100644 --- a/packages/filesystem/src/node/download/file-download-endpoint.ts +++ b/packages/filesystem/src/node/download/file-download-endpoint.ts @@ -43,9 +43,14 @@ export class FileDownloadEndpoint implements BackendApplicationContribution { @named(FileDownloadHandler.MULTI) protected readonly multiFileDownloadHandler: FileDownloadHandler; + @inject(FileDownloadHandler) + @named(FileDownloadHandler.DOWNLOAD_LINK) + protected readonly downloadLinkHandler: FileDownloadHandler; + configure(app: Application): void { const upload = this.upload.bind(this); const router = Router(); + router.get('/download', (request, response) => this.downloadLinkHandler.handle(request, response)); router.get('/', (request, response) => this.singleFileDownloadHandler.handle(request, response)); router.put('/', (request, response) => this.multiFileDownloadHandler.handle(request, response)); router.post('/', upload); diff --git a/packages/filesystem/src/node/download/file-download-handler.ts b/packages/filesystem/src/node/download/file-download-handler.ts index f812b6df71d66..123e953168651 100644 --- a/packages/filesystem/src/node/download/file-download-handler.ts +++ b/packages/filesystem/src/node/download/file-download-handler.ts @@ -17,12 +17,10 @@ import * as os from 'os'; import * as fs from 'fs-extra'; import * as path from 'path'; -import * as rimraf from 'rimraf'; import { v4 } from 'uuid'; -import { lookup } from 'mime-types'; import { Request, Response } from 'express'; import { inject, injectable } from 'inversify'; -import { OK, BAD_REQUEST, METHOD_NOT_ALLOWED, NOT_FOUND, INTERNAL_SERVER_ERROR } from 'http-status-codes'; +import { OK, BAD_REQUEST, METHOD_NOT_ALLOWED, NOT_FOUND, INTERNAL_SERVER_ERROR, REQUESTED_RANGE_NOT_SATISFIABLE, PARTIAL_CONTENT } from 'http-status-codes'; import URI from '@theia/core/lib/common/uri'; import { isEmpty } from '@theia/core/lib/common/objects'; import { ILogger } from '@theia/core/lib/common/logger'; @@ -30,6 +28,14 @@ import { FileUri } from '@theia/core/lib/node/file-uri'; import { FileSystem } from '../../common/filesystem'; import { DirectoryArchiver } from './directory-archiver'; import { FileDownloadData } from '../../common/download/file-download-data'; +import { FileDownloadCache, DownloadStorageItem } from './file-download-cache'; + +interface PrepareDownloadOptions { + filePath: string; + downloadId: string; + remove: boolean; + root?: string; +} @injectable() export abstract class FileDownloadHandler { @@ -43,46 +49,103 @@ export abstract class FileDownloadHandler { @inject(DirectoryArchiver) protected readonly directoryArchiver: DirectoryArchiver; + @inject(FileDownloadCache) + protected readonly fileDownloadCache: FileDownloadCache; + public abstract handle(request: Request, response: Response): Promise; - protected async download(filePath: string, request: Request, response: Response): Promise { - const name = path.basename(filePath); - const mimeType = lookup(filePath); - if (mimeType) { - response.contentType(mimeType); - } else { - this.logger.debug(`Cannot determine the content-type for file: ${filePath}. Skipping the 'Content-type' header from the HTTP response.`); + /** + * Prepares the file and the link for download + */ + protected async prepareDownload(request: Request, response: Response, options: PrepareDownloadOptions): Promise { + const name = path.basename(options.filePath); + try { + await fs.access(options.filePath, fs.constants.R_OK); + const stat = await fs.stat(options.filePath); + this.fileDownloadCache.addDownload(options.downloadId, { file: options.filePath, remove: options.remove, size: stat.size, root: options.root }); + // do not send filePath but instead use the downloadId + const data = { name, id: options.downloadId }; + response.status(OK).send(data).end(); + } catch (e) { + this.handleError(response, e, INTERNAL_SERVER_ERROR); } - response.setHeader('Content-Disposition', `attachment; filename=${encodeURIComponent(name)}`); + } + + protected async download(request: Request, response: Response, downloadInfo: DownloadStorageItem, id: string): Promise { + const filePath = downloadInfo.file; + const statSize = downloadInfo.size; + // this sets the content-disposition and content-type automatically + response.attachment(filePath); try { await fs.access(filePath, fs.constants.R_OK); - fs.readFile(filePath, (error, data) => { - if (error) { - this.handleError(response, error, INTERNAL_SERVER_ERROR); + response.setHeader('Accept-Ranges', 'bytes'); + // parse range header and combine multiple ranges + const range = this.parseRangeHeader(request.headers['range'], statSize); + if (!range) { + response.setHeader('Content-Length', statSize); + this.streamDownload(OK, response, fs.createReadStream(filePath), id); + } else { + const rangeStart = range.start; + const rangeEnd = range.end; + if (rangeStart >= statSize || rangeEnd >= statSize) { + response.setHeader('Content-Range', `bytes */${statSize}`); + // Return the 416 'Requested Range Not Satisfiable'. + response.status(REQUESTED_RANGE_NOT_SATISFIABLE).end(); return; } - response.status(OK).send(data).end(); - }); + response.setHeader('Content-Range', `bytes ${rangeStart}-${rangeEnd}/${statSize}`); + response.setHeader('Content-Length', rangeStart === rangeEnd ? 0 : (rangeEnd - rangeStart + 1)); + response.setHeader('Cache-Control', 'no-cache'); + this.streamDownload(PARTIAL_CONTENT, response, fs.createReadStream(filePath, { start: rangeStart, end: rangeEnd }), id); + } } catch (e) { + this.fileDownloadCache.deleteDownload(id); this.handleError(response, e, INTERNAL_SERVER_ERROR); } } - + /** + * Streams the file and pipe it to the Response to avoid any OOM issues + */ + protected streamDownload(status: number, response: Response, stream: fs.ReadStream, id: string) { + response.status(status); + stream.on('error', error => { + this.fileDownloadCache.deleteDownload(id); + this.handleError(response, error, INTERNAL_SERVER_ERROR); + }); + response.on('error', error => { + this.fileDownloadCache.deleteDownload(id); + this.handleError(response, error, INTERNAL_SERVER_ERROR); + }); + response.on('close', () => { + stream.destroy(); + }); + stream.pipe(response); + } + protected parseRangeHeader(range: string | string[] | undefined, statSize: number): { start: number, end: number } | undefined { + if (!range || range.length === 0 || Array.isArray(range)) { + return; + } + const index = range.indexOf('='); + if (index === -1) { + return; + } + const rangeType = range.slice(0, index); + if (rangeType !== 'bytes') { + return; + } + const [start, end] = range.slice(index + 1).split('-').map(r => parseInt(r, 10)); + return { + start: isNaN(start) ? 0 : start, + end: (isNaN(end) || end > statSize - 1) ? (statSize - 1) : end + }; + } protected async archive(inputPath: string, outputPath: string = path.join(os.tmpdir(), v4()), entries?: string[]): Promise { await this.directoryArchiver.archive(inputPath, outputPath, entries); return outputPath; } - protected async deleteRecursively(pathToDelete: string): Promise { - rimraf(pathToDelete, error => { - if (error) { - this.logger.warn(`An error occurred while deleting the temporary data from the disk. Cannot clean up: ${pathToDelete}.`, error); - } - }); - } - - protected async createTempDir(): Promise { - const outputPath = path.join(os.tmpdir(), v4()); + protected async createTempDir(downloadId: string = v4()): Promise { + const outputPath = path.join(os.tmpdir(), downloadId); await fs.mkdir(outputPath); return outputPath; } @@ -97,6 +160,41 @@ export abstract class FileDownloadHandler { export namespace FileDownloadHandler { export const SINGLE: symbol = Symbol('single'); export const MULTI: symbol = Symbol('multi'); + export const DOWNLOAD_LINK: symbol = Symbol('download'); +} + +@injectable() +export class DownloadLinkHandler extends FileDownloadHandler { + + async handle(request: Request, response: Response): Promise { + const { method, query } = request; + if (method !== 'GET' && method !== 'HEAD') { + this.handleError(response, `Unexpected HTTP method. Expected GET got '${method}' instead.`, METHOD_NOT_ALLOWED); + return; + } + if (query === undefined || query.id === undefined || typeof query.id !== 'string') { + this.handleError(response, `Cannot access the 'id' query from the request. The query was: ${JSON.stringify(query)}.`, BAD_REQUEST); + return; + } + const cancelDownload = query.cancel; + const downloadInfo = this.fileDownloadCache.getDownload(query.id); + if (!downloadInfo) { + this.handleError(response, `Cannot find the file from the request. The query was: ${JSON.stringify(query)}.`, NOT_FOUND); + return; + } + // allow head request to determine the content length for parallel downloaders + if (method === 'HEAD') { + response.setHeader('Content-Length', downloadInfo.size); + response.status(OK).end(); + return; + } + if (!cancelDownload) { + this.download(request, response, downloadInfo, query.id); + } else { + this.logger.info('Download', query.id, 'has been cancelled'); + this.fileDownloadCache.deleteDownload(query.id); + } + } } @injectable() @@ -123,16 +221,19 @@ export class SingleFileDownloadHandler extends FileDownloadHandler { return; } try { + const downloadId = v4(); const filePath = FileUri.fsPath(uri); + const options: PrepareDownloadOptions = { filePath, downloadId, remove: false }; if (!stat.isDirectory) { - await this.download(filePath, request, response); + await this.prepareDownload(request, response, options); } else { - const outputRootPath = path.join(os.tmpdir(), v4()); - await fs.mkdir(outputRootPath); + const outputRootPath = await this.createTempDir(downloadId); const outputPath = path.join(outputRootPath, `${path.basename(filePath)}.tar`); await this.archive(filePath, outputPath); - await this.download(outputPath, request, response); - this.deleteRecursively(outputPath); + options.filePath = outputPath; + options.remove = true; + options.root = outputRootPath; + await this.prepareDownload(request, response, options); } } catch (e) { this.handleError(response, e, INTERNAL_SERVER_ERROR); @@ -170,8 +271,8 @@ export class MultiFileDownloadHandler extends FileDownloadHandler { } } try { - const outputRootPath = path.join(os.tmpdir(), v4()); - await fs.mkdir(outputRootPath); + const downloadId = v4(); + const outputRootPath = await this.createTempDir(downloadId); const distinctUris = Array.from(new Set(body.uris.map(uri => new URI(uri)))); const tarPaths = []; // We should have one key in the map per FS drive. @@ -182,18 +283,18 @@ export class MultiFileDownloadHandler extends FileDownloadHandler { await this.archive(rootPath, outputPath, entries); tarPaths.push(outputPath); } - + const options: PrepareDownloadOptions = { filePath: '', downloadId, remove: true, root: outputRootPath }; if (tarPaths.length === 1) { // tslint:disable-next-line:whitespace const [outputPath,] = tarPaths; - await this.download(outputPath, request, response); - this.deleteRecursively(outputRootPath); + options.filePath = outputPath; + await this.prepareDownload(request, response, options); } else { // We need to tar the tars. const outputPath = path.join(outputRootPath, `theia-archive-${Date.now()}.tar`); + options.filePath = outputPath; await this.archive(outputRootPath, outputPath, tarPaths.map(p => path.relative(outputRootPath, p))); - await this.download(outputPath, request, response); - this.deleteRecursively(outputRootPath); + await this.prepareDownload(request, response, options); } } catch (e) { this.handleError(response, e, INTERNAL_SERVER_ERROR); diff --git a/packages/workspace/src/browser/workspace-commands.ts b/packages/workspace/src/browser/workspace-commands.ts index 318023e612470..057c0a501876c 100644 --- a/packages/workspace/src/browser/workspace-commands.ts +++ b/packages/workspace/src/browser/workspace-commands.ts @@ -146,6 +146,7 @@ export class FileMenuContribution implements MenuContribution { commandId: FileDownloadCommands.DOWNLOAD.id, order: 'b' }); + } }