diff --git a/packages/filesystem/package.json b/packages/filesystem/package.json index b71f7cfe05131..124d684328f67 100644 --- a/packages/filesystem/package.json +++ b/packages/filesystem/package.json @@ -8,7 +8,6 @@ "@types/body-parser": "^1.17.0", "@types/formidable": "^1.0.31", "@types/fs-extra": "^4.0.2", - "@types/mime-types": "^2.1.0", "@types/rimraf": "^2.0.2", "@types/tar-fs": "^1.16.1", "@types/touch": "0.0.1", @@ -19,7 +18,6 @@ "formidable": "^1.2.1", "fs-extra": "^4.0.2", "http-status-codes": "^1.3.0", - "mime-types": "^2.1.18", "minimatch": "^3.0.4", "mv": "^2.1.1", "rimraf": "^2.6.2", diff --git a/packages/filesystem/src/browser/download/file-download-service.ts b/packages/filesystem/src/browser/download/file-download-service.ts index 9f057c905ca38..7bec11ffc29d3 100644 --- a/packages/filesystem/src/browser/download/file-download-service.ts +++ b/packages/filesystem/src/browser/download/file-download-service.ts @@ -19,19 +19,16 @@ import URI from '@theia/core/lib/common/uri'; import { cancelled } from '@theia/core/lib/common/cancellation'; import { ILogger } from '@theia/core/lib/common/logger'; import { Endpoint } from '@theia/core/lib/browser/endpoint'; -import { StatusBar, StatusBarAlignment } from '@theia/core/lib/browser/status-bar'; import { FileSystem } from '../../common/filesystem'; import { FileDownloadData } from '../../common/download/file-download-data'; import { Deferred } from '@theia/core/lib/common/promise-util'; import { MessageService } from '@theia/core/lib/common/message-service'; +import { addClipboardListener } from '@theia/core/lib/browser/widgets'; @injectable() export class FileDownloadService { - protected static PREPARING_DOWNLOAD_ID = 'theia-preparing-download'; - protected anchor: HTMLAnchorElement | undefined; - protected downloadQueue: number[] = []; protected downloadCounter: number = 0; @inject(ILogger) @@ -40,8 +37,6 @@ export class FileDownloadService { @inject(FileSystem) protected readonly fileSystem: FileSystem; - @inject(StatusBar) - protected readonly statusBar: StatusBar; @inject(MessageService) protected readonly messageService: MessageService; @@ -147,54 +142,68 @@ export class FileDownloadService { return this.deferredUpload.promise; } + protected handleCopy(event: ClipboardEvent, downloadUrl: string) { + if (downloadUrl) { + event.clipboardData.setData('text/plain', downloadUrl); + event.preventDefault(); + this.messageService.info('Download link copied!'); + } + } + + async cancelDownload(id: string) { + await fetch(`${this.endpoint()}/download/?id=${id}&cancel=true`); + } + async download(uris: URI[]): Promise { + let cancel = false; if (uris.length === 0) { return; } - let downloadId: number | undefined; try { - downloadId = this.downloadCounter++; - if (this.downloadQueue.length === 0) { - await this.statusBar.setElement(FileDownloadService.PREPARING_DOWNLOAD_ID, { - alignment: StatusBarAlignment.RIGHT, - text: '$(spinner~spin) Preparing download...', - tooltip: 'Preparing download...', - priority: 1 - }); + const [progress, response] = await Promise.all([ + this.messageService.showProgress({ + text: 'Preparing download link...', options: { cancelable: true } + }, () => { cancel = true; }), + fetch(this.request(uris)) + ]); + const jsonResponse = await response.json(); + if (cancel) { + this.cancelDownload(jsonResponse.id); + return; } - this.downloadQueue.push(downloadId); - const response = await fetch(this.request(uris)); - await this.statusBar.removeElement(FileDownloadService.PREPARING_DOWNLOAD_ID); - const title = await this.title(response, uris); const { status, statusText } = response; if (status === 200) { - await this.forceDownload(response, decodeURIComponent(title)); + progress.cancel(); + const downloadUrl = `${this.endpoint()}/download/?id=${jsonResponse.id}`; + this.messageService.info(downloadUrl, 'Download', 'Copy Download Link').then(action => { + if (action === 'Download') { + this.forceDownload(jsonResponse.id, decodeURIComponent(jsonResponse.name)); + this.messageService.info('Download started!'); + } else if (action === 'Copy Download Link') { + if (document.documentElement) { + addClipboardListener(document.documentElement, 'copy', e => this.handleCopy(e, downloadUrl)); + document.execCommand('copy'); + } + } else { + this.cancelDownload(jsonResponse.id); + } + }); } else { throw new Error(`Received unexpected status code: ${status}. [${statusText}]`); } } catch (e) { this.logger.error(`Error occurred when downloading: ${uris.map(u => u.toString(true))}.`, e); - } finally { - if (downloadId !== undefined) { - const indexOf = this.downloadQueue.indexOf(downloadId); - if (indexOf !== -1) { - this.downloadQueue.splice(indexOf, 1); - } - if (this.downloadQueue.length === 0) { - this.statusBar.removeElement(FileDownloadService.PREPARING_DOWNLOAD_ID); - } - } } } - protected async forceDownload(response: Response, title: string): Promise { + protected async forceDownload(id: string, title: string): Promise { let url: string | undefined; try { - const blob = await response.blob(); - url = URL.createObjectURL(blob); if (this.anchor === undefined) { this.anchor = document.createElement('a'); } + const endpoint = this.endpoint(); + url = `${endpoint}/download/?id=${id}`; this.anchor.href = url; this.anchor.style.display = 'none'; this.anchor.download = title; @@ -211,24 +220,6 @@ export class FileDownloadService { } } - protected async title(response: Response, uris: URI[]): Promise { - let title = (response.headers.get('Content-Disposition') || '').split('attachment; filename=').pop(); - if (title) { - return title; - } - // tslint:disable-next-line:whitespace - const [uri,] = uris; - if (uris.length === 1) { - const stat = await this.fileSystem.getFileStat(uri.toString()); - if (stat === undefined) { - throw new Error(`Unexpected error occurred when downloading file. Files does not exist. URI: ${uri.toString(true)}.`); - } - title = uri.path.base; - return stat.isDirectory ? `${title}.tar` : title; - } - return `${uri.parent.path.name}.tar`; - } - protected request(uris: URI[]): Request { const url = this.url(uris); const init = this.requestInit(uris); diff --git a/packages/filesystem/src/node/download/file-download-backend-module.ts b/packages/filesystem/src/node/download/file-download-backend-module.ts index 109f6a0b500f9..c5e893dd3f8ff 100644 --- a/packages/filesystem/src/node/download/file-download-backend-module.ts +++ b/packages/filesystem/src/node/download/file-download-backend-module.ts @@ -17,13 +17,16 @@ import { ContainerModule } from 'inversify'; import { BackendApplicationContribution } from '@theia/core/lib/node/backend-application'; import { FileDownloadEndpoint } from './file-download-endpoint'; -import { FileDownloadHandler, SingleFileDownloadHandler, MultiFileDownloadHandler } from './file-download-handler'; +import { FileDownloadHandler, SingleFileDownloadHandler, MultiFileDownloadHandler, DownloadLinkHandler } from './file-download-handler'; import { DirectoryArchiver } from './directory-archiver'; +import { FileDownloadCache } from './file-download-cache'; export default new ContainerModule(bind => { bind(FileDownloadEndpoint).toSelf().inSingletonScope(); bind(BackendApplicationContribution).toService(FileDownloadEndpoint); + bind(FileDownloadCache).toSelf().inSingletonScope(); bind(FileDownloadHandler).to(SingleFileDownloadHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.SINGLE); bind(FileDownloadHandler).to(MultiFileDownloadHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.MULTI); + bind(FileDownloadHandler).to(DownloadLinkHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.DOWNLOAD_LINK); bind(DirectoryArchiver).toSelf().inSingletonScope(); }); diff --git a/packages/filesystem/src/node/download/file-download-cache.ts b/packages/filesystem/src/node/download/file-download-cache.ts new file mode 100644 index 0000000000000..09591f8aaf83b --- /dev/null +++ b/packages/filesystem/src/node/download/file-download-cache.ts @@ -0,0 +1,88 @@ +/******************************************************************************** + * Copyright (C) 2019 Bitsler and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ +import { injectable, inject } from 'inversify'; +import { ILogger } from '@theia/core/lib/common/logger'; +import * as rimraf from 'rimraf'; + +export interface DownloadStorageItem { + file: string; + root?: string; + size: number; + remove: boolean; + expire?: number; +} + +@injectable() +export class FileDownloadCache { + + @inject(ILogger) + protected readonly logger: ILogger; + protected readonly downloads = new Map(); + protected readonly expireTimeInMinutes: number = 1; + + addDownload(id: string, downloadInfo: DownloadStorageItem): void { + downloadInfo.file = encodeURIComponent(downloadInfo.file); + if (downloadInfo.root) { + downloadInfo.root = encodeURIComponent(downloadInfo.root); + } + // expires in 1 minute enough for parallel connections to be connected. + downloadInfo.expire = Date.now() + (this.expireTimeInMinutes * 600000); + this.downloads.set(id, downloadInfo); + } + + getDownload(id: string): DownloadStorageItem | undefined { + this.expireDownloads(); + const downloadInfo = this.downloads.get(id); + if (downloadInfo) { + + downloadInfo.file = decodeURIComponent(downloadInfo.file); + if (downloadInfo.root) { + downloadInfo.root = decodeURIComponent(downloadInfo.root); + } + } + return downloadInfo; + } + + deleteDownload(id: string): void { + const downloadInfo = this.downloads.get(id); + if (downloadInfo && downloadInfo.remove) { + this.deleteRecursively(downloadInfo.root || downloadInfo.file); + } + this.downloads.delete(id); + } + + values(): { [key: string]: DownloadStorageItem } { + this.expireDownloads(); + return [...this.downloads.entries()].reduce((downloads, [key, value]) => ({ ...downloads, [key]: value }), {}); + } + + protected deleteRecursively(pathToDelete: string): void { + rimraf(pathToDelete, error => { + if (error) { + this.logger.warn(`An error occurred while deleting the temporary data from the disk. Cannot clean up: ${pathToDelete}.`, error); + } + }); + } + + protected expireDownloads(): void { + const time = Date.now(); + for (const [id, download] of this.downloads.entries()) { + if (download.expire && download.expire <= time) { + this.deleteDownload(id); + } + } + } +} diff --git a/packages/filesystem/src/node/download/file-download-endpoint.ts b/packages/filesystem/src/node/download/file-download-endpoint.ts index be81f6306df09..afb7b0eed337d 100644 --- a/packages/filesystem/src/node/download/file-download-endpoint.ts +++ b/packages/filesystem/src/node/download/file-download-endpoint.ts @@ -43,9 +43,14 @@ export class FileDownloadEndpoint implements BackendApplicationContribution { @named(FileDownloadHandler.MULTI) protected readonly multiFileDownloadHandler: FileDownloadHandler; + @inject(FileDownloadHandler) + @named(FileDownloadHandler.DOWNLOAD_LINK) + protected readonly downloadLinkHandler: FileDownloadHandler; + configure(app: Application): void { const upload = this.upload.bind(this); const router = Router(); + router.get('/download', (request, response) => this.downloadLinkHandler.handle(request, response)); router.get('/', (request, response) => this.singleFileDownloadHandler.handle(request, response)); router.put('/', (request, response) => this.multiFileDownloadHandler.handle(request, response)); router.post('/', upload); diff --git a/packages/filesystem/src/node/download/file-download-handler.ts b/packages/filesystem/src/node/download/file-download-handler.ts index f812b6df71d66..123e953168651 100644 --- a/packages/filesystem/src/node/download/file-download-handler.ts +++ b/packages/filesystem/src/node/download/file-download-handler.ts @@ -17,12 +17,10 @@ import * as os from 'os'; import * as fs from 'fs-extra'; import * as path from 'path'; -import * as rimraf from 'rimraf'; import { v4 } from 'uuid'; -import { lookup } from 'mime-types'; import { Request, Response } from 'express'; import { inject, injectable } from 'inversify'; -import { OK, BAD_REQUEST, METHOD_NOT_ALLOWED, NOT_FOUND, INTERNAL_SERVER_ERROR } from 'http-status-codes'; +import { OK, BAD_REQUEST, METHOD_NOT_ALLOWED, NOT_FOUND, INTERNAL_SERVER_ERROR, REQUESTED_RANGE_NOT_SATISFIABLE, PARTIAL_CONTENT } from 'http-status-codes'; import URI from '@theia/core/lib/common/uri'; import { isEmpty } from '@theia/core/lib/common/objects'; import { ILogger } from '@theia/core/lib/common/logger'; @@ -30,6 +28,14 @@ import { FileUri } from '@theia/core/lib/node/file-uri'; import { FileSystem } from '../../common/filesystem'; import { DirectoryArchiver } from './directory-archiver'; import { FileDownloadData } from '../../common/download/file-download-data'; +import { FileDownloadCache, DownloadStorageItem } from './file-download-cache'; + +interface PrepareDownloadOptions { + filePath: string; + downloadId: string; + remove: boolean; + root?: string; +} @injectable() export abstract class FileDownloadHandler { @@ -43,46 +49,103 @@ export abstract class FileDownloadHandler { @inject(DirectoryArchiver) protected readonly directoryArchiver: DirectoryArchiver; + @inject(FileDownloadCache) + protected readonly fileDownloadCache: FileDownloadCache; + public abstract handle(request: Request, response: Response): Promise; - protected async download(filePath: string, request: Request, response: Response): Promise { - const name = path.basename(filePath); - const mimeType = lookup(filePath); - if (mimeType) { - response.contentType(mimeType); - } else { - this.logger.debug(`Cannot determine the content-type for file: ${filePath}. Skipping the 'Content-type' header from the HTTP response.`); + /** + * Prepares the file and the link for download + */ + protected async prepareDownload(request: Request, response: Response, options: PrepareDownloadOptions): Promise { + const name = path.basename(options.filePath); + try { + await fs.access(options.filePath, fs.constants.R_OK); + const stat = await fs.stat(options.filePath); + this.fileDownloadCache.addDownload(options.downloadId, { file: options.filePath, remove: options.remove, size: stat.size, root: options.root }); + // do not send filePath but instead use the downloadId + const data = { name, id: options.downloadId }; + response.status(OK).send(data).end(); + } catch (e) { + this.handleError(response, e, INTERNAL_SERVER_ERROR); } - response.setHeader('Content-Disposition', `attachment; filename=${encodeURIComponent(name)}`); + } + + protected async download(request: Request, response: Response, downloadInfo: DownloadStorageItem, id: string): Promise { + const filePath = downloadInfo.file; + const statSize = downloadInfo.size; + // this sets the content-disposition and content-type automatically + response.attachment(filePath); try { await fs.access(filePath, fs.constants.R_OK); - fs.readFile(filePath, (error, data) => { - if (error) { - this.handleError(response, error, INTERNAL_SERVER_ERROR); + response.setHeader('Accept-Ranges', 'bytes'); + // parse range header and combine multiple ranges + const range = this.parseRangeHeader(request.headers['range'], statSize); + if (!range) { + response.setHeader('Content-Length', statSize); + this.streamDownload(OK, response, fs.createReadStream(filePath), id); + } else { + const rangeStart = range.start; + const rangeEnd = range.end; + if (rangeStart >= statSize || rangeEnd >= statSize) { + response.setHeader('Content-Range', `bytes */${statSize}`); + // Return the 416 'Requested Range Not Satisfiable'. + response.status(REQUESTED_RANGE_NOT_SATISFIABLE).end(); return; } - response.status(OK).send(data).end(); - }); + response.setHeader('Content-Range', `bytes ${rangeStart}-${rangeEnd}/${statSize}`); + response.setHeader('Content-Length', rangeStart === rangeEnd ? 0 : (rangeEnd - rangeStart + 1)); + response.setHeader('Cache-Control', 'no-cache'); + this.streamDownload(PARTIAL_CONTENT, response, fs.createReadStream(filePath, { start: rangeStart, end: rangeEnd }), id); + } } catch (e) { + this.fileDownloadCache.deleteDownload(id); this.handleError(response, e, INTERNAL_SERVER_ERROR); } } - + /** + * Streams the file and pipe it to the Response to avoid any OOM issues + */ + protected streamDownload(status: number, response: Response, stream: fs.ReadStream, id: string) { + response.status(status); + stream.on('error', error => { + this.fileDownloadCache.deleteDownload(id); + this.handleError(response, error, INTERNAL_SERVER_ERROR); + }); + response.on('error', error => { + this.fileDownloadCache.deleteDownload(id); + this.handleError(response, error, INTERNAL_SERVER_ERROR); + }); + response.on('close', () => { + stream.destroy(); + }); + stream.pipe(response); + } + protected parseRangeHeader(range: string | string[] | undefined, statSize: number): { start: number, end: number } | undefined { + if (!range || range.length === 0 || Array.isArray(range)) { + return; + } + const index = range.indexOf('='); + if (index === -1) { + return; + } + const rangeType = range.slice(0, index); + if (rangeType !== 'bytes') { + return; + } + const [start, end] = range.slice(index + 1).split('-').map(r => parseInt(r, 10)); + return { + start: isNaN(start) ? 0 : start, + end: (isNaN(end) || end > statSize - 1) ? (statSize - 1) : end + }; + } protected async archive(inputPath: string, outputPath: string = path.join(os.tmpdir(), v4()), entries?: string[]): Promise { await this.directoryArchiver.archive(inputPath, outputPath, entries); return outputPath; } - protected async deleteRecursively(pathToDelete: string): Promise { - rimraf(pathToDelete, error => { - if (error) { - this.logger.warn(`An error occurred while deleting the temporary data from the disk. Cannot clean up: ${pathToDelete}.`, error); - } - }); - } - - protected async createTempDir(): Promise { - const outputPath = path.join(os.tmpdir(), v4()); + protected async createTempDir(downloadId: string = v4()): Promise { + const outputPath = path.join(os.tmpdir(), downloadId); await fs.mkdir(outputPath); return outputPath; } @@ -97,6 +160,41 @@ export abstract class FileDownloadHandler { export namespace FileDownloadHandler { export const SINGLE: symbol = Symbol('single'); export const MULTI: symbol = Symbol('multi'); + export const DOWNLOAD_LINK: symbol = Symbol('download'); +} + +@injectable() +export class DownloadLinkHandler extends FileDownloadHandler { + + async handle(request: Request, response: Response): Promise { + const { method, query } = request; + if (method !== 'GET' && method !== 'HEAD') { + this.handleError(response, `Unexpected HTTP method. Expected GET got '${method}' instead.`, METHOD_NOT_ALLOWED); + return; + } + if (query === undefined || query.id === undefined || typeof query.id !== 'string') { + this.handleError(response, `Cannot access the 'id' query from the request. The query was: ${JSON.stringify(query)}.`, BAD_REQUEST); + return; + } + const cancelDownload = query.cancel; + const downloadInfo = this.fileDownloadCache.getDownload(query.id); + if (!downloadInfo) { + this.handleError(response, `Cannot find the file from the request. The query was: ${JSON.stringify(query)}.`, NOT_FOUND); + return; + } + // allow head request to determine the content length for parallel downloaders + if (method === 'HEAD') { + response.setHeader('Content-Length', downloadInfo.size); + response.status(OK).end(); + return; + } + if (!cancelDownload) { + this.download(request, response, downloadInfo, query.id); + } else { + this.logger.info('Download', query.id, 'has been cancelled'); + this.fileDownloadCache.deleteDownload(query.id); + } + } } @injectable() @@ -123,16 +221,19 @@ export class SingleFileDownloadHandler extends FileDownloadHandler { return; } try { + const downloadId = v4(); const filePath = FileUri.fsPath(uri); + const options: PrepareDownloadOptions = { filePath, downloadId, remove: false }; if (!stat.isDirectory) { - await this.download(filePath, request, response); + await this.prepareDownload(request, response, options); } else { - const outputRootPath = path.join(os.tmpdir(), v4()); - await fs.mkdir(outputRootPath); + const outputRootPath = await this.createTempDir(downloadId); const outputPath = path.join(outputRootPath, `${path.basename(filePath)}.tar`); await this.archive(filePath, outputPath); - await this.download(outputPath, request, response); - this.deleteRecursively(outputPath); + options.filePath = outputPath; + options.remove = true; + options.root = outputRootPath; + await this.prepareDownload(request, response, options); } } catch (e) { this.handleError(response, e, INTERNAL_SERVER_ERROR); @@ -170,8 +271,8 @@ export class MultiFileDownloadHandler extends FileDownloadHandler { } } try { - const outputRootPath = path.join(os.tmpdir(), v4()); - await fs.mkdir(outputRootPath); + const downloadId = v4(); + const outputRootPath = await this.createTempDir(downloadId); const distinctUris = Array.from(new Set(body.uris.map(uri => new URI(uri)))); const tarPaths = []; // We should have one key in the map per FS drive. @@ -182,18 +283,18 @@ export class MultiFileDownloadHandler extends FileDownloadHandler { await this.archive(rootPath, outputPath, entries); tarPaths.push(outputPath); } - + const options: PrepareDownloadOptions = { filePath: '', downloadId, remove: true, root: outputRootPath }; if (tarPaths.length === 1) { // tslint:disable-next-line:whitespace const [outputPath,] = tarPaths; - await this.download(outputPath, request, response); - this.deleteRecursively(outputRootPath); + options.filePath = outputPath; + await this.prepareDownload(request, response, options); } else { // We need to tar the tars. const outputPath = path.join(outputRootPath, `theia-archive-${Date.now()}.tar`); + options.filePath = outputPath; await this.archive(outputRootPath, outputPath, tarPaths.map(p => path.relative(outputRootPath, p))); - await this.download(outputPath, request, response); - this.deleteRecursively(outputRootPath); + await this.prepareDownload(request, response, options); } } catch (e) { this.handleError(response, e, INTERNAL_SERVER_ERROR); diff --git a/packages/workspace/src/browser/workspace-commands.ts b/packages/workspace/src/browser/workspace-commands.ts index 318023e612470..057c0a501876c 100644 --- a/packages/workspace/src/browser/workspace-commands.ts +++ b/packages/workspace/src/browser/workspace-commands.ts @@ -146,6 +146,7 @@ export class FileMenuContribution implements MenuContribution { commandId: FileDownloadCommands.DOWNLOAD.id, order: 'b' }); + } }