diff --git a/packages/filesystem/package.json b/packages/filesystem/package.json index e52debaea06ae..9cd8b66e075d6 100644 --- a/packages/filesystem/package.json +++ b/packages/filesystem/package.json @@ -6,6 +6,7 @@ "@theia/core": "^0.8.0", "@types/body-parser": "^1.17.0", "@types/fs-extra": "^4.0.2", + "@types/mime-types": "^2.1.0", "@types/rimraf": "^2.0.2", "@types/tar-fs": "^1.16.1", "@types/touch": "0.0.1", @@ -14,6 +15,7 @@ "drivelist": "^6.4.3", "fs-extra": "^4.0.2", "http-status-codes": "^1.3.0", + "mime-types": "^2.1.18", "minimatch": "^3.0.4", "mv": "^2.1.1", "rimraf": "^2.6.2", diff --git a/packages/filesystem/src/browser/download/file-download-service.ts b/packages/filesystem/src/browser/download/file-download-service.ts index 818a0f5308eba..7c522ab9716bd 100644 --- a/packages/filesystem/src/browser/download/file-download-service.ts +++ b/packages/filesystem/src/browser/download/file-download-service.ts @@ -18,15 +18,18 @@ import { inject, injectable } from 'inversify'; import URI from '@theia/core/lib/common/uri'; import { ILogger } from '@theia/core/lib/common/logger'; import { Endpoint } from '@theia/core/lib/browser/endpoint'; +import { StatusBar, StatusBarAlignment } from '@theia/core/lib/browser/status-bar'; import { FileSystem } from '../../common/filesystem'; import { FileDownloadData } from '../../common/download/file-download-data'; import { MessageService } from '@theia/core/lib/common/message-service'; -import { addClipboardListener } from '@theia/core/lib/browser/widgets'; @injectable() export class FileDownloadService { + protected static PREPARING_DOWNLOAD_ID = 'theia-preparing-download'; + protected anchor: HTMLAnchorElement | undefined; + protected downloadQueue: number[] = []; protected downloadCounter: number = 0; @inject(ILogger) @@ -35,71 +38,60 @@ export class FileDownloadService { @inject(FileSystem) protected readonly fileSystem: FileSystem; + @inject(StatusBar) + protected readonly statusBar: StatusBar; + @inject(MessageService) protected readonly messageService: MessageService; - protected handleCopy(event: ClipboardEvent, downloadUrl: string) { - if (downloadUrl) { - event.clipboardData.setData('text/plain', downloadUrl); - event.preventDefault(); - this.messageService.info('Download link copied!'); - } - } - - async cancelDownload(id: string) { - await fetch(`${this.endpoint()}/download/?id=${id}&cancel=true`); - } - async download(uris: URI[]): Promise { - let cancel = false; if (uris.length === 0) { return; } + let downloadId: number | undefined; try { - const [progress, response] = await Promise.all([ - this.messageService.showProgress({ - text: 'Preparing download link...', options: { cancelable: true } - }, () => { cancel = true; }), - fetch(this.request(uris)) - ]); - const jsonResponse = await response.json(); - if (cancel) { - this.cancelDownload(jsonResponse.id); - return; + downloadId = this.downloadCounter++; + if (this.downloadQueue.length === 0) { + await this.statusBar.setElement(FileDownloadService.PREPARING_DOWNLOAD_ID, { + alignment: StatusBarAlignment.RIGHT, + text: '$(spinner~spin) Preparing download...', + tooltip: 'Preparing download...', + priority: 1 + }); } + this.downloadQueue.push(downloadId); + const response = await fetch(this.request(uris)); + await this.statusBar.removeElement(FileDownloadService.PREPARING_DOWNLOAD_ID); + const title = await this.title(response, uris); const { status, statusText } = response; if (status === 200) { - progress.cancel(); - const downloadUrl = `${this.endpoint()}/download/?id=${jsonResponse.id}`; - this.messageService.info(downloadUrl, 'Download', 'Copy Download Link').then(action => { - if (action === 'Download') { - this.forceDownload(jsonResponse.id, decodeURIComponent(jsonResponse.name)); - this.messageService.info('Download started!'); - } else if (action === 'Copy Download Link') { - if (document.documentElement) { - addClipboardListener(document.documentElement, 'copy', e => this.handleCopy(e, downloadUrl)); - document.execCommand('copy'); - } - } else { - this.cancelDownload(jsonResponse.id); - } - }); + await this.forceDownload(response, decodeURIComponent(title)); } else { throw new Error(`Received unexpected status code: ${status}. [${statusText}]`); } } catch (e) { this.logger.error(`Error occurred when downloading: ${uris.map(u => u.toString(true))}.`, e); + } finally { + if (downloadId !== undefined) { + const indexOf = this.downloadQueue.indexOf(downloadId); + if (indexOf !== -1) { + this.downloadQueue.splice(indexOf, 1); + } + if (this.downloadQueue.length === 0) { + this.statusBar.removeElement(FileDownloadService.PREPARING_DOWNLOAD_ID); + } + } } } - protected async forceDownload(id: string, title: string): Promise { + protected async forceDownload(response: Response, title: string): Promise { let url: string | undefined; try { + const blob = await response.blob(); + url = URL.createObjectURL(blob); if (this.anchor === undefined) { this.anchor = document.createElement('a'); } - const endpoint = this.endpoint(); - url = `${endpoint}/download/?id=${id}`; this.anchor.href = url; this.anchor.style.display = 'none'; this.anchor.download = title; @@ -116,6 +108,24 @@ export class FileDownloadService { } } + protected async title(response: Response, uris: URI[]): Promise { + let title = (response.headers.get('Content-Disposition') || '').split('attachment; filename=').pop(); + if (title) { + return title; + } + // tslint:disable-next-line:whitespace + const [uri,] = uris; + if (uris.length === 1) { + const stat = await this.fileSystem.getFileStat(uri.toString()); + if (stat === undefined) { + throw new Error(`Unexpected error occurred when downloading file. Files does not exist. URI: ${uri.toString(true)}.`); + } + title = uri.path.base; + return stat.isDirectory ? `${title}.tar` : title; + } + return `${uri.parent.path.name}.tar`; + } + protected request(uris: URI[]): Request { const url = this.url(uris); const init = this.requestInit(uris); diff --git a/packages/filesystem/src/node/download/file-download-backend-module.ts b/packages/filesystem/src/node/download/file-download-backend-module.ts index c5e893dd3f8ff..109f6a0b500f9 100644 --- a/packages/filesystem/src/node/download/file-download-backend-module.ts +++ b/packages/filesystem/src/node/download/file-download-backend-module.ts @@ -17,16 +17,13 @@ import { ContainerModule } from 'inversify'; import { BackendApplicationContribution } from '@theia/core/lib/node/backend-application'; import { FileDownloadEndpoint } from './file-download-endpoint'; -import { FileDownloadHandler, SingleFileDownloadHandler, MultiFileDownloadHandler, DownloadLinkHandler } from './file-download-handler'; +import { FileDownloadHandler, SingleFileDownloadHandler, MultiFileDownloadHandler } from './file-download-handler'; import { DirectoryArchiver } from './directory-archiver'; -import { FileDownloadCache } from './file-download-cache'; export default new ContainerModule(bind => { bind(FileDownloadEndpoint).toSelf().inSingletonScope(); bind(BackendApplicationContribution).toService(FileDownloadEndpoint); - bind(FileDownloadCache).toSelf().inSingletonScope(); bind(FileDownloadHandler).to(SingleFileDownloadHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.SINGLE); bind(FileDownloadHandler).to(MultiFileDownloadHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.MULTI); - bind(FileDownloadHandler).to(DownloadLinkHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.DOWNLOAD_LINK); bind(DirectoryArchiver).toSelf().inSingletonScope(); }); diff --git a/packages/filesystem/src/node/download/file-download-cache.ts b/packages/filesystem/src/node/download/file-download-cache.ts deleted file mode 100644 index 09591f8aaf83b..0000000000000 --- a/packages/filesystem/src/node/download/file-download-cache.ts +++ /dev/null @@ -1,88 +0,0 @@ -/******************************************************************************** - * Copyright (C) 2019 Bitsler and others. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * This Source Code may also be made available under the following Secondary - * Licenses when the conditions for such availability set forth in the Eclipse - * Public License v. 2.0 are satisfied: GNU General Public License, version 2 - * with the GNU Classpath Exception which is available at - * https://www.gnu.org/software/classpath/license.html. - * - * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 - ********************************************************************************/ -import { injectable, inject } from 'inversify'; -import { ILogger } from '@theia/core/lib/common/logger'; -import * as rimraf from 'rimraf'; - -export interface DownloadStorageItem { - file: string; - root?: string; - size: number; - remove: boolean; - expire?: number; -} - -@injectable() -export class FileDownloadCache { - - @inject(ILogger) - protected readonly logger: ILogger; - protected readonly downloads = new Map(); - protected readonly expireTimeInMinutes: number = 1; - - addDownload(id: string, downloadInfo: DownloadStorageItem): void { - downloadInfo.file = encodeURIComponent(downloadInfo.file); - if (downloadInfo.root) { - downloadInfo.root = encodeURIComponent(downloadInfo.root); - } - // expires in 1 minute enough for parallel connections to be connected. - downloadInfo.expire = Date.now() + (this.expireTimeInMinutes * 600000); - this.downloads.set(id, downloadInfo); - } - - getDownload(id: string): DownloadStorageItem | undefined { - this.expireDownloads(); - const downloadInfo = this.downloads.get(id); - if (downloadInfo) { - - downloadInfo.file = decodeURIComponent(downloadInfo.file); - if (downloadInfo.root) { - downloadInfo.root = decodeURIComponent(downloadInfo.root); - } - } - return downloadInfo; - } - - deleteDownload(id: string): void { - const downloadInfo = this.downloads.get(id); - if (downloadInfo && downloadInfo.remove) { - this.deleteRecursively(downloadInfo.root || downloadInfo.file); - } - this.downloads.delete(id); - } - - values(): { [key: string]: DownloadStorageItem } { - this.expireDownloads(); - return [...this.downloads.entries()].reduce((downloads, [key, value]) => ({ ...downloads, [key]: value }), {}); - } - - protected deleteRecursively(pathToDelete: string): void { - rimraf(pathToDelete, error => { - if (error) { - this.logger.warn(`An error occurred while deleting the temporary data from the disk. Cannot clean up: ${pathToDelete}.`, error); - } - }); - } - - protected expireDownloads(): void { - const time = Date.now(); - for (const [id, download] of this.downloads.entries()) { - if (download.expire && download.expire <= time) { - this.deleteDownload(id); - } - } - } -} diff --git a/packages/filesystem/src/node/download/file-download-endpoint.ts b/packages/filesystem/src/node/download/file-download-endpoint.ts index 6b66620847687..4e8f6f728e407 100644 --- a/packages/filesystem/src/node/download/file-download-endpoint.ts +++ b/packages/filesystem/src/node/download/file-download-endpoint.ts @@ -36,13 +36,8 @@ export class FileDownloadEndpoint implements BackendApplicationContribution { @named(FileDownloadHandler.MULTI) protected readonly multiFileDownloadHandler: FileDownloadHandler; - @inject(FileDownloadHandler) - @named(FileDownloadHandler.DOWNLOAD_LINK) - protected readonly downloadLinkHandler: FileDownloadHandler; - configure(app: Application): void { const router = Router(); - router.get('/download', (request, response) => this.downloadLinkHandler.handle(request, response)); router.get('/', (request, response) => this.singleFileDownloadHandler.handle(request, response)); router.put('/', (request, response) => this.multiFileDownloadHandler.handle(request, response)); // Content-Type: application/json diff --git a/packages/filesystem/src/node/download/file-download-handler.ts b/packages/filesystem/src/node/download/file-download-handler.ts index 123e953168651..f812b6df71d66 100644 --- a/packages/filesystem/src/node/download/file-download-handler.ts +++ b/packages/filesystem/src/node/download/file-download-handler.ts @@ -17,10 +17,12 @@ import * as os from 'os'; import * as fs from 'fs-extra'; import * as path from 'path'; +import * as rimraf from 'rimraf'; import { v4 } from 'uuid'; +import { lookup } from 'mime-types'; import { Request, Response } from 'express'; import { inject, injectable } from 'inversify'; -import { OK, BAD_REQUEST, METHOD_NOT_ALLOWED, NOT_FOUND, INTERNAL_SERVER_ERROR, REQUESTED_RANGE_NOT_SATISFIABLE, PARTIAL_CONTENT } from 'http-status-codes'; +import { OK, BAD_REQUEST, METHOD_NOT_ALLOWED, NOT_FOUND, INTERNAL_SERVER_ERROR } from 'http-status-codes'; import URI from '@theia/core/lib/common/uri'; import { isEmpty } from '@theia/core/lib/common/objects'; import { ILogger } from '@theia/core/lib/common/logger'; @@ -28,14 +30,6 @@ import { FileUri } from '@theia/core/lib/node/file-uri'; import { FileSystem } from '../../common/filesystem'; import { DirectoryArchiver } from './directory-archiver'; import { FileDownloadData } from '../../common/download/file-download-data'; -import { FileDownloadCache, DownloadStorageItem } from './file-download-cache'; - -interface PrepareDownloadOptions { - filePath: string; - downloadId: string; - remove: boolean; - root?: string; -} @injectable() export abstract class FileDownloadHandler { @@ -49,103 +43,46 @@ export abstract class FileDownloadHandler { @inject(DirectoryArchiver) protected readonly directoryArchiver: DirectoryArchiver; - @inject(FileDownloadCache) - protected readonly fileDownloadCache: FileDownloadCache; - public abstract handle(request: Request, response: Response): Promise; - /** - * Prepares the file and the link for download - */ - protected async prepareDownload(request: Request, response: Response, options: PrepareDownloadOptions): Promise { - const name = path.basename(options.filePath); - try { - await fs.access(options.filePath, fs.constants.R_OK); - const stat = await fs.stat(options.filePath); - this.fileDownloadCache.addDownload(options.downloadId, { file: options.filePath, remove: options.remove, size: stat.size, root: options.root }); - // do not send filePath but instead use the downloadId - const data = { name, id: options.downloadId }; - response.status(OK).send(data).end(); - } catch (e) { - this.handleError(response, e, INTERNAL_SERVER_ERROR); + protected async download(filePath: string, request: Request, response: Response): Promise { + const name = path.basename(filePath); + const mimeType = lookup(filePath); + if (mimeType) { + response.contentType(mimeType); + } else { + this.logger.debug(`Cannot determine the content-type for file: ${filePath}. Skipping the 'Content-type' header from the HTTP response.`); } - } - - protected async download(request: Request, response: Response, downloadInfo: DownloadStorageItem, id: string): Promise { - const filePath = downloadInfo.file; - const statSize = downloadInfo.size; - // this sets the content-disposition and content-type automatically - response.attachment(filePath); + response.setHeader('Content-Disposition', `attachment; filename=${encodeURIComponent(name)}`); try { await fs.access(filePath, fs.constants.R_OK); - response.setHeader('Accept-Ranges', 'bytes'); - // parse range header and combine multiple ranges - const range = this.parseRangeHeader(request.headers['range'], statSize); - if (!range) { - response.setHeader('Content-Length', statSize); - this.streamDownload(OK, response, fs.createReadStream(filePath), id); - } else { - const rangeStart = range.start; - const rangeEnd = range.end; - if (rangeStart >= statSize || rangeEnd >= statSize) { - response.setHeader('Content-Range', `bytes */${statSize}`); - // Return the 416 'Requested Range Not Satisfiable'. - response.status(REQUESTED_RANGE_NOT_SATISFIABLE).end(); + fs.readFile(filePath, (error, data) => { + if (error) { + this.handleError(response, error, INTERNAL_SERVER_ERROR); return; } - response.setHeader('Content-Range', `bytes ${rangeStart}-${rangeEnd}/${statSize}`); - response.setHeader('Content-Length', rangeStart === rangeEnd ? 0 : (rangeEnd - rangeStart + 1)); - response.setHeader('Cache-Control', 'no-cache'); - this.streamDownload(PARTIAL_CONTENT, response, fs.createReadStream(filePath, { start: rangeStart, end: rangeEnd }), id); - } + response.status(OK).send(data).end(); + }); } catch (e) { - this.fileDownloadCache.deleteDownload(id); this.handleError(response, e, INTERNAL_SERVER_ERROR); } } - /** - * Streams the file and pipe it to the Response to avoid any OOM issues - */ - protected streamDownload(status: number, response: Response, stream: fs.ReadStream, id: string) { - response.status(status); - stream.on('error', error => { - this.fileDownloadCache.deleteDownload(id); - this.handleError(response, error, INTERNAL_SERVER_ERROR); - }); - response.on('error', error => { - this.fileDownloadCache.deleteDownload(id); - this.handleError(response, error, INTERNAL_SERVER_ERROR); - }); - response.on('close', () => { - stream.destroy(); - }); - stream.pipe(response); - } - protected parseRangeHeader(range: string | string[] | undefined, statSize: number): { start: number, end: number } | undefined { - if (!range || range.length === 0 || Array.isArray(range)) { - return; - } - const index = range.indexOf('='); - if (index === -1) { - return; - } - const rangeType = range.slice(0, index); - if (rangeType !== 'bytes') { - return; - } - const [start, end] = range.slice(index + 1).split('-').map(r => parseInt(r, 10)); - return { - start: isNaN(start) ? 0 : start, - end: (isNaN(end) || end > statSize - 1) ? (statSize - 1) : end - }; - } + protected async archive(inputPath: string, outputPath: string = path.join(os.tmpdir(), v4()), entries?: string[]): Promise { await this.directoryArchiver.archive(inputPath, outputPath, entries); return outputPath; } - protected async createTempDir(downloadId: string = v4()): Promise { - const outputPath = path.join(os.tmpdir(), downloadId); + protected async deleteRecursively(pathToDelete: string): Promise { + rimraf(pathToDelete, error => { + if (error) { + this.logger.warn(`An error occurred while deleting the temporary data from the disk. Cannot clean up: ${pathToDelete}.`, error); + } + }); + } + + protected async createTempDir(): Promise { + const outputPath = path.join(os.tmpdir(), v4()); await fs.mkdir(outputPath); return outputPath; } @@ -160,41 +97,6 @@ export abstract class FileDownloadHandler { export namespace FileDownloadHandler { export const SINGLE: symbol = Symbol('single'); export const MULTI: symbol = Symbol('multi'); - export const DOWNLOAD_LINK: symbol = Symbol('download'); -} - -@injectable() -export class DownloadLinkHandler extends FileDownloadHandler { - - async handle(request: Request, response: Response): Promise { - const { method, query } = request; - if (method !== 'GET' && method !== 'HEAD') { - this.handleError(response, `Unexpected HTTP method. Expected GET got '${method}' instead.`, METHOD_NOT_ALLOWED); - return; - } - if (query === undefined || query.id === undefined || typeof query.id !== 'string') { - this.handleError(response, `Cannot access the 'id' query from the request. The query was: ${JSON.stringify(query)}.`, BAD_REQUEST); - return; - } - const cancelDownload = query.cancel; - const downloadInfo = this.fileDownloadCache.getDownload(query.id); - if (!downloadInfo) { - this.handleError(response, `Cannot find the file from the request. The query was: ${JSON.stringify(query)}.`, NOT_FOUND); - return; - } - // allow head request to determine the content length for parallel downloaders - if (method === 'HEAD') { - response.setHeader('Content-Length', downloadInfo.size); - response.status(OK).end(); - return; - } - if (!cancelDownload) { - this.download(request, response, downloadInfo, query.id); - } else { - this.logger.info('Download', query.id, 'has been cancelled'); - this.fileDownloadCache.deleteDownload(query.id); - } - } } @injectable() @@ -221,19 +123,16 @@ export class SingleFileDownloadHandler extends FileDownloadHandler { return; } try { - const downloadId = v4(); const filePath = FileUri.fsPath(uri); - const options: PrepareDownloadOptions = { filePath, downloadId, remove: false }; if (!stat.isDirectory) { - await this.prepareDownload(request, response, options); + await this.download(filePath, request, response); } else { - const outputRootPath = await this.createTempDir(downloadId); + const outputRootPath = path.join(os.tmpdir(), v4()); + await fs.mkdir(outputRootPath); const outputPath = path.join(outputRootPath, `${path.basename(filePath)}.tar`); await this.archive(filePath, outputPath); - options.filePath = outputPath; - options.remove = true; - options.root = outputRootPath; - await this.prepareDownload(request, response, options); + await this.download(outputPath, request, response); + this.deleteRecursively(outputPath); } } catch (e) { this.handleError(response, e, INTERNAL_SERVER_ERROR); @@ -271,8 +170,8 @@ export class MultiFileDownloadHandler extends FileDownloadHandler { } } try { - const downloadId = v4(); - const outputRootPath = await this.createTempDir(downloadId); + const outputRootPath = path.join(os.tmpdir(), v4()); + await fs.mkdir(outputRootPath); const distinctUris = Array.from(new Set(body.uris.map(uri => new URI(uri)))); const tarPaths = []; // We should have one key in the map per FS drive. @@ -283,18 +182,18 @@ export class MultiFileDownloadHandler extends FileDownloadHandler { await this.archive(rootPath, outputPath, entries); tarPaths.push(outputPath); } - const options: PrepareDownloadOptions = { filePath: '', downloadId, remove: true, root: outputRootPath }; + if (tarPaths.length === 1) { // tslint:disable-next-line:whitespace const [outputPath,] = tarPaths; - options.filePath = outputPath; - await this.prepareDownload(request, response, options); + await this.download(outputPath, request, response); + this.deleteRecursively(outputRootPath); } else { // We need to tar the tars. const outputPath = path.join(outputRootPath, `theia-archive-${Date.now()}.tar`); - options.filePath = outputPath; await this.archive(outputRootPath, outputPath, tarPaths.map(p => path.relative(outputRootPath, p))); - await this.prepareDownload(request, response, options); + await this.download(outputPath, request, response); + this.deleteRecursively(outputRootPath); } } catch (e) { this.handleError(response, e, INTERNAL_SERVER_ERROR); diff --git a/packages/workspace/src/browser/workspace-commands.ts b/packages/workspace/src/browser/workspace-commands.ts index 02f6928261834..e6f41a541c251 100644 --- a/packages/workspace/src/browser/workspace-commands.ts +++ b/packages/workspace/src/browser/workspace-commands.ts @@ -152,7 +152,6 @@ export class FileMenuContribution implements MenuContribution { commandId: FileDownloadCommands.DOWNLOAD.id, order: 'b' }); - } }