diff --git a/packages/core/src/browser/style/tabs.css b/packages/core/src/browser/style/tabs.css index 23dcdaa6de351..c69f9cee9c04c 100644 --- a/packages/core/src/browser/style/tabs.css +++ b/packages/core/src/browser/style/tabs.css @@ -343,3 +343,7 @@ body.theia-editor-highlightModifiedTabs .p-TabBar-toolbar .item .refresh { background: var(--theia-icon-refresh) no-repeat; } + +.p-TabBar-toolbar .item .cancel { + background: var(--theia-icon-close) no-repeat; +} diff --git a/packages/search-in-workspace/src/browser/search-in-workspace-frontend-contribution.ts b/packages/search-in-workspace/src/browser/search-in-workspace-frontend-contribution.ts index 45a8a7dcb3851..2973e56d9154e 100644 --- a/packages/search-in-workspace/src/browser/search-in-workspace-frontend-contribution.ts +++ b/packages/search-in-workspace/src/browser/search-in-workspace-frontend-contribution.ts @@ -51,6 +51,12 @@ export namespace SearchInWorkspaceCommands { label: 'Refresh', iconClass: 'refresh' }; + export const CANCEL_SEARCH: Command = { + id: 'search-in-workspace.cancel', + category: SEARCH_CATEGORY, + label: 'Cancel Search', + iconClass: 'cancel' + }; export const COLLAPSE_ALL: Command = { id: 'search-in-workspace.collapse-all', category: SEARCH_CATEGORY, @@ -133,6 +139,11 @@ export class SearchInWorkspaceFrontendContribution extends AbstractViewContribut } })); + commands.registerCommand(SearchInWorkspaceCommands.CANCEL_SEARCH, { + execute: w => this.withWidget(w, widget => widget.getCancelIndicator() && widget.getCancelIndicator()!.cancel()), + isEnabled: w => this.withWidget(w, widget => widget.getCancelIndicator() !== undefined), + isVisible: w => this.withWidget(w, widget => widget.getCancelIndicator() !== undefined) + }); commands.registerCommand(SearchInWorkspaceCommands.REFRESH_RESULTS, { execute: w => this.withWidget(w, widget => widget.refresh()), isEnabled: w => this.withWidget(w, widget => (widget.hasResultList() || widget.hasSearchTerm()) && this.workspaceService.tryGetRoots().length > 0), @@ -201,25 +212,32 @@ export class SearchInWorkspaceFrontendContribution extends AbstractViewContribut async registerToolbarItems(toolbarRegistry: TabBarToolbarRegistry): Promise { const widget = await this.widget; const onDidChange = widget.onDidUpdate; + toolbarRegistry.registerItem({ + id: SearchInWorkspaceCommands.CANCEL_SEARCH.id, + command: SearchInWorkspaceCommands.CANCEL_SEARCH.id, + tooltip: SearchInWorkspaceCommands.CANCEL_SEARCH.label, + priority: 0, + onDidChange + }); toolbarRegistry.registerItem({ id: SearchInWorkspaceCommands.REFRESH_RESULTS.id, command: SearchInWorkspaceCommands.REFRESH_RESULTS.id, tooltip: SearchInWorkspaceCommands.REFRESH_RESULTS.label, - priority: 0, + priority: 1, onDidChange }); toolbarRegistry.registerItem({ id: SearchInWorkspaceCommands.CLEAR_ALL.id, command: SearchInWorkspaceCommands.CLEAR_ALL.id, tooltip: SearchInWorkspaceCommands.CLEAR_ALL.label, - priority: 1, + priority: 2, onDidChange }); toolbarRegistry.registerItem({ id: SearchInWorkspaceCommands.COLLAPSE_ALL.id, command: SearchInWorkspaceCommands.COLLAPSE_ALL.id, tooltip: SearchInWorkspaceCommands.COLLAPSE_ALL.label, - priority: 2, + priority: 3, onDidChange }); } diff --git a/packages/search-in-workspace/src/browser/search-in-workspace-result-tree-widget.tsx b/packages/search-in-workspace/src/browser/search-in-workspace-result-tree-widget.tsx index c479ac5ee21a5..b11f26cf96b37 100644 --- a/packages/search-in-workspace/src/browser/search-in-workspace-result-tree-widget.tsx +++ b/packages/search-in-workspace/src/browser/search-in-workspace-result-tree-widget.tsx @@ -34,7 +34,7 @@ import { CancellationTokenSource, Emitter, Event } from '@theia/core'; import { EditorManager, EditorDecoration, TrackedRangeStickiness, OverviewRulerLane, EditorWidget, ReplaceOperation, EditorOpenerOptions } from '@theia/editor/lib/browser'; import { WorkspaceService } from '@theia/workspace/lib/browser'; import { FileResourceResolver } from '@theia/filesystem/lib/browser'; -import { SearchInWorkspaceResult, SearchInWorkspaceOptions } from '../common/search-in-workspace-interface'; +import { SearchInWorkspaceResult, SearchInWorkspaceOptions, SearchMatch } from '../common/search-in-workspace-interface'; import { SearchInWorkspaceService } from './search-in-workspace-service'; import { MEMORY_TEXT } from './in-memory-text-resource'; import URI from '@theia/core/lib/common/uri'; @@ -83,7 +83,7 @@ export namespace SearchInWorkspaceFileNode { } } -export interface SearchInWorkspaceResultLineNode extends SelectableTreeNode, SearchInWorkspaceResult { // line node +export interface SearchInWorkspaceResultLineNode extends SelectableTreeNode, SearchInWorkspaceResult, SearchMatch { // line node parent: SearchInWorkspaceFileNode } export namespace SearchInWorkspaceResultLineNode { @@ -104,7 +104,7 @@ export class SearchInWorkspaceResultTreeWidget extends TreeWidget { protected appliedDecorations = new Map(); - private cancelIndicator = new CancellationTokenSource(); + cancelIndicator?: CancellationTokenSource; protected changeEmitter = new Emitter>(); // tslint:disable-next-line:no-any @@ -200,14 +200,22 @@ export class SearchInWorkspaceResultTreeWidget extends TreeWidget { this.searchTerm = searchTerm; const collapseValue: string = this.searchInWorkspacePreferences['search.collapseResults']; this.resultTree.clear(); - this.cancelIndicator.cancel(); - this.cancelIndicator = new CancellationTokenSource(); - const token = this.cancelIndicator.token; + if (this.cancelIndicator) { + this.cancelIndicator.cancel(); + } if (searchTerm === '') { this.refreshModelChildren(); return; } + this.cancelIndicator = new CancellationTokenSource(); + const cancelIndicator = this.cancelIndicator; + const token = this.cancelIndicator.token; + token.onCancellationRequested(() => { + this.changeEmitter.fire(this.resultTree); + }); const progress = await this.progressService.showProgress({ text: `search: ${searchTerm}`, options: { location: 'search' } }); + // tslint:disable-next-line:no-any + let pendingRefreshTimeout: any; const searchId = await this.searchService.search(searchTerm, { onResult: (aSearchId: number, result: SearchInWorkspaceResult) => { if (token.isCancellationRequested || aSearchId !== searchId) { @@ -215,38 +223,30 @@ export class SearchInWorkspaceResultTreeWidget extends TreeWidget { } const { path } = this.filenameAndPath(result.root, result.fileUri); const tree = this.resultTree; - const rootFolderNode = tree.get(result.root); - - if (rootFolderNode) { - const fileNode = rootFolderNode.children.find(f => f.fileUri === result.fileUri); - if (fileNode) { - const line = this.createResultLineNode(result, fileNode); - if (fileNode.children.findIndex(lineNode => lineNode.id === line.id) < 0) { - fileNode.children.push(line); - } - this.collapseFileNode(fileNode, collapseValue); - } else { - const newFileNode = this.createFileNode(result.root, path, result.fileUri, rootFolderNode); - this.collapseFileNode(newFileNode, collapseValue); - const line = this.createResultLineNode(result, newFileNode); - newFileNode.children.push(line); - rootFolderNode.children.push(newFileNode); + let rootFolderNode = tree.get(result.root); + if (!rootFolderNode) { + rootFolderNode = this.createRootFolderNode(result.root); + tree.set(result.root, rootFolderNode); + } + let fileNode = rootFolderNode.children.find(f => f.fileUri === result.fileUri); + if (!fileNode) { + fileNode = this.createFileNode(result.root, path, result.fileUri, rootFolderNode); + rootFolderNode.children.push(fileNode); + } + for (const match of result.matches) { + const line = this.createResultLineNode(result, match, fileNode); + if (fileNode.children.findIndex(lineNode => lineNode.id === line.id) < 0) { + fileNode.children.push(line); } - - } else { - const newRootFolderNode = this.createRootFolderNode(result.root); - tree.set(result.root, newRootFolderNode); - const newFileNode = this.createFileNode(result.root, path, result.fileUri, newRootFolderNode); - this.collapseFileNode(newFileNode, collapseValue); - newFileNode.children.push(this.createResultLineNode(result, newFileNode)); - newRootFolderNode.children.push(newFileNode); } + this.collapseFileNode(fileNode, collapseValue); + if (pendingRefreshTimeout) { + clearTimeout(pendingRefreshTimeout); + } + pendingRefreshTimeout = setTimeout(() => this.refreshModelChildren(), 100); }, onDone: () => { - progress.cancel(); - if (token.isCancellationRequested) { - return; - } + cancelIndicator.cancel(); // Sort the result map by folder URI. this.resultTree = new Map([...this.resultTree] .sort((a: [string, SearchInWorkspaceRootFolderNode], b: [string, SearchInWorkspaceRootFolderNode]) => this.compare(a[1].folderUri, b[1].folderUri))); @@ -263,6 +263,7 @@ export class SearchInWorkspaceResultTreeWidget extends TreeWidget { if (searchId) { this.searchService.cancel(searchId); } + this.cancelIndicator = undefined; }); } @@ -357,12 +358,13 @@ export class SearchInWorkspaceResultTreeWidget extends TreeWidget { }; } - protected createResultLineNode(result: SearchInWorkspaceResult, fileNode: SearchInWorkspaceFileNode): SearchInWorkspaceResultLineNode { + protected createResultLineNode(result: SearchInWorkspaceResult, match: SearchMatch, fileNode: SearchInWorkspaceFileNode): SearchInWorkspaceResultLineNode { return { ...result, + ...match, selected: false, - id: result.fileUri + '-' + result.line + '-' + result.character + '-' + result.length, - name: result.lineText, + id: result.fileUri + '-' + match.line + '-' + match.character + '-' + match.length, + name: typeof match.lineText === 'string' ? match.lineText : match.lineText.text, parent: fileNode }; } @@ -628,15 +630,27 @@ export class SearchInWorkspaceResultTreeWidget extends TreeWidget { } protected renderResultLineNode(node: SearchInWorkspaceResultLineNode): React.ReactNode { - const prefix = node.character > 26 ? '... ' : ''; - return
+ let before; + let after; + let title; + if (typeof node.lineText === 'string') { + const prefix = node.character > 26 ? '... ' : ''; + before = prefix + node.lineText.substr(0, node.character - 1).substr(-25); + after = node.lineText.substr(node.character - 1 + node.length, 75); + title = node.lineText.trim(); + } else { + before = node.lineText.text.substr(0, node.lineText.character); + after = node.lineText.text.substr(node.lineText.character + node.length); + title = node.lineText.text.trim(); + } + return
{this.searchInWorkspacePreferences['search.lineNumbers'] && {node.line}} - {prefix + node.lineText.substr(0, node.character - 1).substr(-25)} + {before} {this.renderMatchLinePart(node)} - {node.lineText.substr(node.character - 1 + node.length, 75)} + {after}
; } @@ -644,8 +658,11 @@ export class SearchInWorkspaceResultTreeWidget extends TreeWidget { protected renderMatchLinePart(node: SearchInWorkspaceResultLineNode): React.ReactNode { const replaceTerm = this._replaceTerm !== '' && this._showReplaceButtons ? {this._replaceTerm} : ''; const className = `match${this._showReplaceButtons ? ' strike-through' : ''}`; + const match = typeof node.lineText === 'string' ? + node.lineText.substr(node.character - 1, node.length) + : node.lineText.text.substr(node.lineText.character - 1, node.length); return - {node.lineText.substr(node.character - 1, node.length)} + {match} {replaceTerm} ; } diff --git a/packages/search-in-workspace/src/browser/search-in-workspace-widget.tsx b/packages/search-in-workspace/src/browser/search-in-workspace-widget.tsx index d6bf5fc003100..4c3389ea1f3c6 100644 --- a/packages/search-in-workspace/src/browser/search-in-workspace-widget.tsx +++ b/packages/search-in-workspace/src/browser/search-in-workspace-widget.tsx @@ -25,6 +25,7 @@ import { WorkspaceService } from '@theia/workspace/lib/browser'; import { SearchInWorkspaceContextKeyService } from './search-in-workspace-context-key-service'; import { ProgressLocationService } from '@theia/core/lib/browser/progress-location-service'; import { ProgressBar } from '@theia/core/lib/browser/progress-bar'; +import { CancellationTokenSource } from '@theia/core'; export interface SearchFieldState { className: string; @@ -217,6 +218,10 @@ export class SearchInWorkspaceWidget extends BaseWidget implements StatefulWidge this.update(); } + getCancelIndicator(): CancellationTokenSource | undefined { + return this.resultTreeWidget.cancelIndicator; + } + collapseAll(): void { this.resultTreeWidget.collapseAll(); this.update(); diff --git a/packages/search-in-workspace/src/common/search-in-workspace-interface.ts b/packages/search-in-workspace/src/common/search-in-workspace-interface.ts index 2a07a130d19b3..6447b11134bdd 100644 --- a/packages/search-in-workspace/src/common/search-in-workspace-interface.ts +++ b/packages/search-in-workspace/src/common/search-in-workspace-interface.ts @@ -21,6 +21,14 @@ export interface SearchInWorkspaceOptions { * Maximum number of results to return. Defaults to unlimited. */ maxResults?: number; + /** + * accepts suffixes of K, M or G which correspond to kilobytes, + * megabytes and gigabytes, respectively. If no suffix is provided the input is + * treated as bytes. + * + * defaults to '20M' + */ + maxFileSize?: string; /** * Search case sensitively if true. */ @@ -44,7 +52,7 @@ export interface SearchInWorkspaceOptions { /** * Glob pattern for matching files and directories to exclude the search. */ - exclude?: string[] + exclude?: string[]; } export interface SearchInWorkspaceResult { @@ -58,6 +66,13 @@ export interface SearchInWorkspaceResult { */ fileUri: string; + /** + * matches found in the file + */ + matches: SearchMatch[]; +} + +export interface SearchMatch { /** * The (1-based) line number of the result. */ @@ -78,7 +93,13 @@ export interface SearchInWorkspaceResult { /** * The text of the line containing the result. */ - lineText: string; + lineText: string | LinePreview; + +} + +export interface LinePreview { + text: string; + character: number; } export namespace SearchInWorkspaceResult { @@ -90,16 +111,7 @@ export namespace SearchInWorkspaceResult { if (a.fileUri !== b.fileUri) { return a.fileUri < b.fileUri ? -1 : 1; } - - if (a.line !== b.line) { - return a.line - b.line; - } - - if (a.character !== b.character) { - return a.character - b.character; - } - - return a.length - b.length; + return 0; } } diff --git a/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.slow-spec.ts b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.slow-spec.ts index b8fe1cd9c8fdd..ef4a022dfbcf8 100644 --- a/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.slow-spec.ts +++ b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.slow-spec.ts @@ -79,6 +79,7 @@ const getRootPathFromName = (name: string) => { potatoes: rootDirA, pastas: rootDirA, regexes: rootDirA, + 'long-line': rootDirA, small: `${rootDirA}/small`, 'file:with:some:colons': rootDirA, 'file with spaces': rootDirA, @@ -101,6 +102,14 @@ before(() => { rootDirBUri = FileUri.create(rootDirB).toString(); rootSubdirAUri = FileUri.create(rootSubdirA).toString(); + let contents = ''; + for (let x = 0; x < 10000; x++) { + const str = '' + x; + const pad = '00000'; + contents += ('long-' + pad.substring(0, pad.length - str.length) + str); + } + createTestFile('long-line', contents); + createTestFile('carrots', `\ This is a carrot. Most carrots are orange, but some carrots are not. @@ -197,6 +206,10 @@ after(() => { } }); +interface SearchInWorkspaceExpectation { + root: string, fileUri: string, line: number, character: number, length: number, lineText: string +} + // Compare expected and actual search results. // // For convenience, the expected entries do not have their lineText field set @@ -207,8 +220,9 @@ after(() => { // The expected entries should also have the file field set relatively to // rootDir. This function will update the field to contain the absolute path. -function compareSearchResults(expected: SearchInWorkspaceResult[], actual: SearchInWorkspaceResult[]): void { - expect(actual.length).eq(expected.length); +function compareSearchResults(expected: SearchInWorkspaceExpectation[], actual: SearchInWorkspaceResult[]): void { + const allMatches = actual.reduceRight((p, v) => p + v.matches.length, 0); + expect(allMatches).eq(expected.length); if (actual.length !== expected.length) { return; @@ -222,8 +236,15 @@ function compareSearchResults(expected: SearchInWorkspaceResult[], actual: Searc e.lineText = line; e.fileUri = FileUri.create(path.join(getRootPathFromName(e.fileUri), e.fileUri)).toString(); - const a = actual.find(l => l.fileUri === e.fileUri && l.line === e.line && l.character === e.character); - expect(a).deep.eq(e); + const a = actual.find(l => l.fileUri === e.fileUri)!; + const match = a.matches.find(m => m.line === e.line && m.character === e.character)!; + if (!match) { + console.log(a); + } + expect(match.length).eq(e.length); + if (typeof match.lineText === 'string') { + expect(match.lineText).eq(e.lineText); + } } else { // We don't know this file... expect.fail(); @@ -238,7 +259,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = ' pasta'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'pastas', line: 1, character: 6, length: pattern.length, lineText: '' }, ]; compareSearchResults(expected, client.results); @@ -252,7 +273,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'pasta '; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'pastas', line: 1, character: 1, length: pattern.length, lineText: '' }, ]; compareSearchResults(expected, client.results); @@ -267,7 +288,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'carrot'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'carrots', line: 1, character: 11, length: pattern.length, lineText: '' }, { root: rootDirAUri, fileUri: 'carrots', line: 2, character: 6, length: pattern.length, lineText: '' }, { root: rootDirAUri, fileUri: 'carrots', line: 2, character: 35, length: pattern.length, lineText: '' }, @@ -287,7 +308,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'carrot'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'carrots', line: 1, character: 11, length: pattern.length, lineText: '' }, { root: rootDirAUri, fileUri: 'carrots', line: 2, character: 6, length: pattern.length, lineText: '' }, { root: rootDirAUri, fileUri: 'carrots', line: 2, character: 35, length: pattern.length, lineText: '' }, @@ -308,7 +329,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'carrot'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'carrots', line: 1, character: 11, length: pattern.length, lineText: '' }, { root: rootDirAUri, fileUri: 'carrots', line: 3, character: 28, length: pattern.length, lineText: '' }, { root: rootDirAUri, fileUri: 'carrots', line: 3, character: 52, length: pattern.length, lineText: '' }, @@ -328,7 +349,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'carrot'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'carrots', line: 1, character: 11, length: pattern.length, lineText: '' }, { root: rootDirAUri, fileUri: 'carrots', line: 3, character: 28, length: pattern.length, lineText: '' } ]; @@ -345,7 +366,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { it('should return 1 result when searching for "Carrot"', done => { const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'carrots', line: 4, character: 1, length: 6, lineText: '' }, ]; @@ -384,7 +405,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'carrots are orange'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'carrots', line: 2, character: 6, length: pattern.length, lineText: '' }, ]; @@ -395,6 +416,27 @@ describe('ripgrep-search-in-workspace-server', function (): void { ripgrepServer.search(pattern, [rootDirAUri]); }); + it('should deal with very long lines."', done => { + const pattern = 'long-00400'; + + const client = new ResultAccumulator(() => { + const expected: SearchInWorkspaceExpectation[] = [ + { root: rootDirAUri, fileUri: 'long-line', line: 1, character: 4001, length: pattern.length, lineText: '' }, + ]; + + compareSearchResults(expected, client.results); + const match = client.results[0].matches[0]; + if (typeof match.lineText === 'string') { + done(new Error('unexpected lineText')); + } else { + expect(match.lineText.text.substr(match.lineText.character - 1, pattern.length)).eq(pattern); + done(); + } + }); + ripgrepServer.setClient(client); + ripgrepServer.search(pattern, [rootDirAUri]); + }); + // Try with an output size that exceeds the default node buffer size // (200 * 1024) when spawning a new process. it('should work with a lot of results', done => { @@ -403,9 +445,9 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'lots-of-matches'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = []; + const expected: SearchInWorkspaceExpectation[] = []; - for (let i = 1; i <= 100; i++) { + for (let i = 1; i <= 100000; i++) { expected.push({ root: rootDirAUri, fileUri: 'lots-of-matches', @@ -429,9 +471,9 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'lots-of-matches'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = []; + const expected: SearchInWorkspaceExpectation[] = []; - for (let i = 1; i <= 100; i++) { + for (let i = 1; i <= 1000; i++) { expected.push({ root: rootDirAUri, fileUri: 'lots-of-matches', @@ -457,7 +499,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'h[e3]l+[o0]'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'regexes', line: 1, character: 5, length: 5, lineText: '' }, { root: rootDirAUri, fileUri: 'regexes', line: 1, character: 14, length: 4, lineText: '' }, { root: rootDirAUri, fileUri: 'regexes', line: 1, character: 21, length: 5, lineText: '' }, @@ -479,7 +521,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'hello.'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'regexes', line: 1, character: 5, length: 6, lineText: '' } ]; @@ -497,7 +539,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = '-fo+bar'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'file with spaces', line: 1, character: 28, length: 7, lineText: '' }, ]; @@ -519,7 +561,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = '--fo+bar'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'file with spaces', line: 1, character: 27, length: 8, lineText: '' }, ]; @@ -540,7 +582,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = '-foobar'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'file with spaces', line: 1, character: 28, length: 7, lineText: '' }, ]; @@ -561,7 +603,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = '--foobar'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'file with spaces', line: 1, character: 27, length: 8, lineText: '' }, ]; @@ -582,7 +624,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = '-glob'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'glob', line: 1, character: 7, length: 5, lineText: '' }, { root: rootDirAUri, fileUri: 'glob.txt', line: 1, character: 6, length: 5, lineText: '' } ]; @@ -598,7 +640,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = '--glob'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'glob', line: 1, character: 6, length: 6, lineText: '' } ]; @@ -613,7 +655,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = '-glob'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'glob.txt', line: 1, character: 6, length: 5, lineText: '' } ]; @@ -628,7 +670,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = '-glob'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'glob.txt', line: 1, character: 6, length: 5, lineText: '' } ]; @@ -644,7 +686,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = ' jag'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'utf8-file', line: 1, character: 7, length: 4, lineText: '' }, { root: rootDirAUri, fileUri: 'utf8-file', line: 1, character: 23, length: 4, lineText: '' }, ]; @@ -661,7 +703,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = ' h?är'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'utf8-file', line: 1, character: 4, length: 3, lineText: '' }, { root: rootDirAUri, fileUri: 'utf8-file', line: 1, character: 20, length: 3, lineText: '' }, { root: rootDirAUri, fileUri: 'utf8-file', line: 1, character: 27, length: 4, lineText: '' }, @@ -681,7 +723,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = '(hello)?'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ ]; compareSearchResults(expected, client.results); @@ -695,7 +737,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'salut";\' echo foo && echo bar; "'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirAUri, fileUri: 'special shell characters', line: 1, character: 14, length: 32, lineText: '' }, ]; @@ -710,7 +752,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'carrot'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootDirBUri, fileUri: 'orange', line: 1, character: 51, length: pattern.length, lineText: '' }, { root: rootDirAUri, fileUri: 'carrots', line: 1, character: 11, length: pattern.length, lineText: '' }, { root: rootDirAUri, fileUri: 'carrots', line: 2, character: 6, length: pattern.length, lineText: '' }, @@ -732,7 +774,7 @@ describe('ripgrep-search-in-workspace-server', function (): void { const pattern = 'folder'; const client = new ResultAccumulator(() => { - const expected: SearchInWorkspaceResult[] = [ + const expected: SearchInWorkspaceExpectation[] = [ { root: rootSubdirAUri, fileUri: 'folderSubfolder', line: 1, character: 18, length: pattern.length, lineText: '' }, { root: rootSubdirAUri, fileUri: 'folderSubfolder', line: 1, character: 30, length: pattern.length, lineText: '' }, { root: rootSubdirAUri, fileUri: 'folderSubfolder', line: 1, character: 18, length: pattern.length, lineText: '' }, diff --git a/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts index 67b3f81466ecf..45689b171e919 100644 --- a/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts +++ b/packages/search-in-workspace/src/node/ripgrep-search-in-workspace-server.ts @@ -19,7 +19,7 @@ import { RawProcess, RawProcessFactory, RawProcessOptions } from '@theia/process import { FileUri } from '@theia/core/lib/node/file-uri'; import URI from '@theia/core/lib/common/uri'; import { inject, injectable } from 'inversify'; -import { SearchInWorkspaceServer, SearchInWorkspaceOptions, SearchInWorkspaceResult, SearchInWorkspaceClient } from '../common/search-in-workspace-interface'; +import { SearchInWorkspaceServer, SearchInWorkspaceOptions, SearchInWorkspaceResult, SearchInWorkspaceClient, LinePreview } from '../common/search-in-workspace-interface'; export const RgPath = Symbol('RgPath'); @@ -28,34 +28,46 @@ export const RgPath = Symbol('RgPath'); * * https://docs.rs/grep-printer/0.1.0/grep_printer/struct.JSON.html#object-arbitrary-data */ -interface RipGrepArbitraryData { - text?: string; - bytes?: string; +export type IRgBytesOrText = { bytes: string } | { text: string }; + +function bytesOrTextToString(obj: IRgBytesOrText): string { + return 'bytes' in obj ? + Buffer.from(obj.bytes, 'base64').toString() : + obj.text; } -/** - * Convert the length of a range in `text` expressed in bytes to a number of - * characters (or more precisely, code points). The range starts at character - * `charStart` in `text`. - */ -function byteRangeLengthToCharacterLength(text: string, charStart: number, byteLength: number): number { - let char: number = charStart; - for (let byteIdx = 0; byteIdx < byteLength; char++) { - const codePoint: number = text.charCodeAt(char); - if (codePoint < 0x7F) { - byteIdx++; - } else if (codePoint < 0x7FF) { - byteIdx += 2; - } else if (codePoint < 0xFFFF) { - byteIdx += 3; - } else if (codePoint < 0x10FFFF) { - byteIdx += 4; - } else { - throw new Error('Invalid UTF-8 string'); - } - } +type IRgMessage = IRgMatch | IRgBegin | IRgEnd; + +interface IRgMatch { + type: 'match'; + data: { + path: IRgBytesOrText; + lines: IRgBytesOrText; + line_number: number; + absolute_offset: number; + submatches: IRgSubmatch[]; + }; +} - return char - charStart; +export interface IRgSubmatch { + match: IRgBytesOrText; + start: number; + end: number; +} + +interface IRgBegin { + type: 'begin'; + data: { + path: IRgBytesOrText; + lines: string; + }; +} + +interface IRgEnd { + type: 'end'; + data: { + path: IRgBytesOrText; + }; } @injectable() @@ -65,7 +77,7 @@ export class RipgrepSearchInWorkspaceServer implements SearchInWorkspaceServer { private ongoingSearches: Map = new Map(); // Each incoming search is given a unique id, returned to the client. This is the next id we will assigned. - private nextSearchId: number = 0; + private nextSearchId: number = 1; private client: SearchInWorkspaceClient | undefined; @@ -82,10 +94,15 @@ export class RipgrepSearchInWorkspaceServer implements SearchInWorkspaceServer { } protected getArgs(options?: SearchInWorkspaceOptions): string[] { - const args = ['--json', '--max-count=100']; + const args = ['--hidden', '--json']; args.push(options && options.matchCase ? '--case-sensitive' : '--ignore-case'); if (options && options.includeIgnored) { - args.push('-uu'); + args.push('--no-ignore'); + } + if (options && options.maxFileSize) { + args.push('--max-filesize=' + options.maxFileSize.trim()); + } else { + args.push('--max-filesize=20M'); } if (options && options.include) { for (const include of options.include) { @@ -116,7 +133,7 @@ export class RipgrepSearchInWorkspaceServer implements SearchInWorkspaceServer { // line, --color=always to get color control characters that // we'll use to parse the lines. const searchId = this.nextSearchId++; - const args = this.getArgs(opts); + const rgArgs = this.getArgs(opts); // if we use matchWholeWord we use regExp internally, // so, we need to escape regexp characters if we actually not set regexp true in UI. if (opts && opts.matchWholeWord && !opts.useRegExp) { @@ -129,9 +146,10 @@ export class RipgrepSearchInWorkspaceServer implements SearchInWorkspaceServer { } } + const args = [...rgArgs, what].concat(rootUris.map(root => FileUri.fsPath(root))); const processOptions: RawProcessOptions = { command: this.rgPath, - args: [...args, what].concat(rootUris.map(root => FileUri.fsPath(root))) + args }; // TODO: Use child_process directly instead of rawProcessFactory? @@ -159,7 +177,9 @@ export class RipgrepSearchInWorkspaceServer implements SearchInWorkspaceServer { // Buffer to accumulate incoming output. let databuf: string = ''; - rgProcess.outputStream.on('data', (chunk: string) => { + let currentSearchResult: SearchInWorkspaceResult | undefined; + + rgProcess.outputStream.on('data', (chunk: Buffer) => { // We might have already reached the max number of // results, sent a TERM signal to rg, but we still get // the data that was already output in the mean time. @@ -183,41 +203,71 @@ export class RipgrepSearchInWorkspaceServer implements SearchInWorkspaceServer { const lineBuf = databuf.slice(0, eolIdx); databuf = databuf.slice(eolIdx + 1); - const obj = JSON.parse(lineBuf); - - if (obj['type'] === 'match') { - const data = obj['data']; - const file = (data['path']).text; - const line = data['line_number']; - const lineText = (data['lines']).text; + const obj = JSON.parse(lineBuf) as IRgMessage; + if (obj.type === 'begin') { + const file = bytesOrTextToString(obj.data.path); + if (file) { + currentSearchResult = { + fileUri: FileUri.create(file).toString(), + root: this.getRoot(file, rootUris).toString(), + matches: [] + }; + } else { + this.logger.error('Begin message without path. ' + JSON.stringify(obj)); + } + } else if (obj.type === 'end') { + if (currentSearchResult && this.client) { + this.client.onResult(searchId, currentSearchResult); + } + currentSearchResult = undefined; + } else if (obj.type === 'match') { + if (!currentSearchResult) { + continue; + } + const data = obj.data; + const file = bytesOrTextToString(data.path); + const line = data.line_number; + const lineText = bytesOrTextToString(data.lines); if (file === undefined || lineText === undefined) { continue; } - for (const submatch of data['submatches']) { - const startByte = submatch['start']; - const endByte = submatch['end']; - const character = byteRangeLengthToCharacterLength(lineText, 0, startByte); - const length = byteRangeLengthToCharacterLength(lineText, character, endByte - startByte); - - const result: SearchInWorkspaceResult = { - fileUri: FileUri.create(file).toString(), - root: this.getRoot(file, rootUris).toString(), + const lineInBytes = Buffer.from(lineText); + + for (const submatch of data.submatches) { + const startOffset = lineInBytes.slice(0, submatch.start).toString().length; + const match = bytesOrTextToString(submatch.match); + let lineInfo: string | LinePreview = lineText.trimRight(); + if (lineInfo.length > 300) { + const prefixLength = 25; + const start = Math.max(startOffset - prefixLength, 0); + const length = prefixLength + match.length + 70; + let prefix = ''; + if (start >= prefixLength) { + prefix = '...'; + } + const character = (start < prefixLength ? start : prefixLength) + prefix.length + 1; + lineInfo = { + text: prefix + lineInfo.substr(start, length), + character + }; + } + currentSearchResult.matches.push({ line, - character: character + 1, - length, - lineText: lineText.replace(/[\r\n]+$/, ''), - }; - + character: startOffset + 1, + length: match.length, + lineText: lineInfo + }); numResults++; - if (this.client) { - this.client.onResult(searchId, result); - } // Did we reach the maximum number of results? if (opts && opts.maxResults && numResults >= opts.maxResults) { rgProcess.kill(); + if (currentSearchResult && this.client) { + this.client.onResult(searchId, currentSearchResult); + } + currentSearchResult = undefined; this.wrapUpSearch(searchId); break; }