From 67ab7665689b706c742f501b3a776012de6a19e9 Mon Sep 17 00:00:00 2001 From: WinPlay02 Date: Tue, 23 Jan 2024 09:49:19 +0100 Subject: [PATCH] feat: pre-load all relevant workspace files before executing pipeline (#822) - relevant workspace files are preloaded and refreshed on each execution - `executePipeline` function now accepts a `LangiumDocument` instead of a pipeline path - fix: filling console causing unresponsiveness when handling new runner output by truncating too long output --------- Co-authored-by: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com> --- .../src/extension/mainClient.ts | 135 ++++++++++++---- .../src/extension/pythonServer.ts | 148 ++++++++---------- 2 files changed, 175 insertions(+), 108 deletions(-) diff --git a/packages/safe-ds-vscode/src/extension/mainClient.ts b/packages/safe-ds-vscode/src/extension/mainClient.ts index ee6fea602..6fa32c8d3 100644 --- a/packages/safe-ds-vscode/src/extension/mainClient.ts +++ b/packages/safe-ds-vscode/src/extension/mainClient.ts @@ -12,12 +12,12 @@ import { stopPythonServer, tryMapToSafeDSSource, } from './pythonServer.js'; -import { createSafeDsServicesWithBuiltins, SafeDsServices } from '@safe-ds/lang'; +import { ast, createSafeDsServicesWithBuiltins, SafeDsServices } from '@safe-ds/lang'; import { NodeFileSystem } from 'langium/node'; -import { getSafeDSOutputChannel, initializeLog, logOutput, printOutputMessage } from './output.js'; +import { getSafeDSOutputChannel, initializeLog, logError, logOutput, printOutputMessage } from './output.js'; import { createPlaceholderQueryMessage, RuntimeErrorMessage } from './messages.js'; import crypto from 'crypto'; -import { URI } from 'langium'; +import { LangiumDocument, URI } from 'langium'; let client: LanguageClient; let services: SafeDsServices; @@ -82,6 +82,14 @@ const startLanguageClient = function (context: vscode.ExtensionContext): Languag const acceptRunRequests = function (context: vscode.ExtensionContext) { // Register logging message callbacks + registerMessageLoggingCallbacks(); + // Register VS Code Entry Points + registerVSCodeCommands(context); + // Register watchers + registerVSCodeWatchers(); +}; + +const registerMessageLoggingCallbacks = function () { addMessageCallback((message) => { printOutputMessage( `Placeholder value is (${message.id}): ${message.data.name} of type ${message.data.type} = ${message.data.value}`, @@ -126,32 +134,105 @@ const acceptRunRequests = function (context: vscode.ExtensionContext) { .join('\n')}`, ); }, 'runtime_error'); - // Register VS Code Entry Points +}; + +const registerVSCodeCommands = function (context: vscode.ExtensionContext) { context.subscriptions.push( - vscode.commands.registerCommand('extension.safe-ds.runPipelineFile', (filePath: vscode.Uri | undefined) => { - let pipelinePath = filePath; - // Allow execution via command menu - if (!pipelinePath && vscode.window.activeTextEditor) { - pipelinePath = vscode.window.activeTextEditor.document.uri; - } - if ( - pipelinePath && - !services.LanguageMetaData.fileExtensions.some((extension: string) => - pipelinePath!.fsPath.endsWith(extension), - ) - ) { - vscode.window.showErrorMessage(`Could not run ${pipelinePath!.fsPath} as it is not a Safe-DS file`); - return; - } - if (!pipelinePath) { - vscode.window.showErrorMessage('Could not run Safe-DS Pipeline, as no pipeline is currently selected.'); - return; - } - const pipelineId = crypto.randomUUID(); - printOutputMessage(`Launching Pipeline (${pipelineId}): ${pipelinePath}`); - executePipeline(services, pipelinePath.fsPath, pipelineId); - }), + vscode.commands.registerCommand('extension.safe-ds.runPipelineFile', commandRunPipelineFile), + ); +}; + +const commandRunPipelineFile = async function (filePath: vscode.Uri | undefined) { + let pipelinePath = filePath; + // Allow execution via command menu + if (!pipelinePath && vscode.window.activeTextEditor) { + pipelinePath = vscode.window.activeTextEditor.document.uri; + } + if ( + pipelinePath && + !services.LanguageMetaData.fileExtensions.some((extension: string) => pipelinePath!.fsPath.endsWith(extension)) + ) { + vscode.window.showErrorMessage(`Could not run ${pipelinePath!.fsPath} as it is not a Safe-DS file`); + return; + } + if (!pipelinePath) { + vscode.window.showErrorMessage('Could not run Safe-DS Pipeline, as no pipeline is currently selected.'); + return; + } + // Refresh workspace + // Do not delete builtins + services.shared.workspace.LangiumDocuments.all + .filter( + (document) => + !( + ast.isSdsModule(document.parseResult.value) && + (document.parseResult.value).name === 'safeds.lang' + ), + ) + .forEach((oldDocument) => { + services.shared.workspace.LangiumDocuments.deleteDocument(oldDocument.uri); + }); + const workspaceSdsFiles = await vscode.workspace.findFiles('**/*.{sdspipe,sdsstub,sdstest}'); + // Load all documents + const unvalidatedSdsDocuments = workspaceSdsFiles.map((newDocumentUri) => + services.shared.workspace.LangiumDocuments.getOrCreateDocument(newDocumentUri), ); + // Validate them + const validationErrorMessage = await validateDocuments(services, unvalidatedSdsDocuments); + if (validationErrorMessage) { + vscode.window.showErrorMessage(validationErrorMessage); + return; + } + // Run it + const pipelineId = crypto.randomUUID(); + printOutputMessage(`Launching Pipeline (${pipelineId}): ${pipelinePath}`); + let mainDocument; + if (!services.shared.workspace.LangiumDocuments.hasDocument(pipelinePath)) { + mainDocument = services.shared.workspace.LangiumDocuments.getOrCreateDocument(pipelinePath); + const mainDocumentValidationErrorMessage = await validateDocuments(services, [mainDocument]); + if (mainDocumentValidationErrorMessage) { + vscode.window.showErrorMessage(mainDocumentValidationErrorMessage); + return; + } + } else { + mainDocument = services.shared.workspace.LangiumDocuments.getOrCreateDocument(pipelinePath); + } + await executePipeline(services, mainDocument, pipelineId); +}; + +const validateDocuments = async function ( + sdsServices: SafeDsServices, + documents: LangiumDocument[], +): Promise { + await sdsServices.shared.workspace.DocumentBuilder.build(documents, { validation: true }); + + const errors = documents.flatMap((validatedDocument) => { + const validationInfo = { + validatedDocument, + diagnostics: (validatedDocument.diagnostics ?? []).filter((e) => e.severity === 1), + }; + return validationInfo.diagnostics.length > 0 ? [validationInfo] : []; + }); + + if (errors.length > 0) { + for (const validationInfo of errors) { + logError(`File ${validationInfo.validatedDocument.uri.toString()} has errors:`); + for (const validationError of validationInfo.diagnostics) { + logError( + `\tat line ${validationError.range.start.line + 1}: ${ + validationError.message + } [${validationInfo.validatedDocument.textDocument.getText(validationError.range)}]`, + ); + } + } + return `As file(s) ${errors + .map((validationInfo) => validationInfo.validatedDocument.uri.toString()) + .join(', ')} has errors, the main pipeline cannot be run.`; + } + return undefined; +}; + +const registerVSCodeWatchers = function () { vscode.workspace.onDidChangeConfiguration((event) => { if (event.affectsConfiguration('safe-ds.runner.command')) { // Try starting runner diff --git a/packages/safe-ds-vscode/src/extension/pythonServer.ts b/packages/safe-ds-vscode/src/extension/pythonServer.ts index 3c9c63245..313b151e7 100644 --- a/packages/safe-ds-vscode/src/extension/pythonServer.ts +++ b/packages/safe-ds-vscode/src/extension/pythonServer.ts @@ -146,29 +146,6 @@ export const removeMessageCallback = function { - const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(documentUri); - await services.shared.workspace.DocumentBuilder.build([document], { validation: true }); - - const errors = (document.diagnostics ?? []).filter((e) => e.severity === 1); - - if (errors.length > 0) { - logError(`The file ${documentUri.toString()} has errors and cannot be run.`); - for (const validationError of errors) { - logError( - `\tat line ${validationError.range.start.line + 1}: ${ - validationError.message - } [${document.textDocument.getText(validationError.range)}]`, - ); - } - return undefined; - } - return document; -}; - /** * Context containing information about the execution of a pipeline. */ @@ -243,13 +220,13 @@ export const tryMapToSafeDSSource = async function ( * If a valid target placeholder is provided, the pipeline is only executed partially, to calculate the result of the placeholder. * * @param services SafeDsServices object, used to import the pipeline file. - * @param pipelinePath Path to a Safe-DS pipeline file to execute. + * @param pipelineDocument Document containing the main Safe-DS pipeline to execute. * @param id A unique id that is used in further communication with this pipeline. * @param targetPlaceholder The name of the target placeholder, used to do partial execution. If no value or undefined is provided, the entire pipeline is run. */ export const executePipeline = async function ( services: SafeDsServices, - pipelinePath: string, + pipelineDocument: LangiumDocument, id: string, targetPlaceholder: string | undefined = undefined, ) { @@ -261,67 +238,72 @@ export const executePipeline = async function ( return; } } - // TODO include all relevant files from workspace - const documentUri = URI.file(pipelinePath); - const workspaceRoot = path.dirname(documentUri.fsPath); // TODO get actual workspace root - services.shared.workspace.LangiumDocuments.deleteDocument(documentUri); - let document = await importPipeline(services, documentUri); - if (!document) { - vscode.window.showErrorMessage(`The file ${documentUri.fsPath} has errors and cannot be run.`); - return; - } - const lastExecutedSource = document.textDocument.getText(); - - const node = document.parseResult.value; - // - let mainPipelineName; - let mainModuleName; + const node = pipelineDocument.parseResult.value; if (!ast.isSdsModule(node)) { return; } + // Pipeline / Module name handling const mainPythonModuleName = services.builtins.Annotations.getPythonModule(node); - const mainPackage = mainPythonModuleName === undefined ? node?.name.split('.') : [mainPythonModuleName]; + const mainPackage = mainPythonModuleName === undefined ? node.name.split('.') : [mainPythonModuleName]; const firstPipeline = getModuleMembers(node).find(ast.isSdsPipeline); if (firstPipeline === undefined) { logError('Cannot execute: no pipeline found'); vscode.window.showErrorMessage('The current file cannot be executed, as no pipeline could be found.'); return; } - mainPipelineName = services.builtins.Annotations.getPythonName(firstPipeline) || firstPipeline.name; - if (pipelinePath.endsWith('.sdspipe')) { - mainModuleName = services.generation.PythonGenerator.sanitizeModuleNameForPython( - path.basename(pipelinePath, '.sdspipe'), - ); - } else if (pipelinePath.endsWith('.sdstest')) { - mainModuleName = services.generation.PythonGenerator.sanitizeModuleNameForPython( - path.basename(pipelinePath, '.sdstest'), - ); - } else { - mainModuleName = services.generation.PythonGenerator.sanitizeModuleNameForPython(path.basename(pipelinePath)); - } - // - const generatedDocuments = services.generation.PythonGenerator.generate(document, { - destination: URI.file(path.dirname(documentUri.fsPath)), // actual directory of main module file + const mainPipelineName = services.builtins.Annotations.getPythonName(firstPipeline) || firstPipeline.name; + const mainModuleName = getMainModuleName(services, pipelineDocument); + // Code generation + const [codeMap, lastGeneratedSources] = generateCodeForRunner(services, pipelineDocument, targetPlaceholder); + // Store information about the run + executionInformation.set(id, { + generatedSource: lastGeneratedSources, + sourceMappings: new Map(), + path: pipelineDocument.uri.fsPath, + source: pipelineDocument.textDocument.getText(), + calculatedPlaceholders: new Map(), + }); + // Code execution + sendMessageToPythonServer( + createProgramMessage(id, { + code: codeMap, + main: { + modulepath: mainPackage.join('.'), + module: mainModuleName, + pipeline: mainPipelineName, + }, + }), + ); +}; + +const generateCodeForRunner = function ( + services: SafeDsServices, + pipelineDocument: LangiumDocument, + targetPlaceholder: string | undefined, +): [ProgramCodeMap, Map] { + const rootGenerationDir = path.parse(pipelineDocument.uri.fsPath).dir; + const generatedDocuments = services.generation.PythonGenerator.generate(pipelineDocument, { + destination: URI.file(rootGenerationDir), // actual directory of main module file createSourceMaps: true, targetPlaceholder, }); - const lastGeneratedSource = new Map(); + const lastGeneratedSources = new Map(); let codeMap: ProgramCodeMap = {}; for (const generatedDocument of generatedDocuments) { const fsPath = URI.parse(generatedDocument.uri).fsPath; - const workspaceRelativeFilePath = path.relative(workspaceRoot, path.dirname(fsPath)); + const workspaceRelativeFilePath = path.relative(rootGenerationDir, path.dirname(fsPath)); const sdsFileName = path.basename(fsPath); const sdsNoExtFilename = path.extname(sdsFileName).length > 0 ? sdsFileName.substring(0, sdsFileName.length - path.extname(sdsFileName).length) : sdsFileName; - - lastGeneratedSource.set( + // Put code in map for further use in the extension (e.g. to remap errors) + lastGeneratedSources.set( path.join(workspaceRelativeFilePath, sdsFileName).replaceAll('\\', '/'), generatedDocument.getText(), ); // Check for sourcemaps after they are already added to the pipeline context - // This needs to happen after lastGeneratedSource.set, as errors would not get mapped otherwise + // This needs to happen after lastGeneratedSources.set, as errors would not get mapped otherwise if (fsPath.endsWith('.map')) { // exclude sourcemaps from sending to runner continue; @@ -330,26 +312,26 @@ export const executePipeline = async function ( if (!codeMap.hasOwnProperty(modulePath)) { codeMap[modulePath] = {}; } - const content = generatedDocument.getText(); - codeMap[modulePath]![sdsNoExtFilename] = content; + // Put code in object for runner + codeMap[modulePath]![sdsNoExtFilename] = generatedDocument.getText(); + } + return [codeMap, lastGeneratedSources]; +}; + +const getMainModuleName = function (services: SafeDsServices, pipelineDocument: LangiumDocument): string { + if (pipelineDocument.uri.fsPath.endsWith('.sdspipe')) { + return services.generation.PythonGenerator.sanitizeModuleNameForPython( + path.basename(pipelineDocument.uri.fsPath, '.sdspipe'), + ); + } else if (pipelineDocument.uri.fsPath.endsWith('.sdstest')) { + return services.generation.PythonGenerator.sanitizeModuleNameForPython( + path.basename(pipelineDocument.uri.fsPath, '.sdstest'), + ); + } else { + return services.generation.PythonGenerator.sanitizeModuleNameForPython( + path.basename(pipelineDocument.uri.fsPath), + ); } - executionInformation.set(id, { - generatedSource: lastGeneratedSource, - sourceMappings: new Map(), - path: pipelinePath, - source: lastExecutedSource, - calculatedPlaceholders: new Map(), - }); - sendMessageToPythonServer( - createProgramMessage(id, { - code: codeMap, - main: { - modulepath: mainPackage.join('.'), - module: mainModuleName, - pipeline: mainPipelineName, - }, - }), - ); }; /** @@ -435,7 +417,11 @@ const connectToWebSocket = async function (): Promise { logOutput(`[Runner] Message received: (${event.type}, ${typeof event.data}) ${event.data}`); return; } - logOutput(`[Runner] Message received: '${event.data}'`); + logOutput( + `[Runner] Message received: '${ + event.data.length > 128 ? event.data.substring(0, 128) + '' : event.data + }'`, + ); const pythonServerMessage: PythonServerMessage = JSON.parse(event.data); if (!pythonServerMessageCallbacks.has(pythonServerMessage.type)) { logOutput(`[Runner] Message type '${pythonServerMessage.type}' is not handled`);