Skip to content

Commit

Permalink
feat: pre-load all relevant workspace files before executing pipeline (…
Browse files Browse the repository at this point in the history 
…#822)

- relevant workspace files are preloaded and refreshed on each execution
- `executePipeline` function now accepts a `LangiumDocument` instead of
a pipeline path
- fix: filling console causing unresponsiveness when handling new runner
output by truncating too long output

---------

Co-authored-by: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com>
  • Loading branch information
@WinPlay02 @megalinter-bot
WinPlay02 and megalinter-bot authored Jan 23, 2024
1 parent 65aa400 commit 67ab766
Show file tree
Hide file tree
Showing 2 changed files with 175 additions and 108 deletions.
135 changes: 108 additions & 27 deletions packages/safe-ds-vscode/src/extension/mainClient.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ import {
stopPythonServer,
tryMapToSafeDSSource,
} from './pythonServer.js';
import { createSafeDsServicesWithBuiltins, SafeDsServices } from '@safe-ds/lang';
import { ast, createSafeDsServicesWithBuiltins, SafeDsServices } from '@safe-ds/lang';
import { NodeFileSystem } from 'langium/node';
import { getSafeDSOutputChannel, initializeLog, logOutput, printOutputMessage } from './output.js';
import { getSafeDSOutputChannel, initializeLog, logError, logOutput, printOutputMessage } from './output.js';
import { createPlaceholderQueryMessage, RuntimeErrorMessage } from './messages.js';
import crypto from 'crypto';
import { URI } from 'langium';
import { LangiumDocument, URI } from 'langium';

let client: LanguageClient;
let services: SafeDsServices;
Expand Down Expand Up @@ -82,6 +82,14 @@ const startLanguageClient = function (context: vscode.ExtensionContext): Languag

const acceptRunRequests = function (context: vscode.ExtensionContext) {
// Register logging message callbacks
registerMessageLoggingCallbacks();
// Register VS Code Entry Points
registerVSCodeCommands(context);
// Register watchers
registerVSCodeWatchers();
};

const registerMessageLoggingCallbacks = function () {
addMessageCallback((message) => {
printOutputMessage(
`Placeholder value is (${message.id}): ${message.data.name} of type ${message.data.type} = ${message.data.value}`,
Expand Down Expand Up @@ -126,32 +134,105 @@ const acceptRunRequests = function (context: vscode.ExtensionContext) {
.join('\n')}`,
);
}, 'runtime_error');
// Register VS Code Entry Points
};

const registerVSCodeCommands = function (context: vscode.ExtensionContext) {
context.subscriptions.push(
vscode.commands.registerCommand('extension.safe-ds.runPipelineFile', (filePath: vscode.Uri | undefined) => {
let pipelinePath = filePath;
// Allow execution via command menu
if (!pipelinePath && vscode.window.activeTextEditor) {
pipelinePath = vscode.window.activeTextEditor.document.uri;
}
if (
pipelinePath &&
!services.LanguageMetaData.fileExtensions.some((extension: string) =>
pipelinePath!.fsPath.endsWith(extension),
)
) {
vscode.window.showErrorMessage(`Could not run ${pipelinePath!.fsPath} as it is not a Safe-DS file`);
return;
}
if (!pipelinePath) {
vscode.window.showErrorMessage('Could not run Safe-DS Pipeline, as no pipeline is currently selected.');
return;
}
const pipelineId = crypto.randomUUID();
printOutputMessage(`Launching Pipeline (${pipelineId}): ${pipelinePath}`);
executePipeline(services, pipelinePath.fsPath, pipelineId);
}),
vscode.commands.registerCommand('extension.safe-ds.runPipelineFile', commandRunPipelineFile),
);
};

const commandRunPipelineFile = async function (filePath: vscode.Uri | undefined) {
let pipelinePath = filePath;
// Allow execution via command menu
if (!pipelinePath && vscode.window.activeTextEditor) {
pipelinePath = vscode.window.activeTextEditor.document.uri;
}
if (
pipelinePath &&
!services.LanguageMetaData.fileExtensions.some((extension: string) => pipelinePath!.fsPath.endsWith(extension))
) {
vscode.window.showErrorMessage(`Could not run ${pipelinePath!.fsPath} as it is not a Safe-DS file`);
return;
}
if (!pipelinePath) {
vscode.window.showErrorMessage('Could not run Safe-DS Pipeline, as no pipeline is currently selected.');
return;
}
// Refresh workspace
// Do not delete builtins
services.shared.workspace.LangiumDocuments.all
.filter(
(document) =>
!(
ast.isSdsModule(document.parseResult.value) &&
(<ast.SdsModule>document.parseResult.value).name === 'safeds.lang'
),
)
.forEach((oldDocument) => {
services.shared.workspace.LangiumDocuments.deleteDocument(oldDocument.uri);
});
const workspaceSdsFiles = await vscode.workspace.findFiles('**/*.{sdspipe,sdsstub,sdstest}');
// Load all documents
const unvalidatedSdsDocuments = workspaceSdsFiles.map((newDocumentUri) =>
services.shared.workspace.LangiumDocuments.getOrCreateDocument(newDocumentUri),
);
// Validate them
const validationErrorMessage = await validateDocuments(services, unvalidatedSdsDocuments);
if (validationErrorMessage) {
vscode.window.showErrorMessage(validationErrorMessage);
return;
}
// Run it
const pipelineId = crypto.randomUUID();
printOutputMessage(`Launching Pipeline (${pipelineId}): ${pipelinePath}`);
let mainDocument;
if (!services.shared.workspace.LangiumDocuments.hasDocument(pipelinePath)) {
mainDocument = services.shared.workspace.LangiumDocuments.getOrCreateDocument(pipelinePath);
const mainDocumentValidationErrorMessage = await validateDocuments(services, [mainDocument]);
if (mainDocumentValidationErrorMessage) {
vscode.window.showErrorMessage(mainDocumentValidationErrorMessage);
return;
}
} else {
mainDocument = services.shared.workspace.LangiumDocuments.getOrCreateDocument(pipelinePath);
}
await executePipeline(services, mainDocument, pipelineId);
};

const validateDocuments = async function (
sdsServices: SafeDsServices,
documents: LangiumDocument[],
): Promise<undefined | string> {
await sdsServices.shared.workspace.DocumentBuilder.build(documents, { validation: true });

const errors = documents.flatMap((validatedDocument) => {
const validationInfo = {
validatedDocument,
diagnostics: (validatedDocument.diagnostics ?? []).filter((e) => e.severity === 1),
};
return validationInfo.diagnostics.length > 0 ? [validationInfo] : [];
});

if (errors.length > 0) {
for (const validationInfo of errors) {
logError(`File ${validationInfo.validatedDocument.uri.toString()} has errors:`);
for (const validationError of validationInfo.diagnostics) {
logError(
`\tat line ${validationError.range.start.line + 1}: ${
validationError.message
} [${validationInfo.validatedDocument.textDocument.getText(validationError.range)}]`,
);
}
}
return `As file(s) ${errors
.map((validationInfo) => validationInfo.validatedDocument.uri.toString())
.join(', ')} has errors, the main pipeline cannot be run.`;
}
return undefined;
};

const registerVSCodeWatchers = function () {
vscode.workspace.onDidChangeConfiguration((event) => {
if (event.affectsConfiguration('safe-ds.runner.command')) {
// Try starting runner
Expand Down
148 changes: 67 additions & 81 deletions packages/safe-ds-vscode/src/extension/pythonServer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,29 +146,6 @@ export const removeMessageCallback = function <M extends PythonServerMessage['ty
);
};

const importPipeline = async function (
services: SafeDsServices,
documentUri: URI,
): Promise<LangiumDocument | undefined> {
const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(documentUri);
await services.shared.workspace.DocumentBuilder.build([document], { validation: true });

const errors = (document.diagnostics ?? []).filter((e) => e.severity === 1);

if (errors.length > 0) {
logError(`The file ${documentUri.toString()} has errors and cannot be run.`);
for (const validationError of errors) {
logError(
`\tat line ${validationError.range.start.line + 1}: ${
validationError.message
} [${document.textDocument.getText(validationError.range)}]`,
);
}
return undefined;
}
return document;
};

/**
* Context containing information about the execution of a pipeline.
*/
Expand Down Expand Up @@ -243,13 +220,13 @@ export const tryMapToSafeDSSource = async function (
* If a valid target placeholder is provided, the pipeline is only executed partially, to calculate the result of the placeholder.
*
* @param services SafeDsServices object, used to import the pipeline file.
* @param pipelinePath Path to a Safe-DS pipeline file to execute.
* @param pipelineDocument Document containing the main Safe-DS pipeline to execute.
* @param id A unique id that is used in further communication with this pipeline.
* @param targetPlaceholder The name of the target placeholder, used to do partial execution. If no value or undefined is provided, the entire pipeline is run.
*/
export const executePipeline = async function (
services: SafeDsServices,
pipelinePath: string,
pipelineDocument: LangiumDocument,
id: string,
targetPlaceholder: string | undefined = undefined,
) {
Expand All @@ -261,67 +238,72 @@ export const executePipeline = async function (
return;
}
}
// TODO include all relevant files from workspace
const documentUri = URI.file(pipelinePath);
const workspaceRoot = path.dirname(documentUri.fsPath); // TODO get actual workspace root
services.shared.workspace.LangiumDocuments.deleteDocument(documentUri);
let document = await importPipeline(services, documentUri);
if (!document) {
vscode.window.showErrorMessage(`The file ${documentUri.fsPath} has errors and cannot be run.`);
return;
}
const lastExecutedSource = document.textDocument.getText();

const node = document.parseResult.value;
//
let mainPipelineName;
let mainModuleName;
const node = pipelineDocument.parseResult.value;
if (!ast.isSdsModule(node)) {
return;
}
// Pipeline / Module name handling
const mainPythonModuleName = services.builtins.Annotations.getPythonModule(node);
const mainPackage = mainPythonModuleName === undefined ? node?.name.split('.') : [mainPythonModuleName];
const mainPackage = mainPythonModuleName === undefined ? node.name.split('.') : [mainPythonModuleName];
const firstPipeline = getModuleMembers(node).find(ast.isSdsPipeline);
if (firstPipeline === undefined) {
logError('Cannot execute: no pipeline found');
vscode.window.showErrorMessage('The current file cannot be executed, as no pipeline could be found.');
return;
}
mainPipelineName = services.builtins.Annotations.getPythonName(firstPipeline) || firstPipeline.name;
if (pipelinePath.endsWith('.sdspipe')) {
mainModuleName = services.generation.PythonGenerator.sanitizeModuleNameForPython(
path.basename(pipelinePath, '.sdspipe'),
);
} else if (pipelinePath.endsWith('.sdstest')) {
mainModuleName = services.generation.PythonGenerator.sanitizeModuleNameForPython(
path.basename(pipelinePath, '.sdstest'),
);
} else {
mainModuleName = services.generation.PythonGenerator.sanitizeModuleNameForPython(path.basename(pipelinePath));
}
//
const generatedDocuments = services.generation.PythonGenerator.generate(document, {
destination: URI.file(path.dirname(documentUri.fsPath)), // actual directory of main module file
const mainPipelineName = services.builtins.Annotations.getPythonName(firstPipeline) || firstPipeline.name;
const mainModuleName = getMainModuleName(services, pipelineDocument);
// Code generation
const [codeMap, lastGeneratedSources] = generateCodeForRunner(services, pipelineDocument, targetPlaceholder);
// Store information about the run
executionInformation.set(id, {
generatedSource: lastGeneratedSources,
sourceMappings: new Map<string, BasicSourceMapConsumer>(),
path: pipelineDocument.uri.fsPath,
source: pipelineDocument.textDocument.getText(),
calculatedPlaceholders: new Map<string, string>(),
});
// Code execution
sendMessageToPythonServer(
createProgramMessage(id, {
code: codeMap,
main: {
modulepath: mainPackage.join('.'),
module: mainModuleName,
pipeline: mainPipelineName,
},
}),
);
};

const generateCodeForRunner = function (
services: SafeDsServices,
pipelineDocument: LangiumDocument,
targetPlaceholder: string | undefined,
): [ProgramCodeMap, Map<string, string>] {
const rootGenerationDir = path.parse(pipelineDocument.uri.fsPath).dir;
const generatedDocuments = services.generation.PythonGenerator.generate(pipelineDocument, {
destination: URI.file(rootGenerationDir), // actual directory of main module file
createSourceMaps: true,
targetPlaceholder,
});
const lastGeneratedSource = new Map<string, string>();
const lastGeneratedSources = new Map<string, string>();
let codeMap: ProgramCodeMap = {};
for (const generatedDocument of generatedDocuments) {
const fsPath = URI.parse(generatedDocument.uri).fsPath;
const workspaceRelativeFilePath = path.relative(workspaceRoot, path.dirname(fsPath));
const workspaceRelativeFilePath = path.relative(rootGenerationDir, path.dirname(fsPath));
const sdsFileName = path.basename(fsPath);
const sdsNoExtFilename =
path.extname(sdsFileName).length > 0
? sdsFileName.substring(0, sdsFileName.length - path.extname(sdsFileName).length)
: sdsFileName;

lastGeneratedSource.set(
// Put code in map for further use in the extension (e.g. to remap errors)
lastGeneratedSources.set(
path.join(workspaceRelativeFilePath, sdsFileName).replaceAll('\\', '/'),
generatedDocument.getText(),
);
// Check for sourcemaps after they are already added to the pipeline context
// This needs to happen after lastGeneratedSource.set, as errors would not get mapped otherwise
// This needs to happen after lastGeneratedSources.set, as errors would not get mapped otherwise
if (fsPath.endsWith('.map')) {
// exclude sourcemaps from sending to runner
continue;
Expand All @@ -330,26 +312,26 @@ export const executePipeline = async function (
if (!codeMap.hasOwnProperty(modulePath)) {
codeMap[modulePath] = {};
}
const content = generatedDocument.getText();
codeMap[modulePath]![sdsNoExtFilename] = content;
// Put code in object for runner
codeMap[modulePath]![sdsNoExtFilename] = generatedDocument.getText();
}
return [codeMap, lastGeneratedSources];
};

const getMainModuleName = function (services: SafeDsServices, pipelineDocument: LangiumDocument): string {
if (pipelineDocument.uri.fsPath.endsWith('.sdspipe')) {
return services.generation.PythonGenerator.sanitizeModuleNameForPython(
path.basename(pipelineDocument.uri.fsPath, '.sdspipe'),
);
} else if (pipelineDocument.uri.fsPath.endsWith('.sdstest')) {
return services.generation.PythonGenerator.sanitizeModuleNameForPython(
path.basename(pipelineDocument.uri.fsPath, '.sdstest'),
);
} else {
return services.generation.PythonGenerator.sanitizeModuleNameForPython(
path.basename(pipelineDocument.uri.fsPath),
);
}
executionInformation.set(id, {
generatedSource: lastGeneratedSource,
sourceMappings: new Map<string, BasicSourceMapConsumer>(),
path: pipelinePath,
source: lastExecutedSource,
calculatedPlaceholders: new Map<string, string>(),
});
sendMessageToPythonServer(
createProgramMessage(id, {
code: codeMap,
main: {
modulepath: mainPackage.join('.'),
module: mainModuleName,
pipeline: mainPipelineName,
},
}),
);
};

/**
Expand Down Expand Up @@ -435,7 +417,11 @@ const connectToWebSocket = async function (): Promise<void> {
logOutput(`[Runner] Message received: (${event.type}, ${typeof event.data}) ${event.data}`);
return;
}
logOutput(`[Runner] Message received: '${event.data}'`);
logOutput(
`[Runner] Message received: '${
event.data.length > 128 ? event.data.substring(0, 128) + '<truncated>' : event.data
}'`,
);
const pythonServerMessage: PythonServerMessage = JSON.parse(<string>event.data);
if (!pythonServerMessageCallbacks.has(pythonServerMessage.type)) {
logOutput(`[Runner] Message type '${pythonServerMessage.type}' is not handled`);
Expand Down

0 comments on commit 67ab766

Please sign in to comment.