Skip to content

Commit

Permalink
feat: partial code generation for multiple targets (#1114)
Browse files Browse the repository at this point in the history
Closes partially #1079

### Summary of Changes

Partial code generation now works for multiple targets instead of just
one.
  • Loading branch information
lars-reimann authored Apr 27, 2024
1 parent b6d4f16 commit 5461a1b
Show file tree
Hide file tree
Showing 16 changed files with 292 additions and 111 deletions.
2 changes: 1 addition & 1 deletion docs/development/testing/generation-testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ document explains how to add a new generation test.
placeholder with test markers, e.g. `val »a« = 1;`. You may only mark a single placeholder this way. Add a comment in
the preceding line with the following format:
```ts
// $TEST$ run_until
// $TEST$ target
```
5. Add another folder called `generated` inside the folder that you created in step 1. Place folders and Python files
inside the `generated` folder to specify the expected output of the program. The relative paths to the Python files
Expand Down
2 changes: 1 addition & 1 deletion packages/safe-ds-cli/src/cli/generate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export const generate = async (fsPaths: string[], options: GenerateOptions): Pro
const generatedFiles = services.generation.PythonGenerator.generate(document, {
destination: URI.file(path.resolve(options.out)),
createSourceMaps: options.sourcemaps,
targetPlaceholder: undefined,
targetPlaceholders: undefined,
disableRunnerIntegration: false,
});

Expand Down
95 changes: 95 additions & 0 deletions packages/safe-ds-lang/src/language/flow/safe-ds-slicer.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import { SafeDsServices } from '../safe-ds-module.js';
import { isSdsAssignment, isSdsPlaceholder, isSdsReference, SdsPlaceholder, SdsStatement } from '../generated/ast.js';
import { AstUtils, Stream } from 'langium';
import { ImpurityReason } from '../purity/model.js';
import { getAssignees } from '../helpers/nodeProperties.js';
import { SafeDsPurityComputer } from '../purity/safe-ds-purity-computer.js';

export class SafeDsSlicer {
private readonly purityComputer: SafeDsPurityComputer;

constructor(services: SafeDsServices) {
this.purityComputer = services.purity.PurityComputer;
}

/**
* Computes the subset of the given statements that are needed to calculate the target placeholders.
*/
computeBackwardSlice(statements: SdsStatement[], targets: SdsPlaceholder[]): SdsStatement[] {
const aggregator = new BackwardSliceAggregator(this.purityComputer, targets);

for (const statement of statements.reverse()) {
// Keep if it declares a target
if (
isSdsAssignment(statement) &&
getAssignees(statement).some((it) => isSdsPlaceholder(it) && aggregator.targets.has(it))
) {
aggregator.addStatement(statement);
}

// Keep if it has an impurity reason that affects a future impurity reason
else if (
this.purityComputer
.getImpurityReasonsForStatement(statement)
.some((pastReason) =>
aggregator.impurityReasons.some((futureReason) =>
pastReason.canAffectFutureImpurityReason(futureReason),
),
)
) {
aggregator.addStatement(statement);
}
}

return aggregator.statements;
}
}

class BackwardSliceAggregator {
private readonly purityComputer: SafeDsPurityComputer;

/**
* The statements that are needed to calculate the target placeholders.
*/
readonly statements: SdsStatement[] = [];

/**
* The target placeholders that should be calculated.
*/
readonly targets: Set<SdsPlaceholder>;

/**
* The impurity reasons of the collected statements.
*/
readonly impurityReasons: ImpurityReason[] = [];

constructor(purityComputer: SafeDsPurityComputer, initialTargets: SdsPlaceholder[]) {
this.purityComputer = purityComputer;

this.targets = new Set(initialTargets);
}

addStatement(statement: SdsStatement): void {
this.statements.unshift(statement);

// Remember all referenced placeholders
this.getReferencedPlaceholders(statement).forEach((it) => {
this.targets.add(it);
});

// Remember all impurity reasons
this.purityComputer.getImpurityReasonsForStatement(statement).forEach((it) => {
this.impurityReasons.push(it);
});
}

private getReferencedPlaceholders(node: SdsStatement): Stream<SdsPlaceholder> {
return AstUtils.streamAllContents(node).flatMap((it) => {
if (isSdsReference(it) && isSdsPlaceholder(it.target.ref)) {
return [it.target.ref];
} else {
return [];
}
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ import {
SdsParameter,
SdsParameterList,
SdsPipeline,
SdsPlaceholder,
SdsReference,
SdsSegment,
SdsStatement,
Expand Down Expand Up @@ -100,7 +99,7 @@ import {
import { SafeDsPartialEvaluator } from '../../partialEvaluation/safe-ds-partial-evaluator.js';
import { SafeDsServices } from '../../safe-ds-module.js';
import { SafeDsPurityComputer } from '../../purity/safe-ds-purity-computer.js';
import { FileRead, ImpurityReason } from '../../purity/model.js';
import { FileRead } from '../../purity/model.js';
import { SafeDsTypeComputer } from '../../typing/safe-ds-type-computer.js';
import { NamedTupleType } from '../../typing/model.js';
import { getOutermostContainerOfType } from '../../helpers/astUtils.js';
Expand All @@ -114,6 +113,7 @@ import {
UtilityFunction,
} from './utilityFunctions.js';
import { CODEGEN_PREFIX } from './constants.js';
import { SafeDsSlicer } from '../../flow/safe-ds-slicer.js';

const LAMBDA_PREFIX = `${CODEGEN_PREFIX}lambda_`;
const BLOCK_LAMBDA_RESULT_PREFIX = `${CODEGEN_PREFIX}block_lambda_result_`;
Expand All @@ -132,13 +132,15 @@ export class SafeDsPythonGenerator {
private readonly nodeMapper: SafeDsNodeMapper;
private readonly partialEvaluator: SafeDsPartialEvaluator;
private readonly purityComputer: SafeDsPurityComputer;
private readonly slicer: SafeDsSlicer;
private readonly typeComputer: SafeDsTypeComputer;

constructor(services: SafeDsServices) {
this.builtinAnnotations = services.builtins.Annotations;
this.nodeMapper = services.helpers.NodeMapper;
this.partialEvaluator = services.evaluation.PartialEvaluator;
this.purityComputer = services.purity.PurityComputer;
this.slicer = services.flow.Slicer;
this.typeComputer = services.typing.TypeComputer;
}

Expand Down Expand Up @@ -412,7 +414,7 @@ export class SafeDsPythonGenerator {
utilitySet,
typeVariableSet,
true,
generateOptions.targetPlaceholder,
generateOptions.targetPlaceholders,
generateOptions.disableRunnerIntegration,
);
return expandTracedToNode(pipeline)`def ${traceToNode(
Expand Down Expand Up @@ -464,10 +466,12 @@ export class SafeDsPythonGenerator {
frame: GenerationInfoFrame,
generateLambda: boolean = false,
): CompositeGeneratorNode {
const targetPlaceholder = getPlaceholderByName(block, frame.targetPlaceholder);
let statements = getStatements(block).filter((stmt) => this.purityComputer.statementDoesSomething(stmt));
if (targetPlaceholder) {
statements = this.getStatementsNeededForPartialExecution(targetPlaceholder, statements);
if (frame.targetPlaceholders) {
const targetPlaceholders = frame.targetPlaceholders.flatMap((it) => getPlaceholderByName(block, it) ?? []);
if (!isEmpty(targetPlaceholders)) {
statements = this.slicer.computeBackwardSlice(statements, targetPlaceholders);
}
}
if (statements.length === 0) {
return traceToNode(block)('pass');
Expand All @@ -480,68 +484,6 @@ export class SafeDsPythonGenerator {
},
)!;
}

private getStatementsNeededForPartialExecution(
targetPlaceholder: SdsPlaceholder,
statementsWithEffect: SdsStatement[],
): SdsStatement[] {
// Find assignment of placeholder, to search used placeholders and impure dependencies
const assignment = AstUtils.getContainerOfType(targetPlaceholder, isSdsAssignment);
if (!assignment || !assignment.expression) {
/* c8 ignore next 2 */
throw new Error(`No assignment for placeholder: ${targetPlaceholder.name}`);
}
// All collected referenced placeholders that are needed for calculating the target placeholder. An expression in the assignment will always exist here
const referencedPlaceholders = new Set<SdsPlaceholder>(
AstUtils.streamAllContents(assignment.expression!)
.filter(isSdsReference)
.filter((reference) => isSdsPlaceholder(reference.target.ref))
.map((reference) => <SdsPlaceholder>reference.target.ref!)
.toArray(),
);
const impurityReasons = new Set<ImpurityReason>(this.purityComputer.getImpurityReasonsForStatement(assignment));
const collectedStatements: SdsStatement[] = [assignment];
for (const prevStatement of statementsWithEffect.reverse()) {
// Statements after the target assignment can always be skipped
if (prevStatement.$containerIndex! >= assignment.$containerIndex!) {
continue;
}
const prevStmtImpurityReasons: ImpurityReason[] =
this.purityComputer.getImpurityReasonsForStatement(prevStatement);
if (
// Placeholder is relevant
(isSdsAssignment(prevStatement) &&
getAssignees(prevStatement)
.filter(isSdsPlaceholder)
.some((prevPlaceholder) => referencedPlaceholders.has(prevPlaceholder))) ||
// Impurity is relevant
prevStmtImpurityReasons.some((pastReason) =>
Array.from(impurityReasons).some((futureReason) =>
pastReason.canAffectFutureImpurityReason(futureReason),
),
)
) {
collectedStatements.push(prevStatement);
// Collect all referenced placeholders
if (isSdsExpressionStatement(prevStatement) || isSdsAssignment(prevStatement)) {
AstUtils.streamAllContents(prevStatement.expression!)
.filter(isSdsReference)
.filter((reference) => isSdsPlaceholder(reference.target.ref))
.map((reference) => <SdsPlaceholder>reference.target.ref!)
.forEach((prevPlaceholder) => {
referencedPlaceholders.add(prevPlaceholder);
});
}
// Collect impurity reasons
prevStmtImpurityReasons.forEach((prevReason) => {
impurityReasons.add(prevReason);
});
}
}
// Get all statements in sorted order
return collectedStatements.reverse();
}

private generateStatement(statement: SdsStatement, frame: GenerationInfoFrame, generateLambda: boolean): Generated {
const result: Generated[] = [];

Expand Down Expand Up @@ -1251,7 +1193,7 @@ class GenerationInfoFrame {
private readonly utilitySet: Set<UtilityFunction>;
private readonly typeVariableSet: Set<string>;
public readonly isInsidePipeline: boolean;
public readonly targetPlaceholder: string | undefined;
public readonly targetPlaceholders: string[] | undefined;
public readonly disableRunnerIntegration: boolean;
private extraStatements = new Map<SdsExpression, Generated>();

Expand All @@ -1260,15 +1202,15 @@ class GenerationInfoFrame {
utilitySet: Set<UtilityFunction> = new Set<UtilityFunction>(),
typeVariableSet: Set<string> = new Set<string>(),
insidePipeline: boolean = false,
targetPlaceholder: string | undefined = undefined,
targetPlaceholders: string[] | undefined = undefined,
disableRunnerIntegration: boolean = false,
) {
this.idManager = new IdManager();
this.importSet = importSet;
this.utilitySet = utilitySet;
this.typeVariableSet = typeVariableSet;
this.isInsidePipeline = insidePipeline;
this.targetPlaceholder = targetPlaceholder;
this.targetPlaceholders = targetPlaceholders;
this.disableRunnerIntegration = disableRunnerIntegration;
}

Expand Down Expand Up @@ -1325,6 +1267,6 @@ class GenerationInfoFrame {
export interface GenerateOptions {
destination: URI;
createSourceMaps: boolean;
targetPlaceholder: string | undefined;
targetPlaceholders: string[] | undefined;
disableRunnerIntegration: boolean;
}
1 change: 1 addition & 0 deletions packages/safe-ds-lang/src/language/purity/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export abstract class ImpurityReason {

/**
* Returns whether this impurity reason can affect a future impurity reason.
*
* @param future Future Impurity reason to test, if this reason may have an effect on it.
*/
abstract canAffectFutureImpurityReason(future: ImpurityReason): boolean;
Expand Down
14 changes: 7 additions & 7 deletions packages/safe-ds-lang/src/language/runtime/safe-ds-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ export class SafeDsRunner {
}),
];

await this.executePipeline(pipelineExecutionId, document, pipeline.name, placeholder.name);
await this.executePipeline(pipelineExecutionId, document, pipeline.name, [placeholder.name]);
}

async showImage(documentUri: string, nodePath: string) {
Expand Down Expand Up @@ -252,7 +252,7 @@ export class SafeDsRunner {
}),
];

await this.executePipeline(pipelineExecutionId, document, pipeline.name, placeholder.name);
await this.executePipeline(pipelineExecutionId, document, pipeline.name, [placeholder.name]);
}

private async getPlaceholderValue(placeholder: string, pipelineExecutionId: string): Promise<any | undefined> {
Expand Down Expand Up @@ -322,13 +322,13 @@ export class SafeDsRunner {
* @param id A unique id that is used in further communication with this pipeline.
* @param pipelineDocument Document containing the main Safe-DS pipeline to execute.
* @param pipelineName Name of the pipeline that should be run
* @param targetPlaceholder The name of the target placeholder, used to do partial execution. If no value or undefined is provided, the entire pipeline is run.
* @param targetPlaceholders The names of the target placeholders, used to do partial execution. If undefined is provided, the entire pipeline is run.
*/
public async executePipeline(
id: string,
pipelineDocument: LangiumDocument,
pipelineName: string,
targetPlaceholder: string | undefined = undefined,
targetPlaceholders: string[] | undefined = undefined,
) {
const node = pipelineDocument.parseResult.value;
if (!isSdsModule(node)) {
Expand All @@ -339,7 +339,7 @@ export class SafeDsRunner {
const mainPackage = mainPythonModuleName === undefined ? node.name.split('.') : [mainPythonModuleName];
const mainModuleName = this.getMainModuleName(pipelineDocument);
// Code generation
const [codeMap, lastGeneratedSources] = this.generateCodeForRunner(pipelineDocument, targetPlaceholder);
const [codeMap, lastGeneratedSources] = this.generateCodeForRunner(pipelineDocument, targetPlaceholders);
// Store information about the run
this.executionInformation.set(id, {
generatedSource: lastGeneratedSources,
Expand Down Expand Up @@ -466,13 +466,13 @@ export class SafeDsRunner {

public generateCodeForRunner(
pipelineDocument: LangiumDocument,
targetPlaceholder: string | undefined,
targetPlaceholders: string[] | undefined,
): [ProgramCodeMap, Map<string, string>] {
const rootGenerationDir = path.parse(pipelineDocument.uri.fsPath).dir;
const generatedDocuments = this.generator.generate(pipelineDocument, {
destination: URI.file(rootGenerationDir), // actual directory of main module file
createSourceMaps: true,
targetPlaceholder,
targetPlaceholders,
disableRunnerIntegration: false,
});
const lastGeneratedSources = new Map<string, string>();
Expand Down
3 changes: 3 additions & 0 deletions packages/safe-ds-lang/src/language/safe-ds-module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ import { SafeDsCodeLensProvider } from './lsp/safe-ds-code-lens-provider.js';
import { SafeDsExecuteCommandHandler } from './lsp/safe-ds-execute-command-handler.js';
import { SafeDsServiceRegistry } from './safe-ds-service-registry.js';
import { SafeDsPythonServer } from './runtime/safe-ds-python-server.js';
import { SafeDsSlicer } from './flow/safe-ds-slicer.js';

/**
* Declaration of custom services - add your own service classes here.
Expand All @@ -77,6 +78,7 @@ export type SafeDsAddedServices = {
};
flow: {
CallGraphComputer: SafeDsCallGraphComputer;
Slicer: SafeDsSlicer;
};
generation: {
MarkdownGenerator: SafeDsMarkdownGenerator;
Expand Down Expand Up @@ -150,6 +152,7 @@ export const SafeDsModule: Module<SafeDsServices, PartialLangiumServices & SafeD
},
flow: {
CallGraphComputer: (services) => new SafeDsCallGraphComputer(services),
Slicer: (services) => new SafeDsSlicer(services),
},
generation: {
MarkdownGenerator: (services) => new SafeDsMarkdownGenerator(services),
Expand Down
Loading

0 comments on commit 5461a1b

Please sign in to comment.