Skip to content

Commit

Permalink
Amar Tools (#2134)
Browse files Browse the repository at this point in the history 
* example validator

* model validator progress

* wip

* some reporting for model validator

* fixed test case

* fixed testsw

* legacy stuff for Go

* no summary from markdown

* workaround for Amar's cache cleaning

* oav version bump

* comment

* sway fix
  • Loading branch information
@olydis @fearthecowboy
olydis authored and fearthecowboy committed Apr 14, 2017
1 parent d64da84 commit a18bee3
Show file tree
Hide file tree
Showing 8 changed files with 310 additions and 145 deletions.
5 changes: 3 additions & 2 deletions src/autorest-core/legacyCli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,9 @@ export async function CreateConfiguration(baseFolderUri: string, inputScope: Dat

result["add-credentials"] = switches["addcredentials"] === null || ((switches["addcredentials"] + "").toLowerCase() === "true");

if (usedCodeGenerator === "ruby" || usedCodeGenerator === "python") {
result["package-name"] = GetFilenameWithoutExtension(inputFile).replace(/[^a-zA-Z0-9-_]/g, "").replace(/-/g, '_').replace(/([a-z])([A-Z])/g, "$1_$2").toLowerCase();
if (usedCodeGenerator === "ruby" || usedCodeGenerator === "python" || usedCodeGenerator === "go") {
result["package-version"] = switches["pv"] || switches["packageversion"] || undefined;
result["package-name"] = switches["pn"] || switches["packagename"] || GetFilenameWithoutExtension(inputFile).replace(/[^a-zA-Z0-9-_]/g, "").replace(/-/g, '_').replace(/([a-z])([A-Z])/g, "$1_$2").toLowerCase();
}

result["output-file"] = switches["outputfilename"] || undefined;
Expand Down
5 changes: 3 additions & 2 deletions src/autorest-core/lib/configuration.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export interface AutoRestConfigurationImpl {
"directive"?: Directive[] | Directive;
"output-artifact"?: string[] | string;
"message-format"?: "json";
"vscode"?: any;
"vscode"?: any; // activates VS Code specific behavior and does *NOT* influence the core's behavior (only consumed by VS Code extension)

// plugin specific
"output-file"?: string;
Expand All @@ -37,7 +37,8 @@ export interface AutoRestConfigurationImpl {
"namespace"?: string; // TODO: the modeler cares :( because it is badly designed
"license-header"?: string;
"add-credentials"?: boolean;
"package-name"?: string; // Ruby, Python
"package-name"?: string; // Ruby, Python, ...
"package-version"?: string;
"sync-methods"?: "all" | "essential" | "none";
"payload-flattening-threshold"?: number;
}
Expand Down
10 changes: 5 additions & 5 deletions src/autorest-core/lib/parsing/literate-yaml.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ import { MergeYamls, IdentitySourceMapping } from "../source-map/merging";
import { Mapping } from "../ref/source-map";
import { DataHandleRead, DataHandleWrite, DataStoreView } from "../data-store/data-store";
import { Parse as ParseLiterate } from "./literate";
import { IndexToPosition, Lines } from './text-utility';
import { ConfigurationView } from '../autorest-core';
import { Channel, Message, SourceLocation, Range } from '../message';
import { IndexToPosition, Lines } from "./text-utility";
import { ConfigurationView } from "../autorest-core";
import { Channel, Message, SourceLocation, Range } from "../message";
import { safeEval } from "../ref/safe-eval";
import { MessageEmitter } from "../configuration"
import { From } from "../ref/linq"
Expand Down Expand Up @@ -188,7 +188,6 @@ async function ParseCodeBlocksInternal(config: ConfigurationView | MessageEmitte
const scopeEnlightenedCodeBlocks = intermediateScope.CreateScope("enlightened");
const hsConfigFileBlocksWithContext = await ParseLiterate(hLiterate, scopeRawCodeBlocks);

// resolve md documentation (ALPHA)
let codeBlockIndex = 0;
for (const { data, codeBlock } of hsConfigFileBlocksWithContext) {
// only consider YAML/JSON blocks
Expand Down Expand Up @@ -229,11 +228,12 @@ async function ParseCodeBlocksInternal(config: ConfigurationView | MessageEmitte
// fairly confident of no immediate syntax errors.
const yamlAst = CloneAst(ast);

// resolve md documentation (ALPHA)
const deferredErrors: Message[] = []; // ...because the file we wanna blame is not yet written
const mapping: Mapping[] = [];
for (const { path, node } of Descendants(yamlAst)) {
// RESOLVE MARKDOWN INTO THE YAML
if ((path[path.length - 1] === "description" || path[path.length - 1] === "summary") && node.kind === Kind.SEQ) {
if ((path[path.length - 1] === "description") && node.kind === Kind.SEQ) {
// resolve documentation
const mdPath = ParseNode<string[]>(node);
const heading = ResolveMarkdownPath(codeBlock, mdPath);
Expand Down
195 changes: 108 additions & 87 deletions src/autorest-core/lib/pipeline/pipeline.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { AutoRestPlugin } from './plugin-endpoint';
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
Expand All @@ -14,9 +15,7 @@ import { Channel, Message, SourceLocation, Range } from "../message";
import { MultiPromiseUtility, MultiPromise } from "../multi-promise";
import { GetFilename, ResolveUri } from "../ref/uri";
import { ConfigurationView, MessageEmitter } from "../configuration";
import {
DataHandleRead
} from "../data-store/data-store";
import { DataHandleRead, QuickScope } from '../data-store/data-store';
import { AutoRestDotNetPlugin } from "./plugins/autorest-dotnet";
import { ComposeSwaggers, LoadLiterateSwaggers } from "./swagger-loader";
import { From } from "../ref/linq";
Expand All @@ -26,10 +25,90 @@ import { Exception } from '../exception';

export type DataPromise = MultiPromise<DataHandleRead>;

function CreateMessageProcessor(config: ConfigurationView, outstandingTaskAwaiter: OutstandingTaskAwaiter): (m: Message) => Promise<void> {
const supressor = new Supressor(config);

// setup message pipeline (source map resolution, filter, forward)
return async (m: Message) => {
outstandingTaskAwaiter.Enter();
config.Message.Dispatch({ Channel: Channel.Debug, Text: `Incoming validation message (${m.Text}) - starting processing` });

try {
// update source locations to point to loaded Swagger
if (m.Source) {
const blameSources = await Promise.all(m.Source.map(async s => {
try {
const blameTree = await config.DataStore.Blame(s.document, s.Position);
const result = [...blameTree.BlameInputs()];
if (result.length > 0) {
return result.map(r => <SourceLocation>{ document: r.source, Position: Object.assign(TryDecodeEnhancedPositionFromName(r.name) || {}, { line: r.line, column: r.column }) });
}
} catch (e) {
// TODO: activate as soon as .NET swagger loader stuff (inline responses, inline path level parameters, ...)
//console.log(`Failed blaming '${JSON.stringify(s.Position)}' in '${s.document}'`);
//console.log(e);
}
return [s];
}));

//console.log("---");
//console.log(JSON.stringify(m.Source, null, 2));
m.Source = From(blameSources).SelectMany(x => x).ToArray();
//console.log(JSON.stringify(m.Source, null, 2));
//console.log("---");
}

// set range (dummy)
if (m.Source) {
m.Range = m.Source.map(s => {
let positionStart = s.Position;
let positionEnd = <sourceMap.Position>{ line: s.Position.line, column: s.Position.column + (s.Position.length || 3) };

return <Range>{
document: s.document,
start: positionStart,
end: positionEnd
};
});
}

// filter
const mx = supressor.Filter(m);

// forward
if (mx !== null) {
// format message
switch (config.GetEntry("message-format")) {
case "json":
mx.Text = JSON.stringify(mx.Details, null, 2);
break;
default:
let text = `${(mx.Channel || Channel.Information).toString().toUpperCase()}${mx.Key ? ` (${[...mx.Key].join('/')})` : ""}: ${mx.Text}`;
for (const source of mx.Source || []) {
if (source.Position && source.Position.path) {
text += `\n Path: ${source.document}#${stringify(source.Position.path)}`;
}
}
mx.Text = text;
break;
}

config.Message.Dispatch(mx);
}
} catch (e) {
console.error(e);
}

config.Message.Dispatch({ Channel: Channel.Debug, Text: `Incoming validation message (${m.Text}) - finished processing` });
outstandingTaskAwaiter.Exit();
};
}

export async function RunPipeline(config: ConfigurationView, fileSystem: IFileSystem): Promise<void> {
const cancellationToken = config.CancellationToken;

const outstandingTaskAwaiter = new OutstandingTaskAwaiter();
const processMessage = CreateMessageProcessor(config, outstandingTaskAwaiter);
outstandingTaskAwaiter.Enter();

// artifact emitter
Expand Down Expand Up @@ -89,6 +168,28 @@ export async function RunPipeline(config: ConfigurationView, fileSystem: IFileSy
}
config.Message.Dispatch({ Channel: Channel.Debug, Text: `Done Emitting composed documents.` });

// AMAR WORLD
if (!config.DisableValidation && config.GetEntry("amar" as any)) {
const validationPlugin = await AutoRestPlugin.FromModule(`${__dirname}/plugins/openapi-validation-tools`);
const pluginNames = await validationPlugin.GetPluginNames(cancellationToken);
if (pluginNames.length != 2) {
throw new Error("Amar's plugin betrayed us!");
}

for (let pluginIndex = 0; pluginIndex < pluginNames.length; ++pluginIndex) {
const scopeWork = config.DataStore.CreateScope(`amar_${pluginIndex}`);
const result = await validationPlugin.Process(
pluginNames[pluginIndex], _ => null,
new QuickScope([swagger]),
scopeWork.CreateScope("output"),
processMessage,
cancellationToken);
if (!result) {
throw new Error("Amar's plugin failed us!");
}
}
}

const azureValidator = config.AzureArm && !config.DisableValidation;

const allCodeGenerators = ["csharp", "ruby", "nodejs", "python", "go", "java", "azureresourceschema"];
Expand All @@ -100,86 +201,6 @@ export async function RunPipeline(config: ConfigurationView, fileSystem: IFileSy
//
if (azureValidator || usedCodeGenerators.length > 0) {
const autoRestDotNetPlugin = AutoRestDotNetPlugin.Get();
const supressor = new Supressor(config);

// setup message pipeline (source map resolution, filter, forward)
const processMessage = async (sink: IEvent<MessageEmitter, Message>, m: Message) => {
outstandingTaskAwaiter.Enter();
config.Message.Dispatch({ Channel: Channel.Debug, Text: `Incoming validation message (${m.Text}) - starting processing` });

try {
// update source locations to point to loaded Swagger
if (m.Source) {
const blameSources = await Promise.all(m.Source.map(async s => {
try {
const blameTree = await config.DataStore.Blame(s.document, s.Position);
const result = [...blameTree.BlameInputs()];
if (result.length > 0) {
return result.map(r => <SourceLocation>{ document: r.source, Position: Object.assign(TryDecodeEnhancedPositionFromName(r.name) || {}, { line: r.line, column: r.column }) });
}
} catch (e) {
// TODO: activate as soon as .NET swagger loader stuff (inline responses, inline path level parameters, ...)
//console.log(`Failed blaming '${JSON.stringify(s.Position)}' in '${s.document}'`);
//console.log(e);
}
return [s];
}));

//console.log("---");
//console.log(JSON.stringify(m.Source, null, 2));
m.Source = From(blameSources).SelectMany(x => x).ToArray();
//console.log(JSON.stringify(m.Source, null, 2));
//console.log("---");
}

// set range (dummy)
if (m.Source) {
m.Range = m.Source.map(s => {
let positionStart = s.Position;
let positionEnd = <sourceMap.Position>{ line: s.Position.line, column: s.Position.column + (s.Position.length || 3) };

return <Range>{
document: s.document,
start: positionStart,
end: positionEnd
};
});
}

// filter
const mx = supressor.Filter(m);

// forward
if (mx !== null) {
// format message
switch (config.GetEntry("message-format")) {
case "json":
mx.Text = JSON.stringify(mx.Details, null, 2);
break;
default:
let text = `${(mx.Channel || Channel.Information).toString().toUpperCase()}${mx.Key ? ` (${[...mx.Key].join('/')})` : ""}: ${mx.Text}`;
for (const source of mx.Source || []) {
if (source.Position && source.Position.path) {
text += `\n Path: ${source.document}#${stringify(source.Position.path)}`;
}
}
mx.Text = text;
break;
}

sink.Dispatch(mx);
}
} catch (e) {
console.error(e);
}

config.Message.Dispatch({ Channel: Channel.Debug, Text: `Incoming validation message (${m.Text}) - finished processing` });
outstandingTaskAwaiter.Exit();
};

const messageSink = (m: Message) => {
processMessage(config.Message, m);
};

// code generators
if (usedCodeGenerators.length > 0) {
Expand All @@ -188,7 +209,7 @@ export async function RunPipeline(config: ConfigurationView, fileSystem: IFileSy
{
namespace: config.GetEntry("namespace") || ""
},
messageSink);
processMessage);

// GFMer
const codeModelGFM = await ProcessCodeModel(codeModel, config.DataStore.CreateScope("modelgfm"));
Expand All @@ -213,11 +234,11 @@ export async function RunPipeline(config: ConfigurationView, fileSystem: IFileSy
"sync-methods": getXmsCodeGenSetting("syncMethods")
},
genConfig.Raw),
messageSink);
processMessage);

// C# simplifier
if (usedCodeGenerator === "csharp") {
generatedFileScope = await autoRestDotNetPlugin.SimplifyCSharpCode(generatedFileScope, scope.CreateScope("simplify"), messageSink);
generatedFileScope = await autoRestDotNetPlugin.SimplifyCSharpCode(generatedFileScope, scope.CreateScope("simplify"), processMessage);
}

for (const fileName of await generatedFileScope.Enum()) {
Expand All @@ -233,7 +254,7 @@ export async function RunPipeline(config: ConfigurationView, fileSystem: IFileSy

// validator
if (azureValidator) {
await autoRestDotNetPlugin.Validate(swagger, config.DataStore.CreateScope("validate"), messageSink);
await autoRestDotNetPlugin.Validate(swagger, config.DataStore.CreateScope("validate"), processMessage);
}
}

Expand Down
Loading

0 comments on commit a18bee3

Please sign in to comment.