diff --git a/build.gradle b/build.gradle index aeb609e2b41..ce01367eec9 100644 --- a/build.gradle +++ b/build.gradle @@ -118,9 +118,9 @@ subprojects { apply plugin: "signing" apply plugin: "com.github.johnrengelman.shadow" - // This is a little hacky, but currently needed to build a shadowed CLI JAR with the same customizations - // as other JARs. - if (project.name != "smithy-cli") { + // This is a little hacky, but currently needed to build a shadowed CLI JAR and smithy-syntax JAR with the same + // customizations as other JARs. + if (project.name != "smithy-cli" && project.name != "smithy-syntax") { tasks.shadowJar.enabled = false } diff --git a/settings.gradle b/settings.gradle index 8fd75a22602..f127d43b230 100644 --- a/settings.gradle +++ b/settings.gradle @@ -29,3 +29,4 @@ include ":smithy-aws-cloudformation-traits" include ":smithy-aws-cloudformation" include ":smithy-validation-model" include ":smithy-rules-engine" +include ":smithy-syntax" diff --git a/smithy-cli/build.gradle b/smithy-cli/build.gradle index 4f23cc5a4bd..dcf31372a6e 100644 --- a/smithy-cli/build.gradle +++ b/smithy-cli/build.gradle @@ -37,11 +37,13 @@ dependencies { implementation project(":smithy-model") implementation project(":smithy-build") implementation project(":smithy-diff") + implementation project(path: ':smithy-syntax', configuration: 'shadow') // This is needed to ensure the above dependencies are added to the runtime image. shadow project(":smithy-model") shadow project(":smithy-build") shadow project(":smithy-diff") + shadow project(":smithy-syntax") // These maven resolver dependencies are shaded into the smithy-cli JAR. implementation "org.apache.maven:maven-resolver-provider:3.8.6" @@ -82,6 +84,7 @@ shadowJar { exclude(project(':smithy-model')) exclude(project(':smithy-build')) exclude(project(':smithy-diff')) + exclude(project(':smithy-syntax')) } } diff --git a/smithy-cli/src/it/java/software/amazon/smithy/cli/FormatCommandTest.java b/smithy-cli/src/it/java/software/amazon/smithy/cli/FormatCommandTest.java new file mode 100644 index 00000000000..efc888aacd8 --- /dev/null +++ b/smithy-cli/src/it/java/software/amazon/smithy/cli/FormatCommandTest.java @@ -0,0 +1,71 @@ +package software.amazon.smithy.cli; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +import org.junit.jupiter.api.Test; +import software.amazon.smithy.utils.ListUtils; + +public class FormatCommandTest { + @Test + public void failsWhenNoModelsGiven() { + IntegUtils.run("bad-formatting", ListUtils.of("format"), result -> { + assertThat(result.getExitCode(), equalTo(1)); + assertThat(result.getOutput(), + containsString("No .smithy model or directory was provided as a positional argument")); + }); + } + + @Test + public void failsWhenBadFileGiven() { + IntegUtils.run("bad-formatting", ListUtils.of("format", "THIS_FILE_DOES_NOT_EXIST_1234"), result -> { + assertThat(result.getExitCode(), equalTo(1)); + + assertThat(result.getOutput(), + containsString("`THIS_FILE_DOES_NOT_EXIST_1234` is not a valid file or directory")); + }); + } + + @Test + public void formatsSingleFile() { + IntegUtils.run("bad-formatting", ListUtils.of("format", "model/other.smithy"), result -> { + assertThat(result.getExitCode(), equalTo(0)); + + String model = result.getFile("model/other.smithy").replace("\r\n", "\n"); + assertThat(model, equalTo("$version: \"2.0\"\n" + + "\n" + + "namespace smithy.example\n" + + "\n" + + "string MyString\n" + + "\n" + + "string MyString2\n")); + }); + } + + @Test + public void formatsDirectory() { + IntegUtils.run("bad-formatting", ListUtils.of("format", "model"), result -> { + assertThat(result.getExitCode(), equalTo(0)); + + String main = result.getFile("model/main.smithy").replace("\r\n", "\n"); + assertThat(main, equalTo("$version: \"2.0\"\n" + + "\n" + + "metadata this_is_a_long_string = {\n" + + " this_is_a_long_string1: \"a\"\n" + + " this_is_a_long_string2: \"b\"\n" + + " this_is_a_long_string3: \"c\"\n" + + " this_is_a_long_string4: \"d\"\n" + + "}\n")); + + String other = result.getFile("model/other.smithy").replace("\r\n", "\n"); + assertThat(other, equalTo("$version: \"2.0\"\n" + + "\n" + + "namespace smithy.example\n" + + "\n" + + "string MyString\n" + + "\n" + + "string MyString2\n")); + }); + } +} diff --git a/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/model/ast.json b/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/model/ast.json new file mode 100644 index 00000000000..14d094137dc --- /dev/null +++ b/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/model/ast.json @@ -0,0 +1,3 @@ +{ + "smithy": "2.0" +} diff --git a/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/model/main.smithy b/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/model/main.smithy new file mode 100644 index 00000000000..a6f0f428911 --- /dev/null +++ b/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/model/main.smithy @@ -0,0 +1,4 @@ +$version: "2.0" + +metadata this_is_a_long_string = {this_is_a_long_string1: "a", this_is_a_long_string2: "b", + this_is_a_long_string3: "c", this_is_a_long_string4: "d"} diff --git a/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/model/other.smithy b/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/model/other.smithy new file mode 100644 index 00000000000..80897595e1a --- /dev/null +++ b/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/model/other.smithy @@ -0,0 +1,4 @@ +$version: "2.0" +namespace smithy.example +string MyString +string MyString2 diff --git a/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/smithy-build.json b/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/smithy-build.json new file mode 100644 index 00000000000..04450271687 --- /dev/null +++ b/smithy-cli/src/it/resources/software/amazon/smithy/cli/projects/bad-formatting/smithy-build.json @@ -0,0 +1,3 @@ +{ + "version": "1.0" +} diff --git a/smithy-cli/src/main/java/software/amazon/smithy/cli/commands/FormatCommand.java b/smithy-cli/src/main/java/software/amazon/smithy/cli/commands/FormatCommand.java new file mode 100644 index 00000000000..16d5138a2b7 --- /dev/null +++ b/smithy-cli/src/main/java/software/amazon/smithy/cli/commands/FormatCommand.java @@ -0,0 +1,122 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.cli.commands; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import software.amazon.smithy.cli.ArgumentReceiver; +import software.amazon.smithy.cli.Arguments; +import software.amazon.smithy.cli.CliError; +import software.amazon.smithy.cli.ColorBuffer; +import software.amazon.smithy.cli.ColorTheme; +import software.amazon.smithy.cli.Command; +import software.amazon.smithy.cli.HelpPrinter; +import software.amazon.smithy.model.loader.IdlTokenizer; +import software.amazon.smithy.syntax.Formatter; +import software.amazon.smithy.syntax.TokenTree; +import software.amazon.smithy.utils.IoUtils; + +final class FormatCommand implements Command { + + private final String parentCommandName; + + FormatCommand(String parentCommandName) { + this.parentCommandName = parentCommandName; + } + + @Override + public String getName() { + return "format"; + } + + @Override + public String getSummary() { + return "Formats Smithy IDL models."; + } + + private static final class Options implements ArgumentReceiver { + @Override + public void registerHelp(HelpPrinter printer) { + printer.positional("", + "A single `.smithy` model file or a directory of model files to recursively format."); + } + } + + @Override + public int execute(Arguments arguments, Env env) { + arguments.addReceiver(new Options()); + + CommandAction action = HelpActionWrapper.fromCommand(this, parentCommandName, c -> { + ColorBuffer buffer = ColorBuffer.of(c, new StringBuilder()); + buffer.println("Examples:"); + buffer.println(" smithy format model-file.smithy", ColorTheme.LITERAL); + buffer.println(" smithy format model/", ColorTheme.LITERAL); + return buffer.toString(); + }, this::run); + return action.apply(arguments, env); + } + + private int run(Arguments arguments, Env env) { + if (arguments.getPositional().isEmpty()) { + throw new CliError("No .smithy model or directory was provided as a positional argument"); + } else if (arguments.getPositional().size() > 1) { + throw new CliError("Only a single .smithy model or directory can be provided as a positional argument"); + } + + String filename = arguments.getPositional().get(0); + Path path = Paths.get(filename); + + if (Files.isRegularFile(path)) { + if (!filename.endsWith(".smithy")) { + throw new CliError("`" + filename + "` is not a .smithy model file"); + } + } else if (!Files.isDirectory(path)) { + throw new CliError("`" + filename + "` is not a valid file or directory"); + } + + formatFile(path); + return 0; + } + + private void formatFile(Path file) { + if (Files.isDirectory(file)) { + try { + Files.find(file, 100, (p, a) -> a.isRegularFile()).forEach(this::formatFile); + } catch (IOException e) { + throw new CliError("Error formatting " + file + " (directory): " + e.getMessage()); + } + } else if (Files.isRegularFile(file) && file.toString().endsWith(".smithy")) { + TokenTree tree = parse(file); + String formatted = Formatter.format(tree); + try (OutputStream s = Files.newOutputStream(file, StandardOpenOption.TRUNCATE_EXISTING)) { + s.write(formatted.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new CliError("Error formatting " + file + " (file): " + e.getMessage()); + } + } + } + + private TokenTree parse(Path file) { + String contents = IoUtils.readUtf8File(file); + IdlTokenizer tokenizer = IdlTokenizer.create(file.toString(), contents); + return TokenTree.of(tokenizer); + } +} diff --git a/smithy-cli/src/main/java/software/amazon/smithy/cli/commands/SmithyCommand.java b/smithy-cli/src/main/java/software/amazon/smithy/cli/commands/SmithyCommand.java index c8854d73fcb..80bad5a3a78 100644 --- a/smithy-cli/src/main/java/software/amazon/smithy/cli/commands/SmithyCommand.java +++ b/smithy-cli/src/main/java/software/amazon/smithy/cli/commands/SmithyCommand.java @@ -46,6 +46,7 @@ public SmithyCommand(DependencyResolver.Factory dependencyResolverFactory) { new DiffCommand(getName(), dependencyResolverFactory), new AstCommand(getName(), dependencyResolverFactory), new SelectCommand(getName(), dependencyResolverFactory), + new FormatCommand(getName()), new CleanCommand(getName()), migrateCommand, deprecated1To2Command, diff --git a/smithy-cli/src/main/java/software/amazon/smithy/cli/dependencies/FilterCliVersionResolver.java b/smithy-cli/src/main/java/software/amazon/smithy/cli/dependencies/FilterCliVersionResolver.java index bf4004cb032..1c4c2d51ef4 100644 --- a/smithy-cli/src/main/java/software/amazon/smithy/cli/dependencies/FilterCliVersionResolver.java +++ b/smithy-cli/src/main/java/software/amazon/smithy/cli/dependencies/FilterCliVersionResolver.java @@ -37,7 +37,7 @@ public final class FilterCliVersionResolver implements DependencyResolver { private static final Logger LOGGER = Logger.getLogger(FilterCliVersionResolver.class.getName()); private static final String SMITHY_GROUP = "software.amazon.smithy"; private static final Set CLI_ARTIFACTS = SetUtils.of( - "smithy-utils", "smithy-model", "smithy-build", "smithy-cli", "smithy-diff"); + "smithy-utils", "smithy-model", "smithy-build", "smithy-cli", "smithy-diff", "smithy-syntax"); private final String version; private final DependencyResolver delegate; diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/DefaultTokenizer.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/DefaultTokenizer.java new file mode 100644 index 00000000000..1f5434eec0e --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/DefaultTokenizer.java @@ -0,0 +1,441 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.NoSuchElementException; +import software.amazon.smithy.model.SourceLocation; +import software.amazon.smithy.utils.SimpleParser; + +class DefaultTokenizer implements IdlTokenizer { + + private final String filename; + private final SimpleParser parser; + private IdlToken currentTokenType; + private int currentTokenStart = -1; + private int currentTokenEnd = -1; + private int currentTokenLine = -1; + private int currentTokenColumn = -1; + private Number currentTokenNumber; + private CharSequence currentTokenStringSlice; + private String currentTokenError; + + DefaultTokenizer(String filename, CharSequence model) { + this.filename = filename; + this.parser = new SimpleParser(model, 64); + } + + @Override + public final String getSourceFilename() { + return filename; + } + + @Override + public final CharSequence getModel() { + return parser.input(); + } + + @Override + public final int getPosition() { + return parser.position(); + } + + @Override + public final int getLine() { + return parser.line(); + } + + @Override + public final int getColumn() { + return parser.column(); + } + + @Override + public final IdlToken getCurrentToken() { + if (currentTokenType == null) { + next(); + } + return currentTokenType; + } + + @Override + public final int getCurrentTokenLine() { + getCurrentToken(); + return currentTokenLine; + } + + @Override + public final int getCurrentTokenColumn() { + getCurrentToken(); + return currentTokenColumn; + } + + @Override + public final int getCurrentTokenStart() { + getCurrentToken(); + return currentTokenStart; + } + + @Override + public final int getCurrentTokenEnd() { + return currentTokenEnd; + } + + @Override + public final CharSequence getCurrentTokenStringSlice() { + getCurrentToken(); + if (currentTokenStringSlice != null) { + return currentTokenStringSlice; + } else if (currentTokenType == IdlToken.IDENTIFIER) { + return getCurrentTokenLexeme(); + } else { + throw syntax("The current token must be string or identifier but found: " + + currentTokenType.getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); + } + } + + @Override + public final Number getCurrentTokenNumberValue() { + getCurrentToken(); + if (currentTokenNumber == null) { + throw syntax("The current token must be number but found: " + + currentTokenType.getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); + } + return currentTokenNumber; + } + + @Override + public final String getCurrentTokenError() { + getCurrentToken(); + if (currentTokenType != IdlToken.ERROR) { + throw syntax("The current token must be an error but found: " + + currentTokenType.getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); + } + return currentTokenError == null ? "" : currentTokenError; + } + + @Override + public final boolean hasNext() { + return currentTokenType != IdlToken.EOF; + } + + @Override + public IdlToken next() { + currentTokenStringSlice = null; + currentTokenNumber = null; + currentTokenColumn = parser.column(); + currentTokenLine = parser.line(); + currentTokenStart = parser.position(); + currentTokenEnd = currentTokenStart; + int c = parser.peek(); + + switch (c) { + case SimpleParser.EOF: + if (currentTokenType == IdlToken.EOF) { + throw new NoSuchElementException("Expected another token but reached EOF"); + } + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.EOF; + case ' ': + case '\t': + return tokenizeSpace(); + case '\r': + case '\n': + return tokenizeNewline(); + case ',': + return singleCharToken(IdlToken.COMMA); + case '@': + return singleCharToken(IdlToken.AT); + case '$': + return singleCharToken(IdlToken.DOLLAR); + case '.': + return singleCharToken(IdlToken.DOT); + case '{': + return singleCharToken(IdlToken.LBRACE); + case '}': + return singleCharToken(IdlToken.RBRACE); + case '[': + return singleCharToken(IdlToken.LBRACKET); + case ']': + return singleCharToken(IdlToken.RBRACKET); + case '(': + return singleCharToken(IdlToken.LPAREN); + case ')': + return singleCharToken(IdlToken.RPAREN); + case '#': + return singleCharToken(IdlToken.POUND); + case '=': + return singleCharToken(IdlToken.EQUAL); + case ':': + return parseColon(); + case '"': + return parseString(); + case '/': + return parseComment(); + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return parseNumber(); + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': + return parseIdentifier(); + default: + currentTokenError = "Unexpected character: '" + ((char) c) + '\''; + return singleCharToken(IdlToken.ERROR); + } + } + + private ModelSyntaxException syntax(String message, SourceLocation location) { + return new ModelSyntaxException("Syntax error at line " + location.getLine() + ", column " + + location.getColumn() + ": " + message, location); + } + + private IdlToken singleCharToken(IdlToken type) { + parser.skip(); + currentTokenEnd = parser.position(); + return currentTokenType = type; + } + + private IdlToken tokenizeNewline() { + parser.skip(); // this will \n and \r\n. + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.NEWLINE; + } + + private IdlToken tokenizeSpace() { + parser.consumeWhile(c -> c == ' ' || c == '\t'); + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.SPACE; + } + + private IdlToken parseColon() { + parser.skip(); + + if (parser.peek() == '=') { + parser.skip(); + currentTokenType = IdlToken.WALRUS; + } else { + currentTokenType = IdlToken.COLON; + } + + currentTokenEnd = parser.position(); + return currentTokenType; + } + + private IdlToken parseComment() { + // first "/". + parser.expect('/'); + + // A standalone forward slash is an error. + if (parser.peek() != '/') { + currentTokenError = "Expected a '/' to follow '/' to form a comment."; + return singleCharToken(IdlToken.ERROR); + } + + // Skip the next "/". + parser.skip(); + + IdlToken type = IdlToken.COMMENT; + + // Three "///" is a documentation comment. + if (parser.peek() == '/') { + parser.skip(); + type = IdlToken.DOC_COMMENT; + } + + parser.consumeRemainingCharactersOnLine(); + + // Include the newline in the comment and doc comment lexeme. + if (parser.expect('\r', '\n', SimpleParser.EOF) == '\r' && parser.peek() == '\n') { + parser.skip(); + } + + currentTokenEnd = parser.position(); + return currentTokenType = type; + } + + private IdlToken parseNumber() { + try { + String lexeme = ParserUtils.parseNumber(parser); + if (lexeme.contains("e") || lexeme.contains("E") || lexeme.contains(".")) { + double value = Double.parseDouble(lexeme); + if (Double.isFinite(value)) { + currentTokenNumber = value; + } else { + currentTokenNumber = new BigDecimal(lexeme); + } + } else { + try { + currentTokenNumber = Long.parseLong(lexeme); + } catch (NumberFormatException e) { + currentTokenNumber = new BigInteger(lexeme); + } + } + + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.NUMBER; + } catch (RuntimeException e) { + currentTokenEnd = parser.position(); + // Strip off the leading error message information if present. + if (e.getMessage().startsWith("Syntax error")) { + currentTokenError = e.getMessage().substring(e.getMessage().indexOf(':') + 1).trim(); + } else { + currentTokenError = e.getMessage(); + } + return currentTokenType = IdlToken.ERROR; + } + } + + private IdlToken parseIdentifier() { + try { + ParserUtils.consumeIdentifier(parser); + currentTokenType = IdlToken.IDENTIFIER; + } catch (RuntimeException e) { + currentTokenType = IdlToken.ERROR; + currentTokenError = e.getMessage(); + } + currentTokenEnd = parser.position(); + return currentTokenType; + } + + private IdlToken parseString() { + parser.skip(); // skip first quote. + + if (parser.peek() == '"') { + parser.skip(); // skip second quote. + if (parser.peek() == '"') { // A third consecutive quote is a TEXT_BLOCK. + parser.skip(); + return parseTextBlock(); + } else { + // Empty string. + currentTokenEnd = parser.position(); + currentTokenStringSlice = ""; + return currentTokenType = IdlToken.STRING; + } + } + + try { + // Parse the contents of a quoted string. + currentTokenStringSlice = parseQuotedTextAndTextBlock(false); + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.STRING; + } catch (RuntimeException e) { + currentTokenEnd = parser.position(); + currentTokenError = "Error parsing quoted string: " + e.getMessage(); + return currentTokenType = IdlToken.ERROR; + } + } + + private IdlToken parseTextBlock() { + try { + currentTokenStringSlice = parseQuotedTextAndTextBlock(true); + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.TEXT_BLOCK; + } catch (RuntimeException e) { + currentTokenEnd = parser.position(); + currentTokenError = "Error parsing text block: " + e.getMessage(); + return currentTokenType = IdlToken.ERROR; + } + } + + // Parses both quoted_text and text_block + private CharSequence parseQuotedTextAndTextBlock(boolean triple) { + int start = parser.position(); + + while (!parser.eof()) { + char next = parser.peek(); + if (next == '"' && (!triple || (parser.peek(1) == '"' && parser.peek(2) == '"'))) { + // Found closing quotes of quoted_text and/or text_block + break; + } + parser.skip(); + if (next == '\\') { + parser.skip(); + } + } + + // Strip the ending '"'. + CharSequence result = parser.borrowSliceFrom(start); + parser.expect('"'); + + if (triple) { + parser.expect('"'); + parser.expect('"'); + } + + return IdlStringLexer.scanStringContents(result, triple); + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlInternalTokenizer.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlInternalTokenizer.java new file mode 100644 index 00000000000..d30fe637b46 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlInternalTokenizer.java @@ -0,0 +1,177 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.function.Consumer; +import software.amazon.smithy.model.validation.ValidationEvent; + +/** + * A specialized Tokenizer that adds skipping and documentation buffering. + */ +final class IdlInternalTokenizer extends DefaultTokenizer { + + private final CharSequence model; + private final Deque docCommentLines = new ArrayDeque<>(); + private final Consumer validationEventListener; + + IdlInternalTokenizer(String filename, CharSequence model) { + this(filename, model, event -> { }); + } + + IdlInternalTokenizer(String filename, CharSequence model, Consumer validationEventListener) { + super(filename, model); + this.model = model; + this.validationEventListener = validationEventListener; + } + + @Override + public IdlToken next() { + IdlToken token = super.next(); + + // Buffer documentation comments as they're encountered. + if (token == IdlToken.DOC_COMMENT) { + int start = getCurrentTokenStart(); + int span = getCurrentTokenSpan(); + int docStart; + + if (span <= 3) { + // An empty doc comment. + docStart = start; + } else if (model.charAt(start + 3) == ' ') { + // Skip single space. + docStart = start + 4; + } else { + docStart = start + 3; + } + + // Strip the \n and \r\n from the end. + int end = getCurrentTokenEnd(); + if (model.charAt(end - 1) == '\n') { + end--; + if (end >= 0 && model.charAt(end - 1) == '\r') { + end--; + } + } + + docCommentLines.add(getModel(docStart, end)); + } + + return token; + } + + void skipSpaces() { + while (getCurrentToken() == IdlToken.SPACE) { + next(); + } + } + + void skipOptionalComma() { + if (getCurrentToken() == IdlToken.COMMA) { + next(); + } + } + + void skipWs() { + while (getCurrentToken().isWhitespace()) { + next(); + } + } + + void skipWsAndDocs() { + IdlToken currentTokenType = getCurrentToken(); + while (currentTokenType.isWhitespace() || currentTokenType == IdlToken.DOC_COMMENT) { + next(); + currentTokenType = getCurrentToken(); + } + } + + void expectAndSkipSpaces() { + expect(IdlToken.SPACE); + skipSpaces(); + } + + void expectAndSkipWhitespace() { + if (!getCurrentToken().isWhitespace()) { + throw LoaderUtils.idlSyntaxError("Expected one or more whitespace characters, but found " + + getCurrentToken().getDebug(getCurrentTokenLexeme()), + getCurrentTokenLocation()); + } + skipWsAndDocs(); + } + + /** + * Expects that the current token is zero or more spaces/commas followed by a newline, comment, documentation + * comment, or EOF. + * + *

If a documentation comment is detected, the current token remains the documentation comment. If an EOF is + * detected, the current token remains the EOF token. If a comment, newline, or whitespace are detected, they are + * all skipped, leaving the current token the next token after encountering the matched token. Other kinds of + * tokens will raise an exception. + * + *

This method mimics the {@code br} production from the Smithy grammar. When this method is called, any + * previously parsed and buffered documentation comments are discarded. + * + * @throws ModelSyntaxException if the current token is not a newline or followed by a newline. + */ + void expectAndSkipBr() { + skipSpaces(); + clearDocCommentLinesForBr(); + + // The following tokens are allowed tokens that contain newlines. + switch (getCurrentToken()) { + case NEWLINE: + case COMMENT: + case DOC_COMMENT: + next(); + skipWs(); + break; + case EOF: + break; + default: + throw LoaderUtils.idlSyntaxError( + "Expected a line break, but found " + + getCurrentToken().getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); + } + } + + private void clearDocCommentLinesForBr() { + if (!docCommentLines.isEmpty()) { + validationEventListener.accept(LoaderUtils.emitBadDocComment(getCurrentTokenLocation(), + removePendingDocCommentLines())); + } + } + + /** + * Removes any buffered documentation comment lines, and returns a concatenated string value. + * + * @return Returns the combined documentation comment string for the given lines. Returns null if no lines. + */ + String removePendingDocCommentLines() { + if (docCommentLines.isEmpty()) { + return null; + } else { + StringBuilder result = new StringBuilder(); + while (!docCommentLines.isEmpty()) { + result.append(docCommentLines.removeFirst()).append("\n"); + } + // Strip ending \n. + result.setLength(result.length() - 1); + return result.toString(); + } + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java index c8390a5dd46..7b87c6ec3d9 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java @@ -71,6 +71,9 @@ final class IdlModelLoader { private static final String TYPE_KEY = "type"; private static final String ERRORS_KEY = "errors"; + /** Only allow nesting up to 64 arrays/objects in node values. */ + private static final int MAX_NESTING_LEVEL = 64; + static final Collection RESOURCE_PROPERTY_NAMES = ListUtils.of( TYPE_KEY, CREATE_KEY, READ_KEY, UPDATE_KEY, DELETE_KEY, LIST_KEY, IDENTIFIERS_KEY, RESOURCES_KEY, OPERATIONS_KEY, PUT_KEY, PROPERTIES_KEY, COLLECTION_OPERATIONS_KEY); @@ -87,26 +90,22 @@ final class IdlModelLoader { } private final String filename; - private final IdlTokenizer tokenizer; + private final IdlInternalTokenizer tokenizer; private final Map useShapes = new HashMap<>(); - private final IdlReferenceResolver resolver; + private final Function stringTable; private Consumer operations; private Version modelVersion = Version.VERSION_1_0; private String namespace; private boolean emittedVersion = false; + private int nesting = 0; private String operationInputSuffix = "Input"; private String operationOutputSuffix = "Output"; IdlModelLoader(String filename, CharSequence model, Function stringTable) { this.filename = filename; - this.tokenizer = IdlTokenizer.builder() - .filename(filename) - .model(model) - .stringTable(stringTable) - .validationEventListener(this::emit) - .build(); - this.resolver = this::addForwardReference; + this.stringTable = stringTable; + this.tokenizer = new IdlInternalTokenizer(filename, model, this::emit); } void parse(Consumer operationConsumer) { @@ -132,17 +131,31 @@ void addOperation(LoadOperation operation) { operations.accept(operation); } - public ModelSyntaxException syntax(String message) { + IdlInternalTokenizer getTokenizer() { + return tokenizer; + } + + ModelSyntaxException syntax(String message) { return syntax(null, message); } + String internString(CharSequence sequence) { + return stringTable.apply(sequence); + } + + void increaseNestingLevel() { + if (++nesting > MAX_NESTING_LEVEL) { + throw LoaderUtils.idlSyntaxError("Parser exceeded maximum allowed depth of " + MAX_NESTING_LEVEL, + tokenizer.getCurrentTokenLocation()); + } + } + + void decreaseNestingLevel() { + nesting--; + } + ModelSyntaxException syntax(ShapeId shapeId, String message) { - return ModelSyntaxException.builder() - .message(format("Syntax error at line %d, column %d: %s", - tokenizer.getCurrentTokenLine(), tokenizer.getCurrentTokenColumn(), message)) - .sourceLocation(tokenizer.getCurrentTokenLocation()) - .shapeId(shapeId) - .build(); + return LoaderUtils.idlSyntaxError(shapeId, message, tokenizer.getCurrentTokenLocation()); } void addForwardReference(String id, BiFunction receiver) { @@ -230,7 +243,7 @@ private void parseControlSection() { try { tokenizer.next(); tokenizer.expect(IdlToken.IDENTIFIER, IdlToken.STRING); - String key = tokenizer.internString(tokenizer.getCurrentTokenStringSlice()); + String key = internString(tokenizer.getCurrentTokenStringSlice()); tokenizer.next(); tokenizer.skipSpaces(); @@ -242,7 +255,7 @@ private void parseControlSection() { throw syntax(format("Duplicate control statement `%s`", key)); } - Node value = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + Node value = IdlNodeParser.expectAndSkipNode(this); switch (key) { case "version": @@ -291,21 +304,21 @@ private void onVersion(Node value) { } private void parseMetadataSection() { - while (tokenizer.doesCurrentIdentifierStartWith('m')) { + while (tokenizer.isCurrentLexeme("metadata")) { try { - tokenizer.expectCurrentLexeme("metadata"); tokenizer.next(); // skip "metadata" tokenizer.expectAndSkipSpaces(); tokenizer.expect(IdlToken.IDENTIFIER, IdlToken.STRING); - String key = tokenizer.internString(tokenizer.getCurrentTokenStringSlice()); + String key = internString(tokenizer.getCurrentTokenStringSlice()); tokenizer.next(); tokenizer.skipSpaces(); tokenizer.expect(IdlToken.EQUAL); tokenizer.next(); tokenizer.skipSpaces(); - Node value = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + Node value = IdlNodeParser.expectAndSkipNode(this); operations.accept(new LoadOperation.PutMetadata(modelVersion, key, value)); tokenizer.expectAndSkipBr(); + tokenizer.skipWsAndDocs(); } catch (ModelSyntaxException e) { errorRecovery(e); } @@ -313,13 +326,12 @@ private void parseMetadataSection() { } private void parseShapeSection() { - if (tokenizer.doesCurrentIdentifierStartWith('n')) { - tokenizer.expectCurrentLexeme("namespace"); + if (tokenizer.isCurrentLexeme("namespace")) { tokenizer.next(); // skip "namespace" tokenizer.expectAndSkipSpaces(); // Parse the namespace. - namespace = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeIdNamespace(tokenizer)); + namespace = internString(IdlShapeIdParser.expectAndSkipShapeIdNamespace(tokenizer)); tokenizer.expectAndSkipBr(); // An unfortunate side effect of allowing insignificant documentation comments: @@ -340,7 +352,7 @@ private void parseShapeSection() { private void parseUseSection() { while (tokenizer.getCurrentToken() == IdlToken.IDENTIFIER) { // Don't over-parse here for unions. - String keyword = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + String keyword = internString(tokenizer.getCurrentTokenLexeme()); if (!keyword.equals("use")) { break; } @@ -349,7 +361,7 @@ private void parseUseSection() { tokenizer.expectAndSkipSpaces(); SourceLocation idLocation = tokenizer.getCurrentTokenLocation(); - String idString = tokenizer.internString(IdlShapeIdParser.expectAndSkipAbsoluteShapeId(tokenizer)); + String idString = internString(IdlShapeIdParser.expectAndSkipAbsoluteShapeId(tokenizer)); ShapeId id = ShapeId.from(idString); if (id.hasMember()) { @@ -400,21 +412,20 @@ private void parseApplyStatement(List traits) { LoaderUtils.emitBadDocComment(foundDocComments, null); } - tokenizer.expectCurrentLexeme("apply"); tokenizer.next(); tokenizer.expectAndSkipWhitespace(); - String target = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + String target = internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); tokenizer.expectAndSkipWhitespace(); List traitsToApply; if (IdlToken.AT == tokenizer.expect(IdlToken.AT, IdlToken.LBRACE)) { // Parse a single trait. - traitsToApply = Collections.singletonList(IdlTraitParser.expectAndSkipTrait(tokenizer, resolver)); + traitsToApply = Collections.singletonList(IdlTraitParser.expectAndSkipTrait(this)); } else { // Parse a trait block. tokenizer.next(); // skip opening LBRACE. tokenizer.skipWsAndDocs(); - traitsToApply = IdlTraitParser.expectAndSkipTraits(tokenizer, resolver); + traitsToApply = IdlTraitParser.expectAndSkipTraits(this); tokenizer.skipWsAndDocs(); tokenizer.expect(IdlToken.RBRACE); tokenizer.next(); @@ -423,7 +434,7 @@ private void parseApplyStatement(List traits) { // First, resolve the targeted shape. addForwardReference(target, id -> { for (IdlTraitParser.Result trait : traitsToApply) { - String traitNameString = tokenizer.internString(trait.getTraitName()); + String traitNameString = internString(trait.getTraitName()); onDeferredTrait(id, traitNameString, trait.getValue(), trait.getTraitType() == IdlTraitParser.TraitType.ANNOTATION); } @@ -437,7 +448,7 @@ private void parseFirstShapeStatement(SourceLocation possibleDocCommentLocation) try { tokenizer.skipWsAndDocs(); String docLines = tokenizer.removePendingDocCommentLines(); - List traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(tokenizer, resolver); + List traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this); if (docLines != null) { traits.add(new IdlTraitParser.Result(DocumentationTrait.ID.toString(), new StringNode(docLines, possibleDocCommentLocation), @@ -456,7 +467,7 @@ private void parseSubsequentShapeStatements() { while (tokenizer.hasNext()) { try { boolean hasDocComment = tokenizer.getCurrentToken() == IdlToken.DOC_COMMENT; - List traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(tokenizer, resolver); + List traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this); if (parseShapeDefinition(traits, hasDocComment)) { parseShapeOrApply(traits); } @@ -525,7 +536,7 @@ private boolean parseShapeDefinition(List traits, boolean private void parseShapeOrApply(List traits) { SourceLocation location = tokenizer.getCurrentTokenLocation(); tokenizer.expect(IdlToken.IDENTIFIER); - String shapeType = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + String shapeType = internString(tokenizer.getCurrentTokenLexeme()); if (shapeType.equals("apply")) { parseApplyStatement(traits); @@ -586,7 +597,7 @@ private void parseShapeOrApply(List traits) { private void addTraits(ShapeId id, List traits) { for (IdlTraitParser.Result result : traits) { - String traitName = tokenizer.internString(result.getTraitName()); + String traitName = internString(result.getTraitName()); onDeferredTrait(id, traitName, result.getValue(), result.getTraitType() == IdlTraitParser.TraitType.ANNOTATION); } @@ -596,7 +607,7 @@ private ShapeId parseShapeName() { int line = tokenizer.getCurrentTokenLine(); int column = tokenizer.getCurrentTokenColumn(); tokenizer.expect(IdlToken.IDENTIFIER); - String name = tokenizer.internString(tokenizer.getCurrentTokenStringSlice()); + String name = internString(tokenizer.getCurrentTokenStringSlice()); ShapeId id = ShapeId.fromRelative(expectNamespace(), name); if (useShapes.containsKey(name)) { ShapeId previous = useShapes.get(name); @@ -620,12 +631,11 @@ LoadOperation.DefineShape createShape(AbstractShapeBuilder builder) { private void parseMixins(LoadOperation.DefineShape operation) { tokenizer.skipSpaces(); - if (!tokenizer.doesCurrentIdentifierStartWith('w')) { + if (!tokenizer.isCurrentLexeme("with")) { return; } tokenizer.expect(IdlToken.IDENTIFIER); - tokenizer.expectCurrentLexeme("with"); if (!modelVersion.supportsMixins()) { throw syntax(operation.toShapeId(), "Mixins can only be used with Smithy version 2 or later. " @@ -639,7 +649,7 @@ private void parseMixins(LoadOperation.DefineShape operation) { tokenizer.skipWsAndDocs(); do { - String target = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + String target = internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); addForwardReference(target, resolved -> { operation.addDependency(resolved); operation.addModifier(new ApplyMixin(resolved)); @@ -661,10 +671,10 @@ private void parseEnumShape(ShapeId id, SourceLocation location, AbstractShapeBu while (tokenizer.getCurrentToken() != IdlToken.EOF && tokenizer.getCurrentToken() != IdlToken.RBRACE) { List memberTraits = IdlTraitParser - .parseDocsAndTraitsBeforeShape(tokenizer, resolver); + .parseDocsAndTraitsBeforeShape(this); SourceLocation memberLocation = tokenizer.getCurrentTokenLocation(); tokenizer.expect(IdlToken.IDENTIFIER); - String memberName = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + String memberName = internString(tokenizer.getCurrentTokenLexeme()); MemberShape.Builder memberBuilder = MemberShape.builder() .id(id.withMember(memberName)) @@ -680,8 +690,11 @@ private void parseEnumShape(ShapeId id, SourceLocation location, AbstractShapeBu if (tokenizer.getCurrentToken() == IdlToken.EQUAL) { tokenizer.next(); tokenizer.skipSpaces(); - Node value = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + Node value = IdlNodeParser.expectAndSkipNode(this); memberBuilder.addTrait(new EnumValueTrait.Provider().createTrait(memberBuilder.getId(), value)); + // Explicit handling for an optional comma. + tokenizer.skipSpaces(); + tokenizer.skipOptionalComma(); tokenizer.expectAndSkipBr(); } else { tokenizer.skipWs(); @@ -722,7 +735,7 @@ private void parseMember(LoadOperation.DefineShape operation, Set define ShapeId parent = operation.toShapeId(); // Parse optional member traits. - List memberTraits = IdlTraitParser.parseDocsAndTraitsBeforeShape(tokenizer, resolver); + List memberTraits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this); SourceLocation memberLocation = tokenizer.getCurrentTokenLocation(); // Handle the case of a dangling documentation comment followed by "}". @@ -744,7 +757,7 @@ private void parseMember(LoadOperation.DefineShape operation, Set define } tokenizer.expect(IdlToken.IDENTIFIER); - String memberName = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + String memberName = internString(tokenizer.getCurrentTokenLexeme()); if (defined.contains(memberName)) { // This is a duplicate member name. @@ -769,7 +782,7 @@ private void parseMember(LoadOperation.DefineShape operation, Set define tokenizer.expect(IdlToken.COLON); tokenizer.next(); tokenizer.skipSpaces(); - String target = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + String target = internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); addForwardReference(target, memberBuilder::target); } @@ -783,8 +796,11 @@ private void parseMember(LoadOperation.DefineShape operation, Set define tokenizer.expect(IdlToken.EQUAL); tokenizer.next(); tokenizer.skipSpaces(); - Node node = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + Node node = IdlNodeParser.expectAndSkipNode(this); memberBuilder.addTrait(new DefaultTrait(node)); + // Explicit handling for an optional comma. + tokenizer.skipSpaces(); + tokenizer.skipOptionalComma(); tokenizer.expectAndSkipBr(); } @@ -796,12 +812,10 @@ private void parseMember(LoadOperation.DefineShape operation, Set define private void parseForResource(LoadOperation.DefineShape operation) { tokenizer.skipSpaces(); - if (!tokenizer.doesCurrentIdentifierStartWith('f')) { + if (!tokenizer.isCurrentLexeme("for")) { return; } - tokenizer.expectCurrentLexeme("for"); - if (!modelVersion.supportsTargetElision()) { throw syntax(operation.toShapeId(), "Structures can only be bound to resources with Smithy version 2 or " + "later. Attempted to bind a structure to a resource with version `" @@ -811,7 +825,7 @@ private void parseForResource(LoadOperation.DefineShape operation) { tokenizer.next(); tokenizer.expectAndSkipSpaces(); - String forTarget = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + String forTarget = internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); addForwardReference(forTarget, shapeId -> { operation.addDependency(shapeId); operation.addModifier(new ApplyResourceBasedTargets(shapeId)); @@ -824,7 +838,7 @@ private void parseServiceStatement(ShapeId id, SourceLocation location) { parseMixins(operation); tokenizer.skipWsAndDocs(); tokenizer.expect(IdlToken.LBRACE); - ObjectNode shapeNode = IdlNodeParser.expectAndSkipNode(tokenizer, resolver).expectObjectNode(); + ObjectNode shapeNode = IdlNodeParser.expectAndSkipNode(this).expectObjectNode(); LoaderUtils.checkForAdditionalProperties(shapeNode, id, SERVICE_PROPERTY_NAMES).ifPresent(this::emit); shapeNode.getStringMember(VERSION_KEY).map(StringNode::getValue).ifPresent(builder::version); optionalIdList(shapeNode, OPERATIONS_KEY, builder::addOperation); @@ -856,7 +870,7 @@ private void parseResourceStatement(ShapeId id, SourceLocation location) { parseMixins(operation); tokenizer.skipWsAndDocs(); tokenizer.expect(IdlToken.LBRACE); - ObjectNode shapeNode = IdlNodeParser.expectAndSkipNode(tokenizer, resolver).expectObjectNode(); + ObjectNode shapeNode = IdlNodeParser.expectAndSkipNode(this).expectObjectNode(); LoaderUtils.checkForAdditionalProperties(shapeNode, id, RESOURCE_PROPERTY_NAMES).ifPresent(this::emit); optionalId(shapeNode, PUT_KEY, builder::put); @@ -904,7 +918,7 @@ private void parseOperationStatement(ShapeId id, SourceLocation location) { Set defined = new HashSet<>(); while (tokenizer.hasNext() && tokenizer.getCurrentToken() != IdlToken.RBRACE) { tokenizer.expect(IdlToken.IDENTIFIER); - String key = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + String key = internString(tokenizer.getCurrentTokenLexeme()); if (!defined.add(key)) { throw syntax(id, String.format("Duplicate operation %s property for `%s`", key, id)); } @@ -963,13 +977,13 @@ private void parseInlineableOperationMember( consumer.accept(parseInlineStructure(id.getName() + suffix, defaultTrait)); } else { tokenizer.skipWsAndDocs(); - String target = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + String target = internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); addForwardReference(target, consumer); } } private ShapeId parseInlineStructure(String name, IdlTraitParser.Result defaultTrait) { - List traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(tokenizer, resolver); + List traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this); if (defaultTrait != null) { traits.add(defaultTrait); } @@ -982,7 +996,7 @@ private ShapeId parseInlineStructure(String name, IdlTraitParser.Result defaultT } private void parseIdList(Consumer consumer) { - tokenizer.increaseNestingLevel(); + increaseNestingLevel(); tokenizer.skipWsAndDocs(); tokenizer.expect(IdlToken.LBRACKET); tokenizer.next(); @@ -990,13 +1004,13 @@ private void parseIdList(Consumer consumer) { while (tokenizer.hasNext() && tokenizer.getCurrentToken() != IdlToken.RBRACKET) { tokenizer.expect(IdlToken.IDENTIFIER); - String target = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + String target = internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); addForwardReference(target, consumer); tokenizer.skipWsAndDocs(); } tokenizer.expect(IdlToken.RBRACKET); tokenizer.next(); - tokenizer.decreaseNestingLevel(); + decreaseNestingLevel(); } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java index e98e3154979..2aee9178a7b 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java @@ -29,7 +29,7 @@ import software.amazon.smithy.utils.Pair; /** - * Parses Node values from a {@link IdlTokenizer}. + * Parses Node values from a {@link IdlInternalTokenizer}. */ final class IdlNodeParser { @@ -42,13 +42,12 @@ private IdlNodeParser() { } * *

The tokenizer is advanced to the next token after parsing the Node value.

* - * @param tokenizer Tokenizer to consume and advance. - * @param resolver Forward reference resolver. + * @param loader IDL parser. * @return Returns the parsed node value. * @throws ModelSyntaxException if the Node is not well-formed. */ - static Node expectAndSkipNode(IdlTokenizer tokenizer, IdlReferenceResolver resolver) { - return expectAndSkipNode(tokenizer, resolver, tokenizer.getCurrentTokenLocation()); + static Node expectAndSkipNode(IdlModelLoader loader) { + return expectAndSkipNode(loader, loader.getTokenizer().getCurrentTokenLocation()); } /** @@ -57,13 +56,13 @@ static Node expectAndSkipNode(IdlTokenizer tokenizer, IdlReferenceResolver resol * *

The tokenizer is advanced to the next token after parsing the Node value.

* - * @param tokenizer Tokenizer to consume and advance. - * @param resolver Forward reference resolver. + * @param loader IDL loader. * @param location Source location to assign to the node. * @return Returns the parsed node value. * @throws ModelSyntaxException if the Node is not well-formed. */ - static Node expectAndSkipNode(IdlTokenizer tokenizer, IdlReferenceResolver resolver, SourceLocation location) { + static Node expectAndSkipNode(IdlModelLoader loader, SourceLocation location) { + IdlInternalTokenizer tokenizer = loader.getTokenizer(); IdlToken token = tokenizer.expect(IdlToken.STRING, IdlToken.TEXT_BLOCK, IdlToken.NUMBER, IdlToken.IDENTIFIER, IdlToken.LBRACE, IdlToken.LBRACKET); @@ -74,36 +73,32 @@ static Node expectAndSkipNode(IdlTokenizer tokenizer, IdlReferenceResolver resol tokenizer.next(); return result; case IDENTIFIER: - String shapeId = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); - return parseIdentifier(resolver, shapeId, location); + String shapeId = loader.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + return parseIdentifier(loader, shapeId, location); case NUMBER: Number number = tokenizer.getCurrentTokenNumberValue(); tokenizer.next(); return new NumberNode(number, location); case LBRACE: - return parseObjectNode(tokenizer, resolver, location); + return parseObjectNode(loader, location); case LBRACKET: default: - return parseArrayNode(tokenizer, resolver, location); + return parseArrayNode(loader, location); } } /** * Parse a Node identifier String, taking into account keywords and forward references. * - * @param resolver Forward reference resolver. + * @param loader IDL parser. * @param identifier Identifier to parse. * @param location Source location to assign to the identifier. * @return Returns the parsed identifier. */ - static Node parseIdentifier( - IdlReferenceResolver resolver, - String identifier, - SourceLocation location - ) { + static Node parseIdentifier(IdlModelLoader loader, String identifier, SourceLocation location) { Keyword keyword = Keyword.from(identifier); return keyword == null - ? parseSyntacticShapeId(resolver, identifier, location) + ? parseSyntacticShapeId(loader, identifier, location) : keyword.createNode(location); } @@ -144,7 +139,7 @@ static Keyword from(String keyword) { } private static Node parseSyntacticShapeId( - IdlReferenceResolver resolver, + IdlModelLoader loader, String identifier, SourceLocation location ) { @@ -152,7 +147,7 @@ private static Node parseSyntacticShapeId( // used because the shape ID may not be able to be resolved until after the entire model is loaded. Pair> pair = StringNode.createLazyString(identifier, location); Consumer consumer = pair.right; - resolver.resolve(identifier, (id, type) -> { + loader.addForwardReference(identifier, (id, type) -> { consumer.accept(id.toString()); if (type != null) { return null; @@ -170,14 +165,11 @@ private static Node parseSyntacticShapeId( return pair.left; } - private static ArrayNode parseArrayNode( - IdlTokenizer tokenizer, - IdlReferenceResolver resolver, - SourceLocation location - ) { - tokenizer.increaseNestingLevel(); + private static ArrayNode parseArrayNode(IdlModelLoader loader, SourceLocation location) { + loader.increaseNestingLevel(); ArrayNode.Builder builder = ArrayNode.builder().sourceLocation(location); + IdlInternalTokenizer tokenizer = loader.getTokenizer(); tokenizer.expect(IdlToken.LBRACKET); tokenizer.next(); tokenizer.skipWsAndDocs(); @@ -186,26 +178,23 @@ private static ArrayNode parseArrayNode( if (tokenizer.getCurrentToken() == IdlToken.RBRACKET) { break; } else { - builder.withValue(expectAndSkipNode(tokenizer, resolver)); + builder.withValue(expectAndSkipNode(loader)); tokenizer.skipWsAndDocs(); } } while (true); tokenizer.expect(IdlToken.RBRACKET); tokenizer.next(); - tokenizer.decreaseNestingLevel(); + loader.decreaseNestingLevel(); return builder.build(); } - private static ObjectNode parseObjectNode( - IdlTokenizer tokenizer, - IdlReferenceResolver resolver, - SourceLocation location - ) { + private static ObjectNode parseObjectNode(IdlModelLoader loader, SourceLocation location) { + IdlInternalTokenizer tokenizer = loader.getTokenizer(); tokenizer.expect(IdlToken.LBRACE); tokenizer.next(); tokenizer.skipWsAndDocs(); - tokenizer.increaseNestingLevel(); + loader.increaseNestingLevel(); ObjectNode.Builder builder = ObjectNode.builder().sourceLocation(location); while (tokenizer.hasNext()) { @@ -213,7 +202,7 @@ private static ObjectNode parseObjectNode( break; } - String key = tokenizer.internString(tokenizer.getCurrentTokenStringSlice()); + String key = loader.internString(tokenizer.getCurrentTokenStringSlice()); SourceLocation keyLocation = tokenizer.getCurrentTokenLocation(); tokenizer.next(); tokenizer.skipWsAndDocs(); @@ -221,7 +210,7 @@ private static ObjectNode parseObjectNode( tokenizer.next(); tokenizer.skipWsAndDocs(); - Node value = expectAndSkipNode(tokenizer, resolver); + Node value = expectAndSkipNode(loader); if (builder.hasMember(key)) { throw new ModelSyntaxException("Duplicate member: '" + key + '\'', keyLocation); } @@ -231,7 +220,7 @@ private static ObjectNode parseObjectNode( tokenizer.expect(IdlToken.RBRACE); tokenizer.next(); - tokenizer.decreaseNestingLevel(); + loader.decreaseNestingLevel(); return builder.build(); } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlReferenceResolver.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlReferenceResolver.java deleted file mode 100644 index a9e9ade4576..00000000000 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlReferenceResolver.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.smithy.model.loader; - -import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import software.amazon.smithy.model.shapes.ShapeId; -import software.amazon.smithy.model.shapes.ShapeType; -import software.amazon.smithy.model.validation.ValidationEvent; - -/** - * Resolves forward references for parsers, providing them resolved shape IDs and the shape type if found. - */ -@FunctionalInterface -interface IdlReferenceResolver { - /** - * Defer the resolution of a shape ID within a namespace. - * - * @param name Name of the shape to resolve. - * @param receiver Receiver that receives the resolved shape ID and type, or null if the shape was not found. - * The receiver can return a {@link ValidationEvent} if the shape could not be resolved, or it - * can return null if the shape was resolved. - */ - void resolve(String name, BiFunction receiver); - - /** - * Defer the resolution of a shape ID within a namespace. - * - * @param name Name of the shape to resolve. - * @param receiver Receiver that receives the resolved shape ID and type, or null if the shape was not found. - */ - default void resolve(String name, BiConsumer receiver) { - resolve(name, (id, type) -> { - receiver.accept(id, type); - return null; - }); - } -} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java index 8be8375cc96..dc9e350b1eb 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java @@ -16,7 +16,7 @@ package software.amazon.smithy.model.loader; /** - * Parses Shape ID lexemes from a {@link IdlTokenizer}. + * Parses Shape ID lexemes from a {@link IdlInternalTokenizer}. */ final class IdlShapeIdParser { @@ -31,7 +31,7 @@ private IdlShapeIdParser() { } * @return Returns the sequence of characters that make up a namespace. * @throws ModelSyntaxException if the tokenizer is unable to parse a valid namespace. */ - static CharSequence expectAndSkipShapeIdNamespace(IdlTokenizer tokenizer) { + static CharSequence expectAndSkipShapeIdNamespace(IdlInternalTokenizer tokenizer) { int startPosition = tokenizer.getCurrentTokenStart(); int endOffset = skipShapeIdNamespace(tokenizer); return sliceFrom(tokenizer, startPosition, endOffset); @@ -44,7 +44,7 @@ static CharSequence expectAndSkipShapeIdNamespace(IdlTokenizer tokenizer) { * @param tokenizer Tokenizer to consume and advance. * @return Returns the borrowed shape ID. */ - static CharSequence expectAndSkipAbsoluteShapeId(IdlTokenizer tokenizer) { + static CharSequence expectAndSkipAbsoluteShapeId(IdlInternalTokenizer tokenizer) { int startPosition = tokenizer.getCurrentTokenStart(); int endOffset = skipAbsoluteShapeId(tokenizer); return sliceFrom(tokenizer, startPosition, endOffset); @@ -57,7 +57,7 @@ static CharSequence expectAndSkipAbsoluteShapeId(IdlTokenizer tokenizer) { * @param tokenizer Tokenizer to consume and advance. * @return Returns the borrowed shape ID. */ - static CharSequence expectAndSkipShapeId(IdlTokenizer tokenizer) { + static CharSequence expectAndSkipShapeId(IdlInternalTokenizer tokenizer) { int startPosition = tokenizer.getCurrentTokenStart(); int offset = skipShapeIdNamespace(tokenizer); // Keep parsing if we find a $ or a #. @@ -69,11 +69,11 @@ static CharSequence expectAndSkipShapeId(IdlTokenizer tokenizer) { return sliceFrom(tokenizer, startPosition, offset); } - private static CharSequence sliceFrom(IdlTokenizer tokenizer, int startPosition, int endOffset) { - return tokenizer.getInput(startPosition, tokenizer.getPosition() - endOffset); + private static CharSequence sliceFrom(IdlInternalTokenizer tokenizer, int startPosition, int endOffset) { + return tokenizer.getModel(startPosition, tokenizer.getPosition() - endOffset); } - private static int skipShapeIdNamespace(IdlTokenizer tokenizer) { + private static int skipShapeIdNamespace(IdlInternalTokenizer tokenizer) { tokenizer.expect(IdlToken.IDENTIFIER); tokenizer.next(); // Keep track of how many characters from the end to omit (don't include "#" or whatever is next in the slice). @@ -87,14 +87,14 @@ private static int skipShapeIdNamespace(IdlTokenizer tokenizer) { return endOffset; } - private static int skipAbsoluteShapeId(IdlTokenizer tokenizer) { + private static int skipAbsoluteShapeId(IdlInternalTokenizer tokenizer) { skipShapeIdNamespace(tokenizer); tokenizer.expect(IdlToken.POUND); tokenizer.next(); return skipRelativeRootShapeId(tokenizer); } - private static int skipRelativeRootShapeId(IdlTokenizer tokenizer) { + private static int skipRelativeRootShapeId(IdlInternalTokenizer tokenizer) { tokenizer.expect(IdlToken.IDENTIFIER); tokenizer.next(); // Don't include whatever character comes next in the slice. diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlToken.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlToken.java index 9c09e422742..f906f4d47bb 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlToken.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlToken.java @@ -24,11 +24,6 @@ public enum IdlToken { SPACE(" ") { - @Override - public boolean canSkipBeforeBr() { - return true; - } - @Override public boolean isWhitespace() { return true; @@ -41,11 +36,6 @@ public boolean isWhitespace() { } }, COMMA(",") { - @Override - public boolean canSkipBeforeBr() { - return true; - } - @Override public boolean isWhitespace() { return true; @@ -98,13 +88,4 @@ public String getDebug(CharSequence lexeme) { public boolean isWhitespace() { return false; } - - /** - * Can this token be skipped when looking for a newline as part of a "BR" production? - * - * @return Returns true if the token can be skipped before a BR production. - */ - boolean canSkipBeforeBr() { - return false; - } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTokenizer.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTokenizer.java index e1c4183f461..6cb48af6e52 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTokenizer.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTokenizer.java @@ -15,162 +15,59 @@ package software.amazon.smithy.model.loader; -import java.math.BigDecimal; -import java.math.BigInteger; import java.nio.CharBuffer; -import java.util.ArrayDeque; -import java.util.Deque; import java.util.Iterator; -import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.function.Consumer; -import java.util.function.Function; import software.amazon.smithy.model.SourceLocation; -import software.amazon.smithy.model.validation.ValidationEvent; -import software.amazon.smithy.utils.SimpleParser; -import software.amazon.smithy.utils.SmithyBuilder; -import software.amazon.smithy.utils.SmithyUnstableApi; /** * Iterates over a Smithy IDL model as a series of tokens. */ -@SmithyUnstableApi -final class IdlTokenizer implements Iterator { +public interface IdlTokenizer extends Iterator { - /** Only allow nesting up to 64 arrays/objects in node values. */ - private static final int MAX_NESTING_LEVEL = 64; - - private final SimpleParser parser; - private final String filename; - private final Function stringTable; - - private IdlToken currentTokenType; - private int currentTokenStart = -1; - private int currentTokenEnd = -1; - private int currentTokenLine = -1; - private int currentTokenColumn = -1; - private Number currentTokenNumber; - private CharSequence currentTokenStringSlice; - private String currentTokenError; - private final Deque docCommentLines = new ArrayDeque<>(); - private final Consumer validationEventListener; - - private IdlTokenizer(Builder builder) { - this.filename = builder.filename; - this.stringTable = builder.stringTable; - this.parser = new SimpleParser(SmithyBuilder.requiredState("model", builder.model), MAX_NESTING_LEVEL); - this.validationEventListener = builder.validationEventListener; - } - - public static Builder builder() { - return new Builder(); - } - - public static final class Builder implements SmithyBuilder { - private String filename = SourceLocation.NONE.getFilename(); - private CharSequence model; - private Function stringTable = CharSequence::toString; - private Consumer validationEventListener = event -> { }; - - private Builder() { } - - /** - * The filename to associate with each token and errors. - * - *

Defaults to the filename of {@link SourceLocation#NONE}. - * - * @param filename Filename where the model comes from. - * @return Returns builder. - */ - public Builder filename(String filename) { - this.filename = Objects.requireNonNull(filename); - return this; - } - - /** - * Sets the required model IDL content to parse. - * - * @param model Model IDL content. - * @return Returns the builder. - */ - public Builder model(CharSequence model) { - this.model = model; - return this; - } - - /** - * Allows the use of a custom string table, used to convert {@link CharSequence}s into Strings. - * - *

Uses {@link CharSequence#toString()} by default. A string table cache is used when the tokenizer is - * used by {@link ModelAssembler}. - * - * @param stringTable String table to use. - * @return Returns the builder. - */ - public Builder stringTable(Function stringTable) { - this.stringTable = Objects.requireNonNull(stringTable); - return this; - } - - /** - * Sets a listener to receive warnings about syntax issues in the model. - * - *

This is currently package private and only used to warn about invalid documentation comments. - * - * @param validationEventListener Listener that receives warnings. - * @return Returns the builder. - */ - Builder validationEventListener(Consumer validationEventListener) { - this.validationEventListener = Objects.requireNonNull(validationEventListener); - return this; - } - - @Override - public IdlTokenizer build() { - return new IdlTokenizer(this); - } + /** + * Create a tokenizer for the given model. + * + * @param model IDL model contents to parse. + * @return Returns the tokenizer. + */ + static IdlTokenizer create(CharSequence model) { + return create(SourceLocation.NONE.getFilename(), model); } /** - * Get a borrowed slice of the input being parsed. + * Create a tokenizer for the given filename and model. * - * @param start Start position to get, inclusive. - * @param end End position to stop at, exclusive. - * @return Returns the slice. + * @param filename Filename being parsed. + * @param model IDL model contents to parse. + * @return Returns the tokenizer. */ - CharSequence getInput(int start, int end) { - return CharBuffer.wrap(parser.input(), start, end); + static IdlTokenizer create(String filename, CharSequence model) { + return new DefaultTokenizer(filename, model); } /** - * Intern a string and cache the interned value for subsequent retrieval. - * - *

This method should only be used with strings that are frequently used, for example, object keys, shape - * properties, namespaces, keywords (e.g., true, false, null, namespace, use, string, structure, trait IDs, etc). + * Get the filename of the content being tokenized. * - * @param sequence Characters to convert to a string. - * @return Returns the String representation of {@code sequence}. + * @return Returns the filename used in source locations. */ - public String internString(CharSequence sequence) { - return stringTable.apply(sequence); - } + String getSourceFilename(); /** - * Increase the nesting level of the tokenizer. + * Get the model being tokenized. + * + * @return Returns the model. */ - public void increaseNestingLevel() { - try { - parser.increaseNestingLevel(); - } catch (RuntimeException e) { - throw syntax("Parser exceeded maximum allowed depth of " + MAX_NESTING_LEVEL, getCurrentTokenLocation()); - } - } + CharSequence getModel(); /** - * Decrease the nesting level of the tokenizer. + * Get a borrowed slice of the model being tokenized. + * + * @param start Start position to get, inclusive. + * @param end End position to stop at, exclusive. + * @return Returns the slice. */ - public void decreaseNestingLevel() { - parser.decreaseNestingLevel(); + default CharSequence getModel(int start, int end) { + return CharBuffer.wrap(getModel(), start, end); } /** @@ -178,78 +75,64 @@ public void decreaseNestingLevel() { * * @return Returns the absolute position. */ - public int getPosition() { - return parser.position(); - } + int getPosition(); /** * Get the current line number of the tokenizer, starting at 1. * * @return Get the current line number. */ - public int getLine() { - return parser.line(); - } + int getLine(); /** * Get the current column number of the tokenizer, starting at 1. * * @return Get the current column number. */ - public int getColumn() { - return parser.column(); - } + int getColumn(); /** * Get the current {@link IdlToken}. * * @return Return the current token type. */ - public IdlToken getCurrentToken() { - if (currentTokenType == null) { - next(); - } - return currentTokenType; - } + IdlToken getCurrentToken(); /** * Get the line of the current token. * * @return Return the line of the current token. */ - public int getCurrentTokenLine() { - getCurrentToken(); - return currentTokenLine; - } + int getCurrentTokenLine(); /** * Get the column of the current token. * * @return Return the column of the current token. */ - public int getCurrentTokenColumn() { - getCurrentToken(); - return currentTokenColumn; - } + int getCurrentTokenColumn(); /** * Get the start position of the current token. * - * @return Return the start position of the current token. + * @return Return the 0-based start position of the current token. */ - public int getCurrentTokenStart() { - getCurrentToken(); - return currentTokenStart; - } + int getCurrentTokenStart(); + + /** + * Get the end position of the curren token. + * + * @return Return the 0-based end position of the current token. + */ + int getCurrentTokenEnd(); /** - * Get the span, or length, of the current token. + * Get the length of the current token. * * @return Return the current token span. */ - public int getCurrentTokenSpan() { - getCurrentToken(); - return currentTokenEnd - currentTokenStart; + default int getCurrentTokenSpan() { + return getCurrentTokenEnd() - getCurrentTokenStart(); } /** @@ -257,9 +140,8 @@ public int getCurrentTokenSpan() { * * @return Return the current token source location. */ - public SourceLocation getCurrentTokenLocation() { - getCurrentToken(); - return new SourceLocation(filename, currentTokenLine, currentTokenColumn); + default SourceLocation getCurrentTokenLocation() { + return new SourceLocation(getSourceFilename(), getCurrentTokenLine(), getCurrentTokenColumn()); } /** @@ -267,9 +149,8 @@ public SourceLocation getCurrentTokenLocation() { * * @return Returns the lexeme of the current token. */ - public CharSequence getCurrentTokenLexeme() { - getCurrentToken(); - return CharBuffer.wrap(parser.input(), currentTokenStart, currentTokenEnd); + default CharSequence getCurrentTokenLexeme() { + return getModel(getCurrentTokenStart(), getCurrentTokenEnd()); } /** @@ -279,17 +160,7 @@ public CharSequence getCurrentTokenLexeme() { * @return Returns the parsed string content associated with the current token. * @throws ModelSyntaxException if the current token is not a string, text block, or identifier. */ - public CharSequence getCurrentTokenStringSlice() { - getCurrentToken(); - if (currentTokenStringSlice != null) { - return currentTokenStringSlice; - } else if (currentTokenType == IdlToken.IDENTIFIER) { - return getCurrentTokenLexeme(); - } else { - throw syntax("The current token must be string or identifier but found: " - + currentTokenType.getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); - } - } + CharSequence getCurrentTokenStringSlice(); /** * If the current token is a number, get the associated parsed number. @@ -297,14 +168,7 @@ public CharSequence getCurrentTokenStringSlice() { * @return Returns the parsed number associated with the current token. * @throws ModelSyntaxException if the current token is not a number. */ - public Number getCurrentTokenNumberValue() { - getCurrentToken(); - if (currentTokenNumber == null) { - throw syntax("The current token must be number but found: " - + currentTokenType.getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); - } - return currentTokenNumber; - } + Number getCurrentTokenNumberValue(); /** * If the current token is an error, get the error message associated with the token. @@ -312,214 +176,7 @@ public Number getCurrentTokenNumberValue() { * @return Returns the associated error message. * @throws ModelSyntaxException if the current token is not an error. */ - public String getCurrentTokenError() { - getCurrentToken(); - if (currentTokenType != IdlToken.ERROR) { - throw syntax("The current token must be an error but found: " - + currentTokenType.getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); - } - return currentTokenError == null ? "" : currentTokenError; - } - - @Override - public boolean hasNext() { - return currentTokenType != IdlToken.EOF; - } - - @Override - public IdlToken next() { - currentTokenStringSlice = null; - currentTokenNumber = null; - currentTokenColumn = parser.column(); - currentTokenLine = parser.line(); - currentTokenStart = parser.position(); - currentTokenEnd = currentTokenStart; - int c = parser.peek(); - - switch (c) { - case SimpleParser.EOF: - if (currentTokenType == IdlToken.EOF) { - throw new NoSuchElementException("Expected another token but reached EOF"); - } - currentTokenEnd = parser.position(); - return currentTokenType = IdlToken.EOF; - case ' ': - case '\t': - return tokenizeSpace(); - case '\r': - case '\n': - return tokenizeNewline(); - case ',': - return singleCharToken(IdlToken.COMMA); - case '@': - return singleCharToken(IdlToken.AT); - case '$': - return singleCharToken(IdlToken.DOLLAR); - case '.': - return singleCharToken(IdlToken.DOT); - case '{': - return singleCharToken(IdlToken.LBRACE); - case '}': - return singleCharToken(IdlToken.RBRACE); - case '[': - return singleCharToken(IdlToken.LBRACKET); - case ']': - return singleCharToken(IdlToken.RBRACKET); - case '(': - return singleCharToken(IdlToken.LPAREN); - case ')': - return singleCharToken(IdlToken.RPAREN); - case '#': - return singleCharToken(IdlToken.POUND); - case '=': - return singleCharToken(IdlToken.EQUAL); - case ':': - return parseColon(); - case '"': - return parseString(); - case '/': - return parseComment(); - case '-': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - return parseNumber(); - case 'A': - case 'B': - case 'C': - case 'D': - case 'E': - case 'F': - case 'G': - case 'H': - case 'I': - case 'J': - case 'K': - case 'L': - case 'M': - case 'N': - case 'O': - case 'P': - case 'Q': - case 'R': - case 'S': - case 'T': - case 'U': - case 'V': - case 'W': - case 'X': - case 'Y': - case 'Z': - case '_': - case 'a': - case 'b': - case 'c': - case 'd': - case 'e': - case 'f': - case 'g': - case 'h': - case 'i': - case 'j': - case 'k': - case 'l': - case 'm': - case 'n': - case 'o': - case 'p': - case 'q': - case 'r': - case 's': - case 't': - case 'u': - case 'v': - case 'w': - case 'x': - case 'y': - case 'z': - return parseIdentifier(); - default: - currentTokenError = "Unexpected character: '" + ((char) c) + '\''; - return singleCharToken(IdlToken.ERROR); - } - } - - /** - * Skip spaces. - */ - public void skipSpaces() { - getCurrentToken(); - while (currentTokenType == IdlToken.SPACE) { - next(); - } - } - - /** - * Skip until the current token is not {@link IdlToken#SPACE}, {@link IdlToken#COMMA}, {@link IdlToken#COMMENT}, - * or {@link IdlToken#NEWLINE}. - */ - public void skipWs() { - getCurrentToken(); - while (currentTokenType.isWhitespace()) { - next(); - } - } - - /** - * Skip until the current token is not {@link IdlToken#SPACE}, {@link IdlToken#COMMA}, {@link IdlToken#COMMENT}, - * or {@link IdlToken#NEWLINE}, or {@link IdlToken#DOC_COMMENT}. - */ - public void skipWsAndDocs() { - getCurrentToken(); - while ((currentTokenType.isWhitespace() || currentTokenType == IdlToken.DOC_COMMENT)) { - next(); - } - } - - /** - * Check if the current token is an identifier, and if its lexeme starts with the given character {@code c}. - * - *

This method does not throw if the current token is not an identifier. - * - * @return Returns true if the token is an identifier that starts with {@code c}. - */ - public boolean doesCurrentIdentifierStartWith(char c) { - getCurrentToken(); - return currentTokenType == IdlToken.IDENTIFIER - && currentTokenEnd > currentTokenStart - && parser.input().charAt(currentTokenStart) == c; - } - - /** - * Expects that the lexeme of the current token is the same as the given characters. - * - * @param chars Characters to compare the current lexeme against. - * @return Returns the current lexeme. - * @throws ModelSyntaxException if the current lexeme is not equal to the given characters. - */ - public CharSequence expectCurrentLexeme(CharSequence chars) { - CharSequence lexeme = getCurrentTokenLexeme(); - boolean isError = lexeme.length() != chars.length(); - if (!isError) { - for (int i = 0; i < chars.length(); i++) { - if (lexeme.charAt(i) != chars.charAt(i)) { - isError = true; - break; - } - } - } - if (isError) { - throw syntax("Expected `" + chars + "`, but found `" + lexeme + "`", getCurrentTokenLocation()); - } - return lexeme; - } + String getCurrentTokenError(); /** * Assert that the current token is {@code token}. @@ -529,11 +186,9 @@ public CharSequence expectCurrentLexeme(CharSequence chars) { * @param token Token to expect. * @throws ModelSyntaxException if the current token is unexpected. */ - public void expect(IdlToken token) { - getCurrentToken(); - - if (currentTokenType != token) { - throw syntax(createExpectMessage(token), getCurrentTokenLocation()); + default void expect(IdlToken token) { + if (getCurrentToken() != token) { + throw LoaderUtils.idlSyntaxError(LoaderUtils.idlExpectMessage(this, token), getCurrentTokenLocation()); } } @@ -546,12 +201,8 @@ public void expect(IdlToken token) { * @return Returns the current token. * @throws ModelSyntaxException if the current token is unexpected. */ - public IdlToken expect(IdlToken... tokens) { - getCurrentToken(); - - if (currentTokenType == IdlToken.ERROR) { - throw syntax(createExpectMessage(tokens), getCurrentTokenLocation()); - } + default IdlToken expect(IdlToken... tokens) { + IdlToken currentTokenType = getCurrentToken(); for (IdlToken token : tokens) { if (currentTokenType == token) { @@ -559,305 +210,26 @@ public IdlToken expect(IdlToken... tokens) { } } - throw syntax(createExpectMessage(tokens), getCurrentTokenLocation()); - } - - private String createExpectMessage(IdlToken... tokens) { - StringBuilder result = new StringBuilder(); - if (currentTokenType == IdlToken.ERROR) { - result.append(getCurrentTokenError()); - } else if (tokens.length == 1) { - result.append("Expected ") - .append(tokens[0].getDebug()) - .append(" but found ") - .append(getCurrentToken().getDebug(getCurrentTokenLexeme())); - } else { - result.append("Expected one of "); - for (IdlToken token : tokens) { - result.append(token.getDebug()).append(", "); - } - result.delete(result.length() - 2, result.length()); - result.append("; but found ").append(getCurrentToken().getDebug(getCurrentTokenLexeme())); - } - return result.toString(); - } - - /** - * Expect that one or more spaces are found at the current token, and skip over them. - * - * @throws ModelSyntaxException if the current token is not a space. - */ - public void expectAndSkipSpaces() { - if (getCurrentToken() != IdlToken.SPACE) { - throw syntax("Expected one or more spaces, but found " - + getCurrentToken().getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); - } - skipSpaces(); - } - - private ModelSyntaxException syntax(String message, SourceLocation location) { - return new ModelSyntaxException("Syntax error at line " + location.getLine() + ", column " - + location.getColumn() + ": " + message, location); + throw LoaderUtils.idlSyntaxError(LoaderUtils.idlExpectMessage(this, tokens), getCurrentTokenLocation()); } /** - * Expect that one or more whitespace characters or documentation comments are found at the current token, and - * skip over them. - * - * @throws ModelSyntaxException if the current token is not whitespace. - */ - public void expectAndSkipWhitespace() { - if (!getCurrentToken().isWhitespace()) { - throw syntax("Expected one or more whitespace characters, but found " - + getCurrentToken().getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); - } - skipWsAndDocs(); - } - - /** - * Expects that the current token is zero or more spaces/commas followed by a newline, comment, documentation - * comment, or EOF. - * - *

If a documentation comment is detected, the current token remains the documentation comment. If an EOF is - * detected, the current token remains the EOF token. If a comment, newline, or whitespace are detected, they are - * all skipped, leaving the current token the next token after encountering the matched token. Other kinds of - * tokens will raise an exception. + * Test if the current token lexeme is equal to the give {@code chars}. * - *

This method mimics the {@code br} production from the Smithy grammar. When this method is called, any - * previously parsed and buffered documentation comments are discarded. - * - * @throws ModelSyntaxException if the current token is not a newline or followed by a newline. - */ - public void expectAndSkipBr() { - // Skip spaces and commas leading up to a required newline. - getCurrentToken(); - while (currentTokenType.canSkipBeforeBr()) { - next(); - } - - // The following tokens are allowed tokens that contain newlines. - switch (getCurrentToken()) { - case NEWLINE: - case COMMENT: - case DOC_COMMENT: - clearDocCommentLinesForBr(); - next(); - skipWs(); - break; - case EOF: - break; - default: - throw syntax("Expected a line break, but found " - + getCurrentToken().getDebug(getCurrentTokenLexeme()), - getCurrentTokenLocation()); - } - } - - private void clearDocCommentLinesForBr() { - if (!docCommentLines.isEmpty()) { - validationEventListener.accept(LoaderUtils.emitBadDocComment(getCurrentTokenLocation(), - removePendingDocCommentLines())); - } - } - - private IdlToken singleCharToken(IdlToken type) { - parser.skip(); - currentTokenEnd = parser.position(); - return currentTokenType = type; - } - - private IdlToken tokenizeNewline() { - parser.skip(); // this will \n and \r\n. - currentTokenEnd = parser.position(); - return currentTokenType = IdlToken.NEWLINE; - } - - private IdlToken tokenizeSpace() { - parser.consumeWhile(c -> c == ' ' || c == '\t'); - currentTokenEnd = parser.position(); - return currentTokenType = IdlToken.SPACE; - } - - private IdlToken parseColon() { - parser.skip(); - - if (parser.peek() == '=') { - parser.skip(); - currentTokenType = IdlToken.WALRUS; - } else { - currentTokenType = IdlToken.COLON; - } - - currentTokenEnd = parser.position(); - return currentTokenType; - } - - private IdlToken parseComment() { - // first "/". - parser.expect('/'); - - // A standalone forward slash is an error. - if (parser.peek() != '/') { - currentTokenError = "Expected a '/' to follow '/' to form a comment."; - return singleCharToken(IdlToken.ERROR); - } - - // Skip the next "/". - parser.expect('/'); - - IdlToken type = IdlToken.COMMENT; - - // Three "///" is a documentation comment. - if (parser.peek() == '/') { - parser.expect('/'); - type = IdlToken.DOC_COMMENT; - // When capturing the doc comment lexeme, skip a single leading space if found. - if (parser.peek() == ' ') { - parser.skip(); - } - int lineStart = parser.position(); - parser.consumeRemainingCharactersOnLine(); - docCommentLines.add(getInput(lineStart, parser.position())); - } else { - parser.consumeRemainingCharactersOnLine(); - } - - // Include the newline in the comment and doc comment lexeme. - if (parser.expect('\r', '\n', SimpleParser.EOF) == '\r' && parser.peek() == '\n') { - parser.skip(); - } - - currentTokenEnd = parser.position(); - return currentTokenType = type; - } - - /** - * Removes any buffered documentation comment lines, and returns a concatenated string value. - * - * @return Returns the combined documentation comment string for the given lines. Returns null if no lines. + * @param chars Characters to compare the current lexeme against. + * @return Returns true if the current lexeme is equal to {@code chars}. */ - String removePendingDocCommentLines() { - if (docCommentLines.isEmpty()) { - return null; - } else { - StringBuilder result = new StringBuilder(); - result.append(docCommentLines.removeFirst()); - while (!docCommentLines.isEmpty()) { - result.append('\n').append(docCommentLines.removeFirst()); - } - return result.toString(); - } - } - - private IdlToken parseNumber() { - try { - String lexeme = ParserUtils.parseNumber(parser); - if (lexeme.contains("e") || lexeme.contains("E") || lexeme.contains(".")) { - double value = Double.parseDouble(lexeme); - if (Double.isFinite(value)) { - currentTokenNumber = value; - } else { - currentTokenNumber = new BigDecimal(lexeme); - } - } else { - try { - currentTokenNumber = Long.parseLong(lexeme); - } catch (NumberFormatException e) { - currentTokenNumber = new BigInteger(lexeme); - } - } - - currentTokenEnd = parser.position(); - return currentTokenType = IdlToken.NUMBER; - } catch (RuntimeException e) { - currentTokenEnd = parser.position(); - // Strip off the leading error message information if present. - if (e.getMessage().startsWith("Syntax error")) { - currentTokenError = e.getMessage().substring(e.getMessage().indexOf(':') + 1).trim(); - } else { - currentTokenError = e.getMessage(); - } - return currentTokenType = IdlToken.ERROR; - } - } - - private IdlToken parseIdentifier() { - try { - ParserUtils.consumeIdentifier(parser); - currentTokenType = IdlToken.IDENTIFIER; - } catch (RuntimeException e) { - currentTokenType = IdlToken.ERROR; - currentTokenError = e.getMessage(); - } - currentTokenEnd = parser.position(); - return currentTokenType; - } - - private IdlToken parseString() { - parser.skip(); // skip first quote. - - if (parser.peek() == '"') { - parser.skip(); // skip second quote. - if (parser.peek() == '"') { // A third consecutive quote is a TEXT_BLOCK. - parser.skip(); - return parseTextBlock(); - } else { - // Empty string. - currentTokenEnd = parser.position(); - currentTokenStringSlice = ""; - return currentTokenType = IdlToken.STRING; - } - } - - try { - // Parse the contents of a quoted string. - currentTokenStringSlice = parseQuotedTextAndTextBlock(false); - currentTokenEnd = parser.position(); - return currentTokenType = IdlToken.STRING; - } catch (RuntimeException e) { - currentTokenEnd = parser.position(); - currentTokenError = "Error parsing quoted string: " + e.getMessage(); - return currentTokenType = IdlToken.ERROR; - } - } - - private IdlToken parseTextBlock() { - try { - currentTokenStringSlice = parseQuotedTextAndTextBlock(true); - currentTokenEnd = parser.position(); - return currentTokenType = IdlToken.TEXT_BLOCK; - } catch (RuntimeException e) { - currentTokenEnd = parser.position(); - currentTokenError = "Error parsing text block: " + e.getMessage(); - return currentTokenType = IdlToken.ERROR; + default boolean isCurrentLexeme(CharSequence chars) { + CharSequence lexeme = getCurrentTokenLexeme(); + int testLength = chars.length(); + if (lexeme.length() != testLength) { + return false; } - } - - // Parses both quoted_text and text_block - private CharSequence parseQuotedTextAndTextBlock(boolean triple) { - int start = parser.position(); - - while (!parser.eof()) { - char next = parser.peek(); - if (next == '"' && (!triple || (parser.peek(1) == '"' && parser.peek(2) == '"'))) { - // Found closing quotes of quoted_text and/or text_block - break; - } - parser.skip(); - if (next == '\\') { - parser.skip(); + for (int i = 0; i < testLength; i++) { + if (lexeme.charAt(i) != chars.charAt(i)) { + return false; } } - - // Strip the ending '"'. - CharSequence result = parser.borrowSliceFrom(start); - parser.expect('"'); - - if (triple) { - parser.expect('"'); - parser.expect('"'); - } - - return IdlStringLexer.scanStringContents(result, triple); + return true; } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java index 88fcda647cc..0d8006e2b79 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java @@ -66,11 +66,11 @@ private IdlTraitParser() { } * comments and applied traits into a list of {@link Result} values that can later be turned into instances of * {@link Trait}s to apply to shapes. * - * @param tokenizer Tokenizer to consume and advance. - * @param resolver Forward reference resolver. + * @param loader IDL parser. * @return Return the parsed traits. */ - static List parseDocsAndTraitsBeforeShape(IdlTokenizer tokenizer, IdlReferenceResolver resolver) { + static List parseDocsAndTraitsBeforeShape(IdlModelLoader loader) { + IdlInternalTokenizer tokenizer = loader.getTokenizer(); tokenizer.skipWs(); Result docComment = null; @@ -86,7 +86,7 @@ static List parseDocsAndTraitsBeforeShape(IdlTokenizer tokenizer, IdlRef // Parse traits, if any. tokenizer.skipWsAndDocs(); - List traits = expectAndSkipTraits(tokenizer, resolver); + List traits = expectAndSkipTraits(loader); if (docComment != null) { traits.add(docComment); } @@ -95,7 +95,7 @@ static List parseDocsAndTraitsBeforeShape(IdlTokenizer tokenizer, IdlRef return traits; } - private static Result parseDocComment(IdlTokenizer tokenizer, SourceLocation location) { + private static Result parseDocComment(IdlInternalTokenizer tokenizer, SourceLocation location) { String result = tokenizer.removePendingDocCommentLines(); if (result == null) { return null; @@ -108,14 +108,14 @@ private static Result parseDocComment(IdlTokenizer tokenizer, SourceLocation loc /** * Parse all traits before a shape or member, or inside an {@code apply} block. * - * @param tokenizer Tokenizer to consume and advance. - * @param resolver Forward reference resolver. + * @param loader IDL parser. * @return Returns the parsed traits. */ - static List expectAndSkipTraits(IdlTokenizer tokenizer, IdlReferenceResolver resolver) { + static List expectAndSkipTraits(IdlModelLoader loader) { List results = new ArrayList<>(); + IdlInternalTokenizer tokenizer = loader.getTokenizer(); while (tokenizer.getCurrentToken() == IdlToken.AT) { - results.add(expectAndSkipTrait(tokenizer, resolver)); + results.add(expectAndSkipTrait(loader)); tokenizer.skipWsAndDocs(); } return results; @@ -124,12 +124,12 @@ static List expectAndSkipTraits(IdlTokenizer tokenizer, IdlReferenceReso /** * Parses a single trait: "@" trait-id [(trait-body)]. * - * @param tokenizer Tokenizer to consume and advance. - * @param resolver Forward reference resolver. + * @param loader IDL parser. * @return Returns the parsed trait. */ - static Result expectAndSkipTrait(IdlTokenizer tokenizer, IdlReferenceResolver resolver) { + static Result expectAndSkipTrait(IdlModelLoader loader) { // "@" shape_id + IdlInternalTokenizer tokenizer = loader.getTokenizer(); SourceLocation location = tokenizer.getCurrentTokenLocation(); tokenizer.expect(IdlToken.AT); tokenizer.next(); @@ -150,7 +150,7 @@ static Result expectAndSkipTrait(IdlTokenizer tokenizer, IdlReferenceResolver re } // The trait has a value between the '(' and ')'. - Node value = parseTraitValueBody(tokenizer, resolver, location); + Node value = parseTraitValueBody(loader, location); tokenizer.skipWsAndDocs(); tokenizer.expect(IdlToken.RPAREN); tokenizer.next(); @@ -158,18 +158,15 @@ static Result expectAndSkipTrait(IdlTokenizer tokenizer, IdlReferenceResolver re return new Result(id, value, TraitType.VALUE); } - private static Node parseTraitValueBody( - IdlTokenizer tokenizer, - IdlReferenceResolver resolver, - SourceLocation location - ) { + private static Node parseTraitValueBody(IdlModelLoader loader, SourceLocation location) { + IdlInternalTokenizer tokenizer = loader.getTokenizer(); tokenizer.expect(IdlToken.LBRACE, IdlToken.LBRACKET, IdlToken.TEXT_BLOCK, IdlToken.STRING, IdlToken.NUMBER, IdlToken.IDENTIFIER); switch (tokenizer.getCurrentToken()) { case LBRACE: case LBRACKET: - Node result = IdlNodeParser.expectAndSkipNode(tokenizer, resolver, location); + Node result = IdlNodeParser.expectAndSkipNode(loader, location); tokenizer.skipWsAndDocs(); return result; case TEXT_BLOCK: @@ -190,33 +187,30 @@ private static Node parseTraitValueBody( if (tokenizer.getCurrentToken() == IdlToken.COLON) { tokenizer.next(); tokenizer.skipWsAndDocs(); - return parseStructuredTrait(tokenizer, resolver, stringNode); + return parseStructuredTrait(loader, stringNode); } else { return stringNode; } case IDENTIFIER: default: - String identifier = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + String identifier = loader.internString(tokenizer.getCurrentTokenLexeme()); tokenizer.next(); tokenizer.skipWsAndDocs(); if (tokenizer.getCurrentToken() == IdlToken.COLON) { tokenizer.next(); tokenizer.skipWsAndDocs(); - return parseStructuredTrait(tokenizer, resolver, new StringNode(identifier, location)); + return parseStructuredTrait(loader, new StringNode(identifier, location)); } else { - return IdlNodeParser.parseIdentifier(resolver, identifier, location); + return IdlNodeParser.parseIdentifier(loader, identifier, location); } } } - private static ObjectNode parseStructuredTrait( - IdlTokenizer tokenizer, - IdlReferenceResolver resolver, - StringNode firstKey - ) { - tokenizer.increaseNestingLevel(); + private static ObjectNode parseStructuredTrait(IdlModelLoader loader, StringNode firstKey) { + IdlInternalTokenizer tokenizer = loader.getTokenizer(); + loader.increaseNestingLevel(); Map entries = new LinkedHashMap<>(); - Node firstValue = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + Node firstValue = IdlNodeParser.expectAndSkipNode(loader); // This put call can be done safely without checking for duplicates, // as it's always the first member of the trait. @@ -225,14 +219,14 @@ private static ObjectNode parseStructuredTrait( while (tokenizer.getCurrentToken() != IdlToken.RPAREN) { tokenizer.expect(IdlToken.IDENTIFIER, IdlToken.STRING); - String key = tokenizer.internString(tokenizer.getCurrentTokenStringSlice()); + String key = loader.internString(tokenizer.getCurrentTokenStringSlice()); StringNode keyNode = new StringNode(key, tokenizer.getCurrentTokenLocation()); tokenizer.next(); tokenizer.skipWsAndDocs(); tokenizer.expect(IdlToken.COLON); tokenizer.next(); tokenizer.skipWsAndDocs(); - Node nextValue = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + Node nextValue = IdlNodeParser.expectAndSkipNode(loader); Node previous = entries.put(keyNode, nextValue); if (previous != null) { throw new ModelSyntaxException("Duplicate member of trait: '" + keyNode.getValue() + '\'', keyNode); @@ -240,7 +234,7 @@ private static ObjectNode parseStructuredTrait( tokenizer.skipWsAndDocs(); } - tokenizer.decreaseNestingLevel(); + loader.decreaseNestingLevel(); return new ObjectNode(entries, firstKey.getSourceLocation()); } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoaderUtils.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoaderUtils.java index f680a3d6a8e..3a07c44da15 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoaderUtils.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoaderUtils.java @@ -15,6 +15,8 @@ package software.amazon.smithy.model.loader; +import static java.lang.String.format; + import java.util.Collection; import java.util.List; import java.util.Optional; @@ -123,4 +125,38 @@ static ValidationEvent emitBadDocComment(SourceLocation location, String comment .sourceLocation(location) .build(); } + + static String idlExpectMessage(IdlTokenizer tokenizer, IdlToken... tokens) { + StringBuilder result = new StringBuilder(); + IdlToken current = tokenizer.getCurrentToken(); + if (current == IdlToken.ERROR) { + result.append(tokenizer.getCurrentTokenError()); + } else if (tokens.length == 1) { + result.append("Expected ") + .append(tokens[0].getDebug()) + .append(" but found ") + .append(current.getDebug(tokenizer.getCurrentTokenLexeme())); + } else { + result.append("Expected one of "); + for (IdlToken token : tokens) { + result.append(token.getDebug()).append(", "); + } + result.delete(result.length() - 2, result.length()); + result.append("; but found ").append(current.getDebug(tokenizer.getCurrentTokenLexeme())); + } + return result.toString(); + } + + static ModelSyntaxException idlSyntaxError(String message, SourceLocation location) { + return idlSyntaxError(null, message, location); + } + + static ModelSyntaxException idlSyntaxError(ShapeId shape, String message, SourceLocation location) { + return ModelSyntaxException.builder() + .message(format("Syntax error at line %d, column %d: %s", + location.getLine(), location.getColumn(), message)) + .sourceLocation(location) + .shapeId(shape) + .build(); + } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/StringTable.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/StringTable.java index 54ff56859ea..6712fe12efb 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/StringTable.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/StringTable.java @@ -23,7 +23,7 @@ * *

The implementation uses an FNV-1a hash, and collisions simply overwrite the previously cached value. */ -final class StringTable implements Function { +public final class StringTable implements Function { private static final int FNV_OFFSET_BIAS = 0x811c9dc5; private static final int FNV_PRIME = 0x1000193; @@ -33,12 +33,20 @@ final class StringTable implements Function { private final int size; private final int sizeMask; - StringTable() { - // Defaults to 1024 entries. + /** + * Create a string table with 2048 entries. + */ + public StringTable() { + // Defaults to 2048 entries. this(11); } - StringTable(int sizeBits) { + /** + * Create a string table with a specific number of entries. + * + * @param sizeBits Size of the table based on bit shifting (e.g., 1 -> 2, 2 -> 4, ..., 10 -> 1024, 11 -> 2048). + */ + public StringTable(int sizeBits) { if (sizeBits <= 0) { throw new IllegalArgumentException("Cache sizeBits must be >= 1"); } else if (sizeBits >= 17) { diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlInternalTokenizerTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlInternalTokenizerTest.java new file mode 100644 index 00000000000..dfce68688a8 --- /dev/null +++ b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlInternalTokenizerTest.java @@ -0,0 +1,184 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; + +import org.hamcrest.MatcherAssert; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class IdlInternalTokenizerTest { + + @Test + public void skipSpaces() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", " hi"); + + tokenizer.skipSpaces(); + + MatcherAssert.assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + MatcherAssert.assertThat(tokenizer.getCurrentTokenColumn(), is(5)); + } + + @Test + public void skipsExpectedSpaces() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", " hi"); + + tokenizer.expectAndSkipSpaces(); + + MatcherAssert.assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + MatcherAssert.assertThat(tokenizer.getCurrentTokenColumn(), is(5)); + } + + @Test + public void failsWhenExpectedSpacesNotThere() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", "abc"); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, tokenizer::expectAndSkipSpaces); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected SPACE(' ') but found " + + "IDENTIFIER('abc')")); + } + + @Test + public void skipWhitespace() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", " \n\n hi"); + + tokenizer.skipWs(); + + MatcherAssert.assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + MatcherAssert.assertThat(tokenizer.getCurrentTokenLine(), is(3)); + MatcherAssert.assertThat(tokenizer.getCurrentTokenColumn(), is(2)); + } + + @Test + public void expectAndSkipWhitespace() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", " \n\n hi"); + + tokenizer.expectAndSkipWhitespace(); + + MatcherAssert.assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + MatcherAssert.assertThat(tokenizer.getCurrentTokenLine(), is(3)); + MatcherAssert.assertThat(tokenizer.getCurrentTokenColumn(), is(2)); + } + + @Test + public void throwsWhenExpectedWhitespaceNotFound() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", "hi"); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, + tokenizer::expectAndSkipWhitespace); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected one or more whitespace " + + "characters, but found IDENTIFIER('hi')")); + } + + @Test + public void skipDocsAndWhitespace() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", + " \n\n /// Docs\n/// Docs\n\n hi"); + + tokenizer.skipWsAndDocs(); + + MatcherAssert.assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + MatcherAssert.assertThat(tokenizer.getCurrentTokenLine(), is(6)); + MatcherAssert.assertThat(tokenizer.getCurrentTokenColumn(), is(2)); + } + + @Test + public void expectsAndSkipsBr() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", "\n Hi"); + + tokenizer.expectAndSkipBr(); + + MatcherAssert.assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + MatcherAssert.assertThat(tokenizer.getCurrentTokenLine(), is(2)); + MatcherAssert.assertThat(tokenizer.getCurrentTokenColumn(), is(3)); + } + + @Test + public void throwsWhenBrNotFound() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", "Hi"); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, + tokenizer::expectAndSkipBr); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected a line break, but " + + "found IDENTIFIER('Hi')")); + } + + @Test + public void testsCurrentTokenLexeme() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", "Hi"); + + assertThat(tokenizer.isCurrentLexeme("Hi"), is(true)); + } + + @Test + public void expectsSingleTokenType() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", "Hi"); + + tokenizer.expect(IdlToken.IDENTIFIER); + } + + @Test + public void failsForSingleExpectedToken() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", "Hi"); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, + () -> tokenizer.expect(IdlToken.NUMBER)); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected NUMBER but " + + "found IDENTIFIER('Hi')")); + } + + @Test + public void expectsMultipleTokenTypes() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", "Hi"); + + tokenizer.expect(IdlToken.STRING, IdlToken.IDENTIFIER); + } + + @Test + public void failsForMultipleExpectedTokens() { + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", "Hi"); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, + () -> tokenizer.expect(IdlToken.NUMBER, IdlToken.LBRACE)); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected one of NUMBER, LBRACE('{'); " + + "but found IDENTIFIER('Hi')")); + } + + @Test + public void returnsCapturedDocsInRange() { + String model = "/// Hi\n" + + "/// There\n" + + "/// 123\n" + + "/// 456\n"; + IdlInternalTokenizer tokenizer = new IdlInternalTokenizer("a.smithy", model); + + tokenizer.skipWsAndDocs(); + String lines = tokenizer.removePendingDocCommentLines(); + + assertThat(lines, equalTo("Hi\nThere\n123\n456")); + MatcherAssert.assertThat(tokenizer.removePendingDocCommentLines(), nullValue()); + } +} diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java index e457b638936..9e8da58561e 100644 --- a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java +++ b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java @@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.ArrayList; @@ -172,7 +173,10 @@ public void handlesMultilineDocComments() { .unwrap(); Shape shape = model.expectShape(ShapeId.from("smithy.example#MyStruct$myMember")); - String docs = shape.getTrait(DocumentationTrait.class).map(StringTrait::getValue).orElse(""); + String docs = shape.getTrait(DocumentationTrait.class) + .map(StringTrait::getValue) + .orElse("") + .replace("\r\n", "\n"); assertThat(docs, equalTo("This is the first line.\nThis is the second line.")); } @@ -473,4 +477,18 @@ public void doesBasicErrorRecoveryInMetadata() { assertThat(foundSyntax, equalTo(4L)); } + + @Test + public void throwsWhenTooNested() { + IdlModelLoader loader = new IdlModelLoader("foo.smithy", "", new StringTable()); + + for (int i = 0; i < 64; i++) { + loader.increaseNestingLevel(); + } + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, loader::increaseNestingLevel); + + assertThat(e.getMessage(), + startsWith("Syntax error at line 1, column 1: Parser exceeded maximum allowed depth of 64")); + } } diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/TokenizerTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/TokenizerTest.java index dab187dd1e1..9d1d8d34664 100644 --- a/smithy-model/src/test/java/software/amazon/smithy/model/loader/TokenizerTest.java +++ b/smithy-model/src/test/java/software/amazon/smithy/model/loader/TokenizerTest.java @@ -16,16 +16,10 @@ package software.amazon.smithy.model.loader; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.startsWith; -import java.util.ArrayList; -import java.util.List; import java.util.NoSuchElementException; -import java.util.function.Function; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -33,7 +27,7 @@ public class TokenizerTest { @Test public void tokenizesIdentifierFollowedByQuote() { String contents = "metadata\"foo\"=\"bar\""; - IdlTokenizer tokenizer = IdlTokenizer.builder().model(contents).build(); + IdlTokenizer tokenizer = IdlTokenizer.create(contents); tokenizer.next(); assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); @@ -64,7 +58,7 @@ public void tokenizesIdentifierFollowedByQuote() { @Test public void tokenizesSingleCharacterLexemes() { String contents = "\t\r\n\r,@$.:{}()[]# "; - IdlTokenizer tokenizer = IdlTokenizer.builder().model(contents).build(); + IdlTokenizer tokenizer = IdlTokenizer.create(contents); tokenizer.next(); assertThat(tokenizer.getCurrentToken(), is(IdlToken.SPACE)); @@ -171,7 +165,7 @@ public void tokenizesSingleCharacterLexemes() { @Test public void tokenizesMultipleSpacesAndTabsIntoSingleLexeme() { String contents = " \t "; - IdlTokenizer tokenizer = IdlTokenizer.builder().model(contents).build(); + IdlTokenizer tokenizer = IdlTokenizer.create(contents); tokenizer.next(); assertThat(tokenizer.getCurrentToken(), is(IdlToken.SPACE)); @@ -188,14 +182,14 @@ public void tokenizesMultipleSpacesAndTabsIntoSingleLexeme() { @Test public void throwsWhenAccessingErrorAndNoError() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("a").build(); + IdlTokenizer tokenizer = IdlTokenizer.create("a"); Assertions.assertThrows(ModelSyntaxException.class, tokenizer::getCurrentTokenError); } @Test public void storesErrorForInvalidSyntax() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("!").build(); + IdlTokenizer tokenizer = IdlTokenizer.create("!"); assertThat(tokenizer.next(), is(IdlToken.ERROR)); assertThat(tokenizer.getCurrentTokenError(), equalTo("Unexpected character: '!'")); @@ -203,14 +197,14 @@ public void storesErrorForInvalidSyntax() { @Test public void throwsWhenAccessingNumberAndNoNumber() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("a").build(); + IdlTokenizer tokenizer = IdlTokenizer.create("a"); Assertions.assertThrows(ModelSyntaxException.class, tokenizer::getCurrentTokenNumberValue); } @Test public void storesCurrentTokenNumber() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("10").build(); + IdlTokenizer tokenizer = IdlTokenizer.create("10"); assertThat(tokenizer.next(), is(IdlToken.NUMBER)); assertThat(tokenizer.getCurrentTokenNumberValue(), equalTo(10L)); @@ -218,14 +212,14 @@ public void storesCurrentTokenNumber() { @Test public void throwsWhenAccessingStringValue() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("10").build(); + IdlTokenizer tokenizer = IdlTokenizer.create("10"); Assertions.assertThrows(ModelSyntaxException.class, tokenizer::getCurrentTokenStringSlice); } @Test public void storesCurrentTokenString() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("\"hello\"").build(); + IdlTokenizer tokenizer = IdlTokenizer.create("\"hello\""); assertThat(tokenizer.next(), is(IdlToken.STRING)); assertThat(tokenizer.getCurrentTokenStringSlice().toString(), equalTo("hello")); @@ -233,233 +227,15 @@ public void storesCurrentTokenString() { @Test public void storesCurrentTokenStringForIdentifier() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("hello").build(); + IdlTokenizer tokenizer = IdlTokenizer.create("hello"); assertThat(tokenizer.next(), is(IdlToken.IDENTIFIER)); assertThat(tokenizer.getCurrentTokenStringSlice().toString(), equalTo("hello")); } - @Test - public void throwsWhenTooNested() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("").build(); - - for (int i = 0; i < 64; i++) { - tokenizer.increaseNestingLevel(); - } - - ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, tokenizer::increaseNestingLevel); - - assertThat(e.getMessage(), - startsWith("Syntax error at line 1, column 1: Parser exceeded maximum allowed depth of 64")); - } - - @Test - public void tokenizerInternsStrings() { - List tracked = new ArrayList<>(); - Function table = c -> { - tracked.add(c); - return c.toString(); - }; - - IdlTokenizer tokenizer = IdlTokenizer.builder() - .model("foo") - .stringTable(table) - .build(); - - tokenizer.internString("hi"); - - assertThat(tracked, contains("hi")); - } - - @Test - public void skipSpaces() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model(" hi").build(); - - tokenizer.skipSpaces(); - - assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); - assertThat(tokenizer.getCurrentTokenColumn(), is(5)); - } - - @Test - public void skipsExpectedSpaces() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model(" hi").build(); - - tokenizer.expectAndSkipSpaces(); - - assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); - assertThat(tokenizer.getCurrentTokenColumn(), is(5)); - } - - @Test - public void failsWhenExpectedSpacesNotThere() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("abc").build(); - - ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, tokenizer::expectAndSkipSpaces); - - assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected one or more spaces, but " - + "found IDENTIFIER('abc')")); - } - - @Test - public void skipWhitespace() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model(" \n\n hi").build(); - - tokenizer.skipWs(); - - assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); - assertThat(tokenizer.getCurrentTokenLine(), is(3)); - assertThat(tokenizer.getCurrentTokenColumn(), is(2)); - } - - @Test - public void expectAndSkipWhitespace() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model(" \n\n hi").build(); - - tokenizer.expectAndSkipWhitespace(); - - assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); - assertThat(tokenizer.getCurrentTokenLine(), is(3)); - assertThat(tokenizer.getCurrentTokenColumn(), is(2)); - } - - @Test - public void throwsWhenExpectedWhitespaceNotFound() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("hi").build(); - - ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, - tokenizer::expectAndSkipWhitespace); - - assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected one or more whitespace " - + "characters, but found IDENTIFIER('hi')")); - } - - @Test - public void skipDocsAndWhitespace() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model(" \n\n /// Docs\n/// Docs\n\n hi").build(); - - tokenizer.skipWsAndDocs(); - - assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); - assertThat(tokenizer.getCurrentTokenLine(), is(6)); - assertThat(tokenizer.getCurrentTokenColumn(), is(2)); - } - - @Test - public void expectsAndSkipsBr() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("\n Hi").build(); - - tokenizer.expectAndSkipBr(); - - assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); - assertThat(tokenizer.getCurrentTokenLine(), is(2)); - assertThat(tokenizer.getCurrentTokenColumn(), is(3)); - } - - @Test - public void throwsWhenBrNotFound() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); - - ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, - tokenizer::expectAndSkipBr); - - assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected a line break, but " - + "found IDENTIFIER('Hi')")); - } - - @Test - public void expectCurrentTokenLexeme() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); - - tokenizer.expectCurrentLexeme("Hi"); - } - - @Test - public void throwsWhenCurrentTokenLexemeUnexpected() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); - - ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, () -> { - tokenizer.expectCurrentLexeme("Bye"); - }); - - assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected `Bye`, but found `Hi`")); - assertThat(e.getSourceLocation().getLine(), is(1)); - assertThat(e.getSourceLocation().getColumn(), is(1)); - } - - @Test - public void throwsWhenCurrentTokenLexemeUnexpectedAndSameLength() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("A").build(); - - ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, () -> { - tokenizer.expectCurrentLexeme("B"); - }); - - assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected `B`, but found `A`")); - assertThat(e.getSourceLocation().getLine(), is(1)); - assertThat(e.getSourceLocation().getColumn(), is(1)); - } - - @Test - public void tokenDoesNotStartWith() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); - - assertThat(tokenizer.doesCurrentIdentifierStartWith('B'), is(false)); - } - - @Test - public void tokenDoesStartWith() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); - - assertThat(tokenizer.doesCurrentIdentifierStartWith('H'), is(true)); - } - - @Test - public void tokenDoesNotStartWithBecauseNotIdentifier() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("\"Hi\"").build(); - - assertThat(tokenizer.doesCurrentIdentifierStartWith('H'), is(false)); - } - - @Test - public void expectsSingleTokenType() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); - - tokenizer.expect(IdlToken.IDENTIFIER); - } - - @Test - public void failsForSingleExpectedToken() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); - - ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, - () -> tokenizer.expect(IdlToken.NUMBER)); - - assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected NUMBER but " - + "found IDENTIFIER('Hi')")); - } - - @Test - public void expectsMultipleTokenTypes() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); - - tokenizer.expect(IdlToken.STRING, IdlToken.IDENTIFIER); - } - - @Test - public void failsForMultipleExpectedTokens() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); - - ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, - () -> tokenizer.expect(IdlToken.NUMBER, IdlToken.LBRACE)); - - assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected one of NUMBER, LBRACE('{'); " - + "but found IDENTIFIER('Hi')")); - } - @Test public void failsWhenSingleForwardSlashFound() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("/").build(); + IdlTokenizer tokenizer = IdlTokenizer.create("/"); tokenizer.next(); @@ -469,33 +245,16 @@ public void failsWhenSingleForwardSlashFound() { @Test public void throwsWhenTraversingPastEof() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("").build(); + IdlTokenizer tokenizer = IdlTokenizer.create(""); assertThat(tokenizer.next(), is(IdlToken.EOF)); Assertions.assertThrows(NoSuchElementException.class, tokenizer::next); } - @Test - public void returnsCapturedDocsInRange() { - IdlTokenizer tokenizer = IdlTokenizer - .builder() - .model("/// Hi\n" - + "/// There\n" - + "/// 123\n" - + "/// 456\n") - .build(); - - tokenizer.skipWsAndDocs(); - String lines = tokenizer.removePendingDocCommentLines(); - - assertThat(lines, equalTo("Hi\nThere\n123\n456")); - assertThat(tokenizer.removePendingDocCommentLines(), nullValue()); - } - @Test public void tokenizesStringWithNewlines() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("\"hi\nthere\"").build(); + IdlTokenizer tokenizer = IdlTokenizer.create("\"hi\nthere\""); tokenizer.next(); @@ -512,7 +271,7 @@ public void tokenizesStringWithNewlines() { @Test public void tokenizesEmptyStrings() { - IdlTokenizer tokenizer = IdlTokenizer.builder().model("\"\"").build(); + IdlTokenizer tokenizer = IdlTokenizer.create("\"\""); tokenizer.next(); diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-quoted.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-quoted.smithy index ffbdc455fda..62948f1ad0f 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-quoted.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-quoted.smithy @@ -1,3 +1,3 @@ -// Syntax error at line 3, column 9: Expected one or more spaces, but found STRING('"foo"') | Model +// Syntax error at line 3, column 9: Expected SPACE(' ') but found STRING('"foo"') | Model $version: "2.0" metadata"foo"="bar" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-unquoted.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-unquoted.smithy index 039ecd8791a..3b556fbed09 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-unquoted.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-unquoted.smithy @@ -1,3 +1,3 @@ -// Syntax error at line 3, column 1: Expected `metadata`, but found `metadatafoo` | Model +// Syntax error at line 3, column 1: Expected a namespace definition but found IDENTIFIER('metadatafoo') | Model $version: "2.0" metadatafoo"="bar" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/missing-rest-of-with-word.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/missing-rest-of-with-word.smithy index 25fb65ece73..2e6ac9e4b8a 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/missing-rest-of-with-word.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/missing-rest-of-with-word.smithy @@ -1,4 +1,4 @@ -// Syntax error at line 5, column 15: Expected `with`, but found `wit` | Model +// Syntax error at line 5, column 15: Expected LBRACE('{') but found IDENTIFIER('wit') | Model $version: "2" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-name-but-eof.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-name-but-eof.smithy index 22cbce4d2b5..c2e7e70d10b 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-name-but-eof.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-name-but-eof.smithy @@ -1,4 +1,4 @@ -// Syntax error at line 4, column 7: Expected one or more spaces, but found NEWLINE('\n') | Model +// Syntax error at line 4, column 7: Expected SPACE(' ') but found NEWLINE('\n') | Model namespace com.foo string diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/invalid-newline-after-shape-type.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/invalid-newline-after-shape-type.smithy index 16282aa804e..4d1ed288f59 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/invalid-newline-after-shape-type.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/invalid-newline-after-shape-type.smithy @@ -1,4 +1,4 @@ -// Syntax error at line 5, column 10: Expected one or more spaces, but found NEWLINE('\n') | Model +// Syntax error at line 5, column 10: Expected SPACE(' ') but found NEWLINE('\n') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json index f426f83526d..ea06639a006 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json @@ -1,5 +1,8 @@ { "smithy": "2.0", + "metadata": { + "foo": ["bar"] + }, "shapes": { "smithy.example#NotDocumented": { "type": "string" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy index 4b002a90693..2ded3aa192e 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy @@ -1,4 +1,12 @@ +/// This comment is ignored. $version: "2.0" + +/// This comment is ignored. +metadata foo = [ + /// This comment is ignored. + "bar" +] + /// This comment is ignored. namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/expects-br.json b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/expects-br.json index 9bb21859bea..33d76d1a168 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/expects-br.json +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/expects-br.json @@ -1,15 +1,9 @@ { "smithy": "2.0", "shapes": { - "com.example#Bam": { - "type": "string" - }, "com.example#Bar": { "type": "string" }, - "com.example#Baz": { - "type": "string" - }, "com.example#Foo": { "type": "string" } diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/expects-br.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/expects-br.smithy index 8b908e99930..ff6eeb3dad0 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/expects-br.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/expects-br.smithy @@ -5,7 +5,3 @@ namespace com.example /// Ignored string Foo /// Ignored string Bar /// Ignored - -string Baz, /// Ignored - -string Bam,,,/// Ignored diff --git a/smithy-syntax/build.gradle b/smithy-syntax/build.gradle new file mode 100644 index 00000000000..68284cd339e --- /dev/null +++ b/smithy-syntax/build.gradle @@ -0,0 +1,50 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +description = "Provides a parse tree and formatter for Smithy models." + +ext { + displayName = "Smithy :: Syntax" + moduleName = "software.amazon.smithy.syntax" +} + +dependencies { + api project(":smithy-utils") + api project(":smithy-model") + implementation "com.opencastsoftware:prettier4j:0.1.1" + + // This is needed to export these as dependencies since we aren't shading them. + shadow project(":smithy-model") + shadow project(":smithy-utils") +} + +shadowJar { + // Replace the normal JAR with the shaded JAR. We don't want to publish a JAR that isn't shaded. + archiveClassifier = '' + + mergeServiceFiles() + + // Shade and relocate prettier4j. + relocate('com.opencastsoftware.prettier4j', 'software.amazon.smithy.syntax.shaded.prettier4j') + + // Despite the "shadow" configuration under dependencies, we unfortunately need to also list here that + // smithy-model and smithy-utils aren't shaded. These are normal dependencies that we want consumers to resolve. + dependencies { + exclude(project(':smithy-utils')) + exclude(project(':smithy-model')) + } +} + +tasks['jar'].finalizedBy(tasks['shadowJar']) diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/CapturedToken.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/CapturedToken.java new file mode 100644 index 00000000000..67be6b257ba --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/CapturedToken.java @@ -0,0 +1,313 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import java.math.BigDecimal; +import java.util.Objects; +import java.util.function.Function; +import software.amazon.smithy.model.FromSourceLocation; +import software.amazon.smithy.model.SourceLocation; +import software.amazon.smithy.model.loader.IdlToken; +import software.amazon.smithy.model.loader.IdlTokenizer; +import software.amazon.smithy.utils.SmithyBuilder; +import software.amazon.smithy.utils.ToSmithyBuilder; + +/** + * A persisted token captured from an {@link IdlTokenizer}. + * + *

For performance, {@code IdlTokenizer} does not create new tokens types for each encountered token. Instead, it + * updates the current state of the tokenizer and allows the caller to inspect the tokenizer for information about each + * token. Because smithy-syntax needs to create a token-tree rather than go directly to an AST, it requires arbitrary + * lookahead of tokens, requiring it to persist tokens in memory. + */ +public final class CapturedToken implements FromSourceLocation, ToSmithyBuilder { + + private final IdlToken token; + private final String filename; + private final int position; + private final int startLine; + private final int startColumn; + private final int endLine; + private final int endColumn; + private final CharSequence lexeme; + private final String stringContents; + private final String errorMessage; + private final Number numberValue; + + private CapturedToken( + IdlToken token, + String filename, + int position, + int startLine, + int startColumn, + int endLine, + int endColumn, + CharSequence lexeme, + String stringContents, + Number numberValue, + String errorMessage + ) { + this.token = Objects.requireNonNull(token, "Missing required token"); + this.lexeme = Objects.requireNonNull(lexeme, "Missing required lexeme"); + this.filename = filename == null ? SourceLocation.none().getFilename() : filename; + this.position = position; + this.startLine = startLine; + this.startColumn = startColumn; + this.endLine = endLine; + this.endColumn = endColumn; + + if (stringContents == null + && (token == IdlToken.IDENTIFIER || token == IdlToken.STRING || token == IdlToken.TEXT_BLOCK)) { + this.stringContents = lexeme.toString(); + } else { + this.stringContents = stringContents; + } + + if (errorMessage == null && token == IdlToken.ERROR) { + this.errorMessage = ""; + } else { + this.errorMessage = errorMessage; + } + + if (numberValue == null && token == IdlToken.NUMBER) { + this.numberValue = new BigDecimal(lexeme.toString()); + } else { + this.numberValue = numberValue; + } + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder implements SmithyBuilder { + private IdlToken token; + private String filename; + private int position; + private int startLine; + private int startColumn; + private int endLine; + private int endColumn; + private CharSequence lexeme; + private String stringContents; + private String errorMessage; + private Number numberValue; + + private Builder() {} + + @Override + public CapturedToken build() { + return new CapturedToken( + token, + filename, + position, + startLine, + startColumn, + endLine, + endColumn, + lexeme, + stringContents, + numberValue, + errorMessage + ); + } + + public Builder token(IdlToken token) { + this.token = token; + return this; + } + + public Builder filename(String filename) { + this.filename = filename; + return this; + } + + public Builder position(int position) { + this.position = position; + return this; + } + + public Builder startLine(int startLine) { + this.startLine = startLine; + return this; + } + + public Builder startColumn(int startColumn) { + this.startColumn = startColumn; + return this; + } + + public Builder endLine(int endLine) { + this.endLine = endLine; + return this; + } + + public Builder endColumn(int endColumn) { + this.endColumn = endColumn; + return this; + } + + public Builder lexeme(CharSequence lexeme) { + this.lexeme = lexeme; + return this; + } + + public Builder stringContents(String stringContents) { + this.stringContents = stringContents; + return this; + } + + public Builder errorMessage(String errorMessage) { + this.errorMessage = errorMessage; + return this; + } + + public Builder numberValue(Number numberValue) { + this.numberValue = numberValue; + return this; + } + } + + /** + * Persist the current token of an {@link IdlTokenizer}. + * + * @param tokenizer Tokenizer to capture. + * @return Returns the persisted token. + */ + public static CapturedToken from(IdlTokenizer tokenizer) { + return from(tokenizer, CharSequence::toString); + } + + /** + * Persist the current token of an {@link IdlTokenizer}. + * + * @param tokenizer Tokenizer to capture. + * @param stringTable String table that caches previously created strings. + * @return Returns the persisted token. + */ + public static CapturedToken from(IdlTokenizer tokenizer, Function stringTable) { + IdlToken tok = tokenizer.getCurrentToken(); + return builder() + .token(tok) + .filename(tokenizer.getSourceFilename()) + .position(tokenizer.getCurrentTokenStart()) + .startLine(tokenizer.getCurrentTokenLine()) + .startColumn(tokenizer.getCurrentTokenColumn()) + .endLine(tokenizer.getLine()) + .endColumn(tokenizer.getColumn()) + .lexeme(tokenizer.getCurrentTokenLexeme()) + .stringContents(tok == IdlToken.STRING || tok == IdlToken.TEXT_BLOCK || tok == IdlToken.IDENTIFIER + ? stringTable.apply(tokenizer.getCurrentTokenStringSlice()) + : null) + .numberValue(tok == IdlToken.NUMBER ? tokenizer.getCurrentTokenNumberValue() : null) + .errorMessage(tok == IdlToken.ERROR ? tokenizer.getCurrentTokenError() : null) + .build(); + } + + @Override + public Builder toBuilder() { + return builder() + .token(token) + .filename(filename) + .position(position) + .startLine(startLine) + .startColumn(startColumn) + .endLine(endLine) + .endColumn(endColumn) + .lexeme(lexeme) + .errorMessage(errorMessage) + .numberValue(numberValue) + .stringContents(stringContents); + } + + /** + * Get the token IDL token of the captured token. + * + * @return Returns the underlying token type. + */ + public IdlToken getIdlToken() { + return token; + } + + @Override + public SourceLocation getSourceLocation() { + return new SourceLocation(getFilename(), getStartLine(), getStartColumn()); + } + + public String getFilename() { + return filename; + } + + public int getPosition() { + return position; + } + + public int getStartLine() { + return startLine; + } + + public int getStartColumn() { + return startColumn; + } + + public int getEndLine() { + return endLine; + } + + public int getEndColumn() { + return endColumn; + } + + public int getSpan() { + return lexeme.length(); + } + + /** + * Get the raw lexeme of the current token. + * + * @return Returns the underlying lexeme of the token. + */ + public CharSequence getLexeme() { + return lexeme; + } + + /** + * Get the associated String contents of the token if it's a string, text block, or identifier. + * + * @return Returns the string contents of the lexeme, or null if not a string|text block|identifier. + */ + public String getStringContents() { + return stringContents; + } + + /** + * Gets the associated error message with the token if it's an error. + * + * @return Returns the error message or null if not an error. + */ + public String getErrorMessage() { + return errorMessage; + } + + /** + * Get the computed {@link Number} of the current token if it's a number. + * + * @return Returns the computed Number or null if not a number. + */ + public Number getNumberValue() { + return numberValue; + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/CapturingTokenizer.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/CapturingTokenizer.java new file mode 100644 index 00000000000..1a35c5f1697 --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/CapturingTokenizer.java @@ -0,0 +1,200 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Function; +import java.util.function.Predicate; +import software.amazon.smithy.model.loader.IdlToken; +import software.amazon.smithy.model.loader.IdlTokenizer; +import software.amazon.smithy.model.loader.ModelSyntaxException; +import software.amazon.smithy.model.loader.StringTable; + +/** + * Captures tokens to a stack of token trees. + */ +final class CapturingTokenizer implements IdlTokenizer { + + private final IdlTokenizer delegate; + private final TokenTree root = TokenTree.of(TreeType.IDL); + private final Deque trees = new ArrayDeque<>(); + private final Function stringTable = new StringTable(10); // 1024 entries + private final List tokens = new ArrayList<>(); + private int cursor = 0; + + CapturingTokenizer(IdlTokenizer delegate) { + this.delegate = delegate; + trees.add(root); + + while (delegate.hasNext()) { + delegate.next(); + tokens.add(CapturedToken.from(delegate, stringTable)); + } + } + + TokenTree getRoot() { + return root; + } + + @Override + public String getSourceFilename() { + return delegate.getSourceFilename(); + } + + @Override + public CharSequence getModel() { + return delegate.getModel(); + } + + private CapturedToken getToken() { + return tokens.get(cursor); + } + + @Override + public int getPosition() { + return getToken().getPosition() + getToken().getSpan(); + } + + @Override + public int getLine() { + return getToken().getEndLine(); + } + + @Override + public int getColumn() { + return getToken().getEndColumn(); + } + + @Override + public IdlToken getCurrentToken() { + return getToken().getIdlToken(); + } + + @Override + public int getCurrentTokenLine() { + return getToken().getStartLine(); + } + + @Override + public int getCurrentTokenColumn() { + return getToken().getStartColumn(); + } + + @Override + public int getCurrentTokenStart() { + return getToken().getPosition(); + } + + @Override + public int getCurrentTokenEnd() { + return getPosition(); + } + + @Override + public CharSequence getCurrentTokenStringSlice() { + return getToken().getStringContents(); + } + + @Override + public Number getCurrentTokenNumberValue() { + return getToken().getNumberValue(); + } + + @Override + public String getCurrentTokenError() { + return getToken().getErrorMessage(); + } + + @Override + public boolean hasNext() { + return getToken().getIdlToken() != IdlToken.EOF; + } + + @Override + public IdlToken next() { + if (getToken().getIdlToken() == IdlToken.EOF) { + throw new NoSuchElementException(); + } + trees.getFirst().appendChild(TokenTree.of(CapturedToken.from(this, this.stringTable))); + return tokens.get(++cursor).getIdlToken(); + } + + CapturedToken peekPastSpaces() { + return peekWhile(0, token -> token == IdlToken.SPACE); + } + + CapturedToken peekWhile(int offsetFromPosition, Predicate predicate) { + int position = cursor + offsetFromPosition; + // If the start position is out of bounds, return the EOF token. + if (position >= tokens.size()) { + return tokens.get(tokens.size() - 1); + } + CapturedToken token = tokens.get(position); + while (token.getIdlToken() != IdlToken.EOF && predicate.test(token.getIdlToken())) { + token = tokens.get(++position); + } + return token; + } + + String internString(CharSequence charSequence) { + return stringTable.apply(charSequence); + } + + TokenTree withState(TreeType state, Runnable parser) { + return withState(state, this::defaultErrorRecovery, parser); + } + + TokenTree withState(TreeType state, Runnable errorRecovery, Runnable parser) { + TokenTree tree = TokenTree.of(state); + trees.getFirst().appendChild(tree); + // Temporarily make this tree the current tree to capture tokens. + trees.addFirst(tree); + try { + parser.run(); + } catch (ModelSyntaxException e) { + TokenTree errorTree = TokenTree.fromError(e.getMessageWithoutLocation()); + tree.appendChild(errorTree); + if (getCurrentToken() != IdlToken.EOF) { + // Temporarily make the error tree the current tree to capture error recovery tokens. + trees.addFirst(errorTree); + next(); + errorRecovery.run(); + trees.removeFirst(); + } + } finally { + trees.removeFirst(); + } + return tree; + } + + // Performs basic error recovery by skipping tokens until a $, identifier, or @ is found at column 1. + private void defaultErrorRecovery() { + while (hasNext()) { + if (getCurrentTokenColumn() == 1) { + IdlToken token = getCurrentToken(); + if (token == IdlToken.DOLLAR || token == IdlToken.IDENTIFIER || token == IdlToken.AT + || token == IdlToken.RBRACE) { + return; + } + } + next(); + } + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/FixBadDocComments.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/FixBadDocComments.java new file mode 100644 index 00000000000..7a62655cf78 --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/FixBadDocComments.java @@ -0,0 +1,130 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import java.util.List; +import java.util.function.Function; +import software.amazon.smithy.model.loader.IdlToken; + +/** + * Rewrites invalid documentation comments that don't come directly before a shape or member to be a normal comment. + * + *

TODO: This does not remove every possible invalid documentation comment (e.g., doc comments after traits). + */ +final class FixBadDocComments implements Function { + @Override + public TokenTree apply(TokenTree tree) { + TreeCursor root = tree.zipper(); + + // These sections can never have doc comments. + updateDirectChildren(root.getFirstChild(TreeType.WS)); + updateNestedChildren(root.getFirstChild(TreeType.CONTROL_SECTION)); + updateNestedChildren(root.getFirstChild(TreeType.METADATA_SECTION)); + + // These sections should always be present in every model. + TreeCursor shapeSection = root.getFirstChild(TreeType.SHAPE_SECTION); + if (shapeSection == null) { + return tree; + } + + TreeCursor useSection = shapeSection.getFirstChild(TreeType.USE_SECTION); + if (useSection == null) { + return tree; + } + + // Remove doc comments from NAMESPACE_STATEMENT if there are use statements (that is, no way a doc comment + // is applied to the next shape statement). + if (!useSection.getTree().isEmpty()) { + updateNestedChildren(shapeSection.getFirstChild(TreeType.NAMESPACE_STATEMENT)); + + // Remove doc comments from all but the last use statement. + List useStatements = useSection.getChildrenByType(TreeType.USE_STATEMENT); + for (int i = 0; i < useStatements.size() - 1; i++) { + updateNestedChildren(useStatements.get(i)); + } + } + + // Doc comments are never allowed in NODE_VALUE values. + for (TreeCursor cursor : shapeSection.findChildrenByType(TreeType.NODE_VALUE)) { + updateNestedChildren(cursor); + } + + // Doc comments are never allowed in TRAIT values. + for (TreeCursor cursor : shapeSection.findChildrenByType(TreeType.TRAIT)) { + updateNestedChildren(cursor); + } + + // Fix doc comments that come before apply statements. + TreeCursor shapeStatements = shapeSection.getFirstChild(TreeType.SHAPE_STATEMENTS); + if (shapeStatements != null) { + // Find BRs in shape statements and look at the next sibling. + for (TreeCursor br : shapeStatements.getChildrenByType(TreeType.BR)) { + TreeCursor nextSibling = br.getNextSibling(); + if (nextSibling == null || nextSibling.getFirstChild(TreeType.APPLY_STATEMENT) != null) { + updateNestedChildren(br); + } + } + + // Remove trailing doc comments in member bodies. + for (TreeCursor members : shapeStatements.findChildrenByType(TreeType.SHAPE_MEMBERS)) { + TreeCursor ws = members.getLastChild(TreeType.WS); + updateDirectChildren(ws); + } + } + + return tree; + } + + private void updateDirectChildren(TreeCursor container) { + if (container != null) { + updateChildren(container.getChildrenByType(TreeType.COMMENT)); + } + } + + private void updateNestedChildren(TreeCursor container) { + if (container != null) { + updateChildren(container.findChildrenByType(TreeType.COMMENT)); + } + } + + private void updateChildren(List children) { + for (TreeCursor comment : children) { + if (isDocComment(comment)) { + updateComment(comment); + } + } + } + + private boolean isDocComment(TreeCursor cursor) { + return cursor.getTree() + .tokens() + .findFirst() + .filter(token -> token.getIdlToken() == IdlToken.DOC_COMMENT) + .isPresent(); + } + + private void updateComment(TreeCursor cursor) { + cursor.getTree().tokens().findFirst().ifPresent(token -> { + CapturedToken updatedToken = token.toBuilder() + // Trim the first "/" from the lexeme. Note that this does make the spans inaccurate. + .lexeme(token.getLexeme().subSequence(1, token.getLexeme().length())) + .build(); + TokenTree updatedTree = TokenTree.of(TreeType.COMMENT); + updatedTree.appendChild(TokenTree.of(updatedToken)); + cursor.getParent().getTree().replaceChild(cursor.getTree(), updatedTree); + }); + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/Formatter.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/Formatter.java new file mode 100644 index 00000000000..caeef6d5b91 --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/Formatter.java @@ -0,0 +1,682 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import com.opencastsoftware.prettier4j.Doc; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Stream; +import software.amazon.smithy.model.loader.ModelSyntaxException; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.utils.StringUtils; + +/** + * Formats valid Smithy IDL models. + * + *

This formatter will by default sort use statements, remove unused use statements, and fix documentation + * comments that should be normal comments. + */ +public final class Formatter { + + private Formatter() {} + + /** + * Formats the given token tree, wrapping lines at 120 characters. + * + * @param root Root {@link TreeType#IDL} tree node to format. + * @return Returns the formatted model as a string. + * @throws ModelSyntaxException if the model contains errors. + */ + public static String format(TokenTree root) { + return format(root, 120); + } + + /** + * Formats the given token tree. + * + * @param root Root {@link TreeType#IDL} tree node to format. + * @param maxWidth Maximum line width. + * @return Returns the formatted model as a string. + * @throws ModelSyntaxException if the model contains errors. + */ + public static String format(TokenTree root, int maxWidth) { + List errors = root.zipper().findChildrenByType(TreeType.ERROR); + + if (!errors.isEmpty()) { + throw new ModelSyntaxException("Cannot format invalid models: " + errors.get(0).getTree().getError(), + errors.get(0)); + } + + root = new SortUseStatements().apply(root); + root = new FixBadDocComments().apply(root); + root = new RemoveUnusedUseStatements().apply(root); + + // Strip trailing spaces from each line. + String result = new TreeVisitor(maxWidth).visit(root.zipper()).render(maxWidth).trim(); + StringBuilder builder = new StringBuilder(); + for (String line : result.split(System.lineSeparator())) { + builder.append(StringUtils.stripEnd(line, " \t")).append(System.lineSeparator()); + } + return builder.toString(); + } + + private static final class TreeVisitor { + + private static final Doc LINE_OR_COMMA = Doc.lineOr(Doc.text(", ")); + private static final Doc SPACE = Doc.text(" "); + private static final Doc LINE_OR_SPACE = Doc.lineOrSpace(); + + // width is needed since intermediate renders are used to detect when newlines are used in a statement. + private final int width; + + // Used to handle extracting comments out of whitespace of prior statements. + private Doc pendingComments = Doc.empty(); + + private TreeVisitor(int width) { + this.width = width; + } + + private Doc visit(TreeCursor cursor) { + if (cursor == null) { + return Doc.empty(); + } + + TokenTree tree = cursor.getTree(); + + switch (tree.getType()) { + case IDL: { + return visit(cursor.getFirstChild(TreeType.WS)) + .append(visit(cursor.getFirstChild(TreeType.CONTROL_SECTION))) + .append(visit(cursor.getFirstChild(TreeType.METADATA_SECTION))) + .append(visit(cursor.getFirstChild(TreeType.SHAPE_SECTION))) + .append(flushBrBuffer()); + } + + case CONTROL_SECTION: { + return section(cursor, TreeType.CONTROL_STATEMENT); + } + + case METADATA_SECTION: { + return section(cursor, TreeType.METADATA_STATEMENT); + } + + case SHAPE_SECTION: { + return Doc.intersperse(Doc.line(), cursor.children().map(this::visit)); + } + + case SHAPE_STATEMENTS: { + Doc result = Doc.empty(); + Iterator childIterator = cursor.getChildren().iterator(); + int i = 0; + while (childIterator.hasNext()) { + if (i++ > 0) { + result = result.append(Doc.line()); + } + result = result.append(visit(childIterator.next())) // SHAPE + .append(visit(childIterator.next())) // BR + .append(Doc.line()); + } + return result; + } + + case CONTROL_STATEMENT: { + return flushBrBuffer() + .append(Doc.text("$")) + .append(visit(cursor.getFirstChild(TreeType.NODE_OBJECT_KEY))) + .append(Doc.text(": ")) + .append(visit(cursor.getFirstChild(TreeType.NODE_VALUE))) + .append(visit(cursor.getFirstChild(TreeType.BR))); + } + + case METADATA_STATEMENT: { + return flushBrBuffer() + .append(Doc.text("metadata ")) + .append(visit(cursor.getFirstChild(TreeType.NODE_OBJECT_KEY))) + .append(Doc.text(" = ")) + .append(visit(cursor.getFirstChild(TreeType.NODE_VALUE))) + .append(visit(cursor.getFirstChild(TreeType.BR))); + } + + case NAMESPACE_STATEMENT: { + return Doc.line() + .append(flushBrBuffer()) + .append(Doc.text("namespace ")) + .append(visit(cursor.getFirstChild(TreeType.NAMESPACE))) + .append(visit(cursor.getFirstChild(TreeType.BR))); + } + + case USE_SECTION: { + return section(cursor, TreeType.USE_STATEMENT); + } + + case USE_STATEMENT: { + return flushBrBuffer() + .append(Doc.text("use ")) + .append(visit(cursor.getFirstChild(TreeType.ABSOLUTE_ROOT_SHAPE_ID))) + .append(visit(cursor.getFirstChild(TreeType.BR))); + } + + case SHAPE_OR_APPLY_STATEMENT: + case SHAPE: + case OPERATION_PROPERTY: + case APPLY_STATEMENT: + case NODE_VALUE: + case NODE_KEYWORD: + case SIMPLE_TYPE_NAME: + case ENUM_TYPE_NAME: + case AGGREGATE_TYPE_NAME: + case ENTITY_TYPE_NAME: { + return visit(cursor.getFirstChild()); + } + + case SHAPE_STATEMENT: { + return flushBrBuffer() + .append(visit(cursor.getFirstChild(TreeType.WS))) + .append(visit(cursor.getFirstChild(TreeType.TRAIT_STATEMENTS))) + .append(visit(cursor.getFirstChild(TreeType.SHAPE))); + } + + case SIMPLE_SHAPE: { + return formatShape(cursor, visit(cursor.getFirstChild(TreeType.SIMPLE_TYPE_NAME)), null); + } + + case ENUM_SHAPE: { + return skippedComments(cursor, false) + .append(formatShape( + cursor, + visit(cursor.getFirstChild(TreeType.ENUM_TYPE_NAME)), + visit(cursor.getFirstChild(TreeType.ENUM_SHAPE_MEMBERS)))); + } + + case ENUM_SHAPE_MEMBERS: { + return renderMembers(cursor, TreeType.ENUM_SHAPE_MEMBER); + } + + case ENUM_SHAPE_MEMBER: { + return visit(cursor.getFirstChild(TreeType.TRAIT_STATEMENTS)) + .append(visit(cursor.getFirstChild(TreeType.IDENTIFIER))) + .append(visit(cursor.getFirstChild(TreeType.VALUE_ASSIGNMENT))); + } + + case AGGREGATE_SHAPE: { + return skippedComments(cursor, false) + .append(formatShape( + cursor, + visit(cursor.getFirstChild(TreeType.AGGREGATE_TYPE_NAME)), + visit(cursor.getFirstChild(TreeType.SHAPE_MEMBERS)))); + } + + case FOR_RESOURCE: { + return Doc.text("for ").append(visit(cursor.getFirstChild(TreeType.SHAPE_ID))); + } + + case SHAPE_MEMBERS: { + return renderMembers(cursor, TreeType.SHAPE_MEMBER); + } + + case SHAPE_MEMBER: { + return visit(cursor.getFirstChild(TreeType.TRAIT_STATEMENTS)) + .append(visit(cursor.getFirstChild(TreeType.ELIDED_SHAPE_MEMBER))) + .append(visit(cursor.getFirstChild(TreeType.EXPLICIT_SHAPE_MEMBER))) + .append(visit(cursor.getFirstChild(TreeType.VALUE_ASSIGNMENT))); + } + + case EXPLICIT_SHAPE_MEMBER: { + return visit(cursor.getFirstChild(TreeType.IDENTIFIER)) + .append(Doc.text(": ")) + .append(visit(cursor.getFirstChild(TreeType.SHAPE_ID))); + } + + case ELIDED_SHAPE_MEMBER: { + return Doc.text("$").append(visit(cursor.getFirstChild(TreeType.IDENTIFIER))); + } + + case ENTITY_SHAPE: { + return skippedComments(cursor, false) + .append(formatShape( + cursor, + visit(cursor.getFirstChild(TreeType.ENTITY_TYPE_NAME)), + Doc.lineOrSpace().append(visit(cursor.getFirstChild(TreeType.NODE_OBJECT))))); + } + + case OPERATION_SHAPE: { + return skippedComments(cursor, false) + .append(formatShape(cursor, Doc.text("operation"), + visit(cursor.getFirstChild(TreeType.OPERATION_BODY)))); + } + + case OPERATION_BODY: { + return renderMembers(cursor, TreeType.OPERATION_PROPERTY); + } + + case OPERATION_INPUT: { + TreeCursor simpleTarget = cursor.getFirstChild(TreeType.SHAPE_ID); + return skippedComments(cursor, false) + .append(Doc.text("input")) + .append(simpleTarget == null + ? visit(cursor.getFirstChild(TreeType.INLINE_AGGREGATE_SHAPE)) + : Doc.text(": ")).append(visit(simpleTarget)); + } + + case OPERATION_OUTPUT: { + TreeCursor simpleTarget = cursor.getFirstChild(TreeType.SHAPE_ID); + return skippedComments(cursor, false) + .append(Doc.text("output")) + .append(simpleTarget == null + ? visit(cursor.getFirstChild(TreeType.INLINE_AGGREGATE_SHAPE)) + : Doc.text(": ")).append(visit(simpleTarget)); + } + + case INLINE_AGGREGATE_SHAPE: { + boolean hasComment = hasComment(cursor); + boolean hasTraits = Optional.ofNullable(cursor.getFirstChild(TreeType.TRAIT_STATEMENTS)) + .filter(c -> !c.getChildrenByType(TreeType.TRAIT).isEmpty()) + .isPresent(); + Doc memberDoc = visit(cursor.getFirstChild(TreeType.SHAPE_MEMBERS)); + if (hasComment || hasTraits) { + return Doc.text(" :=") + .append(Doc.line()) + .append(skippedComments(cursor, false)) + .append(visit(cursor.getFirstChild(TreeType.TRAIT_STATEMENTS))) + .append(formatShape(cursor, Doc.empty(), memberDoc)) + .indent(4); + } + + return formatShape(cursor, Doc.text(" :="), memberDoc); + } + + case OPERATION_ERRORS: { + return skippedComments(cursor, false) + .append(Doc.text("errors: ")) + .append(bracketed("[", "]", cursor, TreeType.SHAPE_ID)); + } + + case MIXINS: { + return Doc.text("with ") + .append(bracketed("[", "]", cursor, cursor, child -> { + return child.getTree().getType() == TreeType.SHAPE_ID + ? Stream.of(child) + : Stream.empty(); + })); + } + + case VALUE_ASSIGNMENT: { + return Doc.text(" = ") + .append(visit(cursor.getFirstChild(TreeType.NODE_VALUE))) + .append(visit(cursor.getFirstChild(TreeType.BR))); + } + + case TRAIT_STATEMENTS: { + return Doc.intersperse( + Doc.line(), + cursor.children() + // Skip WS nodes that have no comments. + .filter(c -> c.getTree().getType() == TreeType.TRAIT || hasComment(c)) + .map(this::visit)) + .append(tree.isEmpty() ? Doc.empty() : Doc.line()); + } + + case TRAIT: { + return Doc.text("@") + .append(visit(cursor.getFirstChild(TreeType.SHAPE_ID))) + .append(visit(cursor.getFirstChild(TreeType.TRAIT_BODY))); + } + + case TRAIT_BODY: { + TreeCursor structuredBody = cursor.getFirstChild(TreeType.TRAIT_STRUCTURE); + if (structuredBody != null) { + return bracketed("(", ")", cursor, cursor, child -> { + if (child.getTree().getType() == TreeType.TRAIT_STRUCTURE) { + // Split WS and NODE_OBJECT_KVP so that they appear on different lines. + return child.getChildrenByType(TreeType.NODE_OBJECT_KVP, TreeType.WS).stream(); + } + return Stream.empty(); + }); + } else { + // Check the inner trait node for hard line breaks rather than the wrapper. + TreeCursor traitNode = cursor + .getFirstChild(TreeType.TRAIT_NODE) + .getFirstChild(TreeType.NODE_VALUE) + .getFirstChild(); // The actual node value. + return bracketed("(", ")", cursor, traitNode, child -> { + if (child.getTree().getType() == TreeType.TRAIT_NODE) { + // Split WS and NODE_VALUE so that they appear on different lines. + return child.getChildrenByType(TreeType.NODE_VALUE, TreeType.WS).stream(); + } else { + return Stream.empty(); + } + }); + } + } + + case TRAIT_NODE: { + return visit(cursor.getFirstChild()).append(visit(cursor.getFirstChild(TreeType.WS))); + } + + case TRAIT_STRUCTURE: { + throw new UnsupportedOperationException("Use TRAIT_BODY"); + } + + case APPLY_STATEMENT_SINGULAR: { + // If there is an awkward comment before the TRAIT value, hoist it above the statement. + return flushBrBuffer() + .append(skippedComments(cursor, false)) + .append(Doc.text("apply ")) + .append(visit(cursor.getFirstChild(TreeType.SHAPE_ID))) + .append(SPACE) + .append(visit(cursor.getFirstChild(TreeType.TRAIT))); + } + + case APPLY_STATEMENT_BLOCK: { + // TODO: This renders the "apply" block as a string so that we can trim the contents before adding + // the trailing newline + closing bracket. Otherwise, we'll get a blank, indented line, before + // the closing brace. + return flushBrBuffer() + .append(Doc.text(skippedComments(cursor, false) + .append(Doc.text("apply ")) + .append(visit(cursor.getFirstChild(TreeType.SHAPE_ID))) + .append(Doc.text(" {")) + .append(Doc.line().append(visit(cursor.getFirstChild(TreeType.TRAIT_STATEMENTS))).indent(4)) + .render(width) + .trim()) + .append(Doc.line()) + .append(Doc.text("}"))); + } + + case NODE_ARRAY: { + return bracketed("[", "]", cursor, TreeType.NODE_VALUE); + } + + case NODE_OBJECT: { + return bracketed("{", "}", cursor, TreeType.NODE_OBJECT_KVP); + } + + case NODE_OBJECT_KVP: { + // Hoist awkward comments in the KVP *before* the KVP rather than between the values and colon. + // If there is an awkward comment before the TRAIT value, hoist it above the statement. + return skippedComments(cursor, false) + .append(visit(cursor.getFirstChild(TreeType.NODE_OBJECT_KEY))) + .append(Doc.text(": ")) + .append(visit(cursor.getFirstChild(TreeType.NODE_VALUE))); + } + + case NODE_OBJECT_KEY: { + // Unquote object keys that can be unquoted. + CharSequence unquoted = Optional.ofNullable(cursor.getFirstChild(TreeType.QUOTED_TEXT)) + .flatMap(quoted -> quoted.getTree().tokens().findFirst()) + .map(token -> token.getLexeme().subSequence(1, token.getSpan() - 1)) + .orElse(""); + return ShapeId.isValidIdentifier(unquoted) + ? Doc.text(unquoted.toString()) + : Doc.text(tree.concatTokens()); + } + + case TOKEN: + case TEXT_BLOCK: + case NODE_STRING_VALUE: + case QUOTED_TEXT: + case NUMBER: + case SHAPE_ID: + case ROOT_SHAPE_ID: + case ABSOLUTE_ROOT_SHAPE_ID: + case SHAPE_ID_MEMBER: + case NAMESPACE: + case IDENTIFIER: { + return Doc.text(tree.concatTokens()); + } + + case COMMENT: { + // Ensure comments have a single space before their content. + String contents = tree.concatTokens().trim(); + if (contents.startsWith("/// ") || contents.startsWith("// ")) { + return Doc.text(contents); + } else if (contents.startsWith("///")) { + return Doc.text("/// " + contents.substring(3)); + } else { + return Doc.text("// " + contents.substring(2)); + } + } + + case WS: { + // Ignore all whitespace except for comments and doc comments. + return Doc.intersperse( + Doc.line(), + cursor.getChildrenByType(TreeType.COMMENT).stream().map(this::visit) + ); + } + + case BR: { + pendingComments = Doc.empty(); + Doc result = Doc.empty(); + List comments = getComments(cursor); + for (TreeCursor comment : comments) { + if (comment.getTree().getStartLine() == tree.getStartLine()) { + result = result.append(SPACE.append(visit(comment))); + } else { + pendingComments = pendingComments.append(visit(comment)).append(Doc.line()); + } + } + return result; + } + + default: { + return Doc.empty(); + } + } + } + + private Doc formatShape(TreeCursor cursor, Doc type, Doc members) { + List docs = new EmptyIgnoringList(); + docs.add(type); + docs.add(visit(cursor.getFirstChild(TreeType.IDENTIFIER))); + docs.add(visit(cursor.getFirstChild(TreeType.FOR_RESOURCE))); + docs.add(visit(cursor.getFirstChild(TreeType.MIXINS))); + Doc result = Doc.intersperse(SPACE, docs); + return members != null ? result.append(Doc.group(members)) : result; + } + + private static final class EmptyIgnoringList extends ArrayList { + @Override + public boolean add(Doc doc) { + return doc != Doc.empty() && super.add(doc); + } + } + + private Doc flushBrBuffer() { + Doc result = pendingComments; + pendingComments = Doc.empty(); + return result; + } + + // Check if a cursor contains direct child comments or a direct child WS that contains comments. + private boolean hasComment(TreeCursor cursor) { + return !getComments(cursor).isEmpty(); + } + + // Get direct child comments from a cursor, or from direct WS children that have comments. + private List getComments(TreeCursor cursor) { + List result = new ArrayList<>(); + for (TreeCursor wsOrComment : cursor.getChildrenByType(TreeType.COMMENT, TreeType.WS)) { + if (wsOrComment.getTree().getType() == TreeType.WS) { + result.addAll(wsOrComment.getChildrenByType(TreeType.COMMENT)); + } else { + result.add(wsOrComment); + } + } + return result; + } + + // Concatenate all comments in a tree into a single line delimited Doc. + private Doc skippedComments(TreeCursor cursor, boolean leadingLine) { + List comments = getComments(cursor); + if (comments.isEmpty()) { + return Doc.empty(); + } + List docs = new ArrayList<>(comments.size()); + comments.forEach(c -> docs.add(visit(c).append(Doc.line()))); + return (leadingLine ? Doc.line() : Doc.empty()).append(Doc.fold(docs, Doc::append)); + } + + // Brackets children of childType between open and closed brackets. If the children can fit together + // on a single line, they are comma separated. If not, they are split onto multiple lines with no commas. + private Doc bracketed(String open, String close, TreeCursor cursor, TreeType childType) { + return bracketed(open, close, cursor, cursor, child -> child.getTree().getType() == childType + ? Stream.of(child) : Stream.empty()); + } + + private Doc bracketed(String open, String close, TreeCursor cursor, TreeCursor hardLineSubject, + Function> childExtractor) { + Stream children = cursor.children() + .flatMap(c -> { + TreeType type = c.getTree().getType(); + return type == TreeType.WS || type == TreeType.COMMENT + ? Stream.of(c) + : childExtractor.apply(c); + }) + .flatMap(c -> { + // If the child extracts WS, then filter it down to just comments. + return c.getTree().getType() == TreeType.WS + ? c.getChildrenByType(TreeType.COMMENT).stream() + : Stream.of(c); + }) + .map(this::visit) + .filter(doc -> doc != Doc.empty()); // empty lines add extra lines we don't need. + + if (!hasHardLine(hardLineSubject)) { + return Doc.intersperse(LINE_OR_COMMA, children).bracket(4, Doc.lineOrEmpty(), open, close); + } else { + return renderBlock(Doc.text(open), close, Doc.intersperse(Doc.line(), children)); + } + } + + // Check if the given tree has any hard lines. Nested arrays and objects are always considered hard lines. + private static boolean hasHardLine(TreeCursor cursor) { + List children = cursor.findChildrenByType( + TreeType.COMMENT, TreeType.TEXT_BLOCK, TreeType.NODE_ARRAY, TreeType.NODE_OBJECT, + TreeType.QUOTED_TEXT); + for (TreeCursor child : children) { + if (child.getTree().getType() != TreeType.QUOTED_TEXT) { + return true; + } else if (child.getTree().getStartLine() != child.getTree().getEndLine()) { + // Detect strings with line breaks. + return true; + } + } + return false; + } + + // Renders "members" in braces, grouping related comments and members together. + private Doc renderMembers(TreeCursor container, TreeType memberType) { + boolean noComments = container.findChildrenByType(TreeType.COMMENT, TreeType.TRAIT).isEmpty(); + // Separate members by a single line if none have traits or docs, and two lines if any do. + Doc separator = noComments ? Doc.line() : Doc.line().append(Doc.line()); + List members = container.getChildrenByType(memberType, TreeType.WS); + // Remove WS we don't care about. + members.removeIf(c -> c.getTree().getType() == TreeType.WS && !hasComment(c)); + // Empty structures render as "{}". + if (noComments && members.isEmpty()) { + return Doc.group(LINE_OR_SPACE.append(Doc.text("{}"))); + } + + // Group consecutive comments and members together, and add a new line after each member. + List memberDocs = new ArrayList<>(); + // Start the current result with a buffered comment, if any, or an empty Doc. + Doc current = flushBrBuffer(); + boolean newLineNeededAfterComment = false; + + for (TreeCursor member : members) { + if (member.getTree().getType() == TreeType.WS) { + newLineNeededAfterComment = true; + current = current.append(visit(member)); + } else { + if (newLineNeededAfterComment) { + current = current.append(Doc.line()); + newLineNeededAfterComment = false; + } + current = current.append(visit(member)); + memberDocs.add(current); + current = flushBrBuffer(); + } + } + + if (current != Doc.empty()) { + memberDocs.add(current); + } + + Doc open = LINE_OR_SPACE.append(Doc.text("{")); + return renderBlock(open, "}", Doc.intersperse(separator, memberDocs)); + } + + // Renders members and anything bracketed that are known to need expansion on multiple lines. + private Doc renderBlock(Doc open, String close, Doc contents) { + return open + .append(Doc.line().append(contents).indent(4)) + .append(Doc.line()) + .append(Doc.text(close)); + } + + // Renders control, metadata, and use sections so that each statement has a leading and trailing newline + // IFF the statement spans multiple lines (i.e., long value that wraps, comments, etc). + private Doc section(TreeCursor cursor, TreeType childType) { + List children = cursor.getChildrenByType(childType); + + // Empty sections emit no code. + if (children.isEmpty()) { + return Doc.empty(); + } + + // Tracks when a line was just written. + // Initialized to false since there's no need to ever add a leading line in a section of statements. + boolean justWroteTrailingLine = true; + + // Sections need a new line to separate them from the previous content. + // Note: even though this emits a leading newline in every generated model, a top-level String#trim() is + // used to clean this up. + Doc result = Doc.line(); + + for (int i = 0; i < children.size(); i++) { + boolean isLast = i == children.size() - 1; + TreeCursor child = children.get(i); + + // Render the child to a String to detect if a newline was rendered. This is fine to do here since all + // statements that use this method are rooted at column 0 with no indentation. This rendered text is + // also used as part of the generated Doc since there's no need to re-analyze each statement. + String rendered = visit(child).render(width); + + if (rendered.contains(System.lineSeparator())) { + if (!justWroteTrailingLine) { + result = result.append(Doc.line()); + } + result = result.append(Doc.text(rendered)); + if (!isLast) { + result = result.append(Doc.line()); + justWroteTrailingLine = true; + } + } else { + result = result.append(Doc.text(rendered)); + justWroteTrailingLine = false; + } + + result = result.append(Doc.line()); + } + + return result; + } + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/RemoveUnusedUseStatements.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/RemoveUnusedUseStatements.java new file mode 100644 index 00000000000..49c0c4045c3 --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/RemoveUnusedUseStatements.java @@ -0,0 +1,93 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import java.util.logging.Logger; +import software.amazon.smithy.model.shapes.ShapeId; + +final class RemoveUnusedUseStatements implements Function { + + private static final Logger LOGGER = Logger.getLogger(RemoveUnusedUseStatements.class.getName()); + + @Override + public TokenTree apply(TokenTree tree) { + TreeCursor root = tree.zipper(); + Map useShapeNames = parseShapeIds(root); + + if (useShapeNames.isEmpty()) { + return tree; + } + + // SHAPE_SECTION is always present at this point if there are detected use statements. + TreeCursor shapeStatements = Objects.requireNonNull(root.getFirstChild(TreeType.SHAPE_SECTION)) + .getFirstChild(TreeType.SHAPE_STATEMENTS); + + if (shapeStatements == null) { + return tree; + } + + for (TreeCursor identifier : shapeStatements.findChildrenByType(TreeType.SHAPE_ID)) { + String name = identifier.getTree().concatTokens(); + + // Absolute shape IDs don't interact with use statements. + if (name.contains("#")) { + continue; + } + + // Remove the member if found. + if (name.contains("$")) { + name = name.substring(0, name.indexOf("$")); + } + + useShapeNames.remove(name); + } + + // Anything left in the map needs to be removed from the tree. + for (TreeCursor unused : useShapeNames.values()) { + LOGGER.fine(() -> "Removing unused use statement: " + + unused.getFirstChild(TreeType.ABSOLUTE_ROOT_SHAPE_ID) + .getTree().concatTokens()); + unused.getParent().getTree().removeChild(unused.getTree()); + } + + return tree; + } + + // Create a map of shape name to the TreeCursor of the use statement. + private Map parseShapeIds(TreeCursor root) { + List useStatements = Optional.ofNullable(root.getFirstChild(TreeType.SHAPE_SECTION)) + .flatMap(shapeSection -> Optional.ofNullable(shapeSection.getFirstChild(TreeType.USE_SECTION))) + .map(useSection -> useSection.getChildrenByType(TreeType.USE_STATEMENT)) + .orElse(Collections.emptyList()); + + Map result = new HashMap<>(useStatements.size()); + for (TreeCursor useStatement : useStatements) { + TreeCursor idCursor = useStatement.getFirstChild(TreeType.ABSOLUTE_ROOT_SHAPE_ID); + if (idCursor != null) { + result.put(ShapeId.from(idCursor.getTree().concatTokens()).getName(), useStatement); + } + } + + return result; + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/SortUseStatements.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/SortUseStatements.java new file mode 100644 index 00000000000..b9e29bc06a8 --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/SortUseStatements.java @@ -0,0 +1,200 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import software.amazon.smithy.model.loader.IdlToken; + +/** + * Sorts use statements in models alphabetically, case-sensitively. + */ +final class SortUseStatements implements Function { + // One of the complications of this transformation is that comments that might document a use statement are + // captured in the BR production that trails a use statement. So moving a use statement needs to move any trailing + // comments from its previous sibling to the next previous sibling it's moved after. To make this even more + // complicated, comments on the same line as a statement should stay with the statement rather than moved. + @Override + public TokenTree apply(TokenTree tree) { + TreeCursor useSection = Objects.requireNonNull(tree.zipper().getFirstChild(TreeType.SHAPE_SECTION)) + .getFirstChild(TreeType.USE_SECTION); + + // All trees should have a USE_SECTION. If the section is empty, then no use statements need to be sorted. + // Note that if it isn't empty, then there is a guaranteed NAMESPACE_STATEMENT too. + if (useSection == null || useSection.getTree().isEmpty()) { + return tree; + } + + UseDataModel dataModel = new UseDataModel(tree.zipper(), useSection); + + // Remove the original statements since they get sorted and re-added. + useSection.getChildrenByType(TreeType.USE_STATEMENT) + .forEach(s -> useSection.getTree().removeChild(s.getTree())); + + dataModel.addFirstUse(useSection); + dataModel.addSubsequentUse(useSection); + + return tree; + } + + private static final class UseDataModel { + List ids = new ArrayList<>(); + Map lineComments = new HashMap<>(); + Map> leadingComments = new HashMap<>(); + List endComments = new ArrayList<>(); + TreeCursor namespaceBrWs; + + UseDataModel(TreeCursor root, TreeCursor useSection) { + List shapeIds = useSection.findChildrenByType(TreeType.ABSOLUTE_ROOT_SHAPE_ID); + List comments = useSection.findChildrenByType(TreeType.COMMENT); + + int commentIndex = 0; + for (TreeCursor id : shapeIds) { + ids.add(id.getTree()); + while (commentIndex < comments.size()) { + TreeCursor comment = comments.get(commentIndex); + if (comment.getTree().getStartLine() == id.getTree().getStartLine()) { + lineComments.put(id.getTree(), comment); + } else if (comment.getTree().getStartLine() < id.getTree().getStartLine()) { + putLeadingComment(id.getTree(), comment); + } else { + break; + } + commentIndex++; + } + } + + // Any remaining comments are "sticky" since they are on the next section, not the use statement. + while (commentIndex < comments.size()) { + TreeCursor comment = comments.get(commentIndex); + endComments.add(comment); + commentIndex++; + } + + findOrCreateNamespaceBrWs(root); + + ids.sort(Comparator.comparing(TokenTree::concatTokens)); + } + + private void findOrCreateNamespaceBrWs(TreeCursor root) { + // The first use statement could have leading comments that are part of the previous section. If this + // statement is moved, then the comments from the previous section need to be removed and moved into this + // section, and potentially replaced with comments from USE_SECTION. + TreeCursor br = root.findChildrenByType(TreeType.NAMESPACE_STATEMENT) + .get(0) + .findChildrenByType(TreeType.BR) + .get(0); + + namespaceBrWs = br.getFirstChild(TreeType.WS); + + if (namespaceBrWs != null) { + List firstComments = namespaceBrWs.getChildrenByType(TreeType.COMMENT); + putLeadingComments(ids.get(0), firstComments); + // Remove these first comments since they get recreated when the use statements are sorted. + firstComments.forEach(c -> namespaceBrWs.getTree().removeChild(c.getTree())); + } else { + // Create an empty WS node since we might need to add comments here later. + br.getTree().appendChild(TokenTree.of(TreeType.WS)); + namespaceBrWs = br.getFirstChild(TreeType.WS); + } + } + + private void putLeadingComment(TokenTree id, TreeCursor comment) { + putLeadingComments(id, Collections.singletonList(comment)); + } + + private void putLeadingComments(TokenTree id, List comments) { + leadingComments.computeIfAbsent(id, i -> new ArrayList<>()).addAll(comments); + } + + private List getTrailingComments(int index) { + if (index == ids.size() - 1) { + return endComments; + } else { + return leadingComments.computeIfAbsent(ids.get(index + 1), id -> new ArrayList<>()); + } + } + + private void addFirstUse(TreeCursor useSection) { + // Add the first statement. Swap out its leading comments from the preceding NAMESPACE_STATEMENT's BR WS. + TokenTree id = ids.get(0); + TokenTree useStatement = createUseStatement(id, lineComments.get(id), getTrailingComments(0)); + + // Add any leading comments to the namespace BR WS so they aren't trailing comments. + if (leadingComments.get(id) != null) { + leadingComments.get(id).forEach(comment -> namespaceBrWs.getTree().appendChild(comment.getTree())); + } + + useSection.getTree().appendChild(useStatement); + } + + private void addSubsequentUse(TreeCursor useSection) { + for (int i = 1; i < ids.size(); i++) { + TokenTree id = ids.get(i); + TokenTree useStatement = createUseStatement(id, lineComments.get(id), getTrailingComments(i)); + useSection.getTree().appendChild(useStatement); + } + } + + private TokenTree createUseStatement(TokenTree id, TreeCursor lineComment, List trailingComments) { + TokenTree result = TokenTree.of(TreeType.USE_STATEMENT); + + CapturedToken useToken = CapturedToken.builder().token(IdlToken.IDENTIFIER).lexeme("use").build(); + result.appendChild(TokenTree.of(useToken)); + + TokenTree sp = TokenTree.of(TreeType.SP); + sp.appendChild(TokenTree.of(CapturedToken.builder().token(IdlToken.SPACE).lexeme(" ").build())); + result.appendChild(sp); + + result.appendChild(id); + + TokenTree br = TokenTree.of(TreeType.BR); + result.appendChild(br); + + if (lineComment != null) { + br.appendChild(sp); + // Modify the position of the comment so that the formatter knows it's a trailing line comment. + CapturedToken updatedLineComment = lineComment.getTree() + .tokens() + .findFirst() + .get() + .toBuilder() + // Set the line and column to zero so the formatter knows it's on the same line as the + // statement, making it an end-of-line comment. + .startLine(0) + .endLine(0) + .build(); + TokenTree commentTree = TokenTree.of(TreeType.COMMENT); + commentTree.appendChild(TokenTree.of(updatedLineComment)); + br.appendChild(commentTree); + } else { + br.appendChild(TokenTree.of(CapturedToken.builder().token(IdlToken.NEWLINE).lexeme("\n").build())); + } + + TokenTree ws = TokenTree.of(TreeType.WS); + trailingComments.forEach(tc -> ws.appendChild(tc.getTree())); + br.appendChild(ws); + + return result; + } + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTree.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTree.java new file mode 100644 index 00000000000..2967bde648a --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTree.java @@ -0,0 +1,189 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import java.util.List; +import java.util.stream.Stream; +import software.amazon.smithy.model.FromSourceLocation; +import software.amazon.smithy.model.loader.IdlToken; +import software.amazon.smithy.model.loader.IdlTokenizer; + +/** + * Provides a labeled tree of tokens returned from {@link IdlTokenizer}. + * + *

This abstraction is a kind of parse tree based on lexer tokens, with the key difference being it bottoms-out at + * {@link IdlToken} values rather than the more granular grammar productions for identifiers, strings, etc. + * + *

Each consumed IDL token is present in the tree, and grouped together into nodes with labels defined by its + * {@link TreeType}. + */ +public interface TokenTree extends FromSourceLocation { + + /** + * Create the root of a TokenTree from an {@link IdlTokenizer}. + * + * @param tokenizer Tokenizer to traverse. + * @return Returns the root of the created tree. + */ + static TokenTree of(IdlTokenizer tokenizer) { + CapturingTokenizer capturingTokenizer = new CapturingTokenizer(tokenizer); + TreeType.IDL.parse(capturingTokenizer); + return capturingTokenizer.getRoot(); + } + + /** + * Create a leaf tree from a single token. + * + * @param token Token to wrap into a tree. + * @return Returns the created tree. + */ + static TokenTree of(CapturedToken token) { + return new TokenTreeLeaf(token); + } + + /** + * Create an empty tree of a specific {@code type}. + * + * @param type Tree type to create. + * @return Returns the created tree. + */ + static TokenTree of(TreeType type) { + return new TokenTreeNode(type); + } + + /** + * Create an error tree with the given {@code error} message. + * + * @param error Error message. + * @return Returns the created tree. + */ + static TokenTree fromError(String error) { + return new TokenTreeError(error); + } + + /** + * Get the token tree type. + * + * @return Returns the type. + */ + TreeType getType(); + + /** + * Get direct children of the tree. + * + * @return Returns direct children. + */ + List getChildren(); + + /** + * Detect if the tree is empty (that is, a non-leaf that has no children or tokens). + * + * @return Return true if the tree has no children or tokens. + */ + boolean isEmpty(); + + /** + * Append a child to the tree. + * + * @param tree Tree to append. + * @throws UnsupportedOperationException if the tree is a leaf. + */ + void appendChild(TokenTree tree); + + /** + * Remove a child tree by referential equality. + * + * @param tree Tree to remove. + * @return Return true only if this child was found and removed. + */ + boolean removeChild(TokenTree tree); + + /** + * Replace a matching child with the given replacement using referential equality. + * + * @param find Child to find and replace, using referential equality. + * @param replacement Replacement to use instead. + * @return Returns true only if a child was replaced. + */ + boolean replaceChild(TokenTree find, TokenTree replacement); + + /** + * Get a flattened stream of all captured tokens contained within the tree. + * + * @return Returns the contained tokens. + */ + Stream tokens(); + + /** + * Get the tokens contained in the tree as a single concatenated string. + * + * @return Returns a concatenated string of tokens. + */ + String concatTokens(); + + /** + * Get the error associated with the tree, or {@code null} if not present. + * + * @return Returns the nullable error message. + */ + default String getError() { + return null; + } + + /** + * Create a zipper for the current tree node, treating it as the root of the tree. + * + * @return Returns the zipper cursor for the current node. + */ + default TreeCursor zipper() { + return TreeCursor.of(this); + } + + /** + * Get the absolute start position of the tree, starting at 0. + * + * @return Returns the start position of this tree. + */ + int getStartPosition(); + + /** + * Get the line the tree starts, starting at 1. + * + * @return Returns the start line. + */ + int getStartLine(); + + /** + * Get the column the tree starts, starting at 1. + * + * @return Returns the start column. + */ + int getStartColumn(); + + /** + * Get the line the tree ends, starting at 1. + * + * @return Returns the end line. + */ + int getEndLine(); + + /** + * Get the column the tree ends, starting at 1. + * + * @return Returns the end column. + */ + int getEndColumn(); +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTreeError.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTreeError.java new file mode 100644 index 00000000000..1795e98d775 --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTreeError.java @@ -0,0 +1,43 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import java.util.Objects; + +final class TokenTreeError extends TokenTreeNode { + + private final String error; + + TokenTreeError(String error) { + super(TreeType.ERROR); + this.error = Objects.requireNonNull(error); + } + + @Override + public String getError() { + return error; + } + + @Override + public boolean equals(Object o) { + return super.equals(o) && error.equals(((TokenTreeError) o).error); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), getError()); + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTreeLeaf.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTreeLeaf.java new file mode 100644 index 00000000000..562e05f942a --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTreeLeaf.java @@ -0,0 +1,126 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; +import software.amazon.smithy.model.SourceLocation; + +final class TokenTreeLeaf implements TokenTree { + + private final CapturedToken token; + + TokenTreeLeaf(CapturedToken token) { + this.token = token; + } + + @Override + public SourceLocation getSourceLocation() { + return token.getSourceLocation(); + } + + @Override + public Stream tokens() { + return Stream.of(token); + } + + @Override + public String concatTokens() { + return token.getLexeme().toString(); + } + + @Override + public TreeType getType() { + return TreeType.TOKEN; + } + + @Override + public List getChildren() { + return Collections.emptyList(); + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public void appendChild(TokenTree tree) { + throw new UnsupportedOperationException("Cannot append a child to a leaf node"); + } + + @Override + public boolean removeChild(TokenTree tree) { + return false; + } + + @Override + public boolean replaceChild(TokenTree find, TokenTree replacement) { + return false; + } + + @Override + public String toString() { + if (token.getErrorMessage() != null) { + return token.getIdlToken() + "(" + token.getErrorMessage() + ')'; + } else { + return token.getIdlToken().getDebug(token.getLexeme()); + } + } + + @Override + public int getStartPosition() { + return token.getPosition(); + } + + @Override + public int getStartLine() { + return token.getStartLine(); + } + + @Override + public int getStartColumn() { + return token.getStartColumn(); + } + + @Override + public int getEndLine() { + return token.getEndLine(); + } + + @Override + public int getEndColumn() { + return token.getEndColumn(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (o == null || getClass() != o.getClass()) { + return false; + } else { + return this.token.equals(((TokenTreeLeaf) o).token); + } + } + + @Override + public int hashCode() { + return Objects.hash(token); + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTreeNode.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTreeNode.java new file mode 100644 index 00000000000..3b1b2466c49 --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TokenTreeNode.java @@ -0,0 +1,146 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; +import software.amazon.smithy.model.SourceLocation; + +class TokenTreeNode implements TokenTree { + + private final TreeType treeType; + private final List children = new ArrayList<>(); + + TokenTreeNode(TreeType treeType) { + this.treeType = treeType; + } + + @Override + public SourceLocation getSourceLocation() { + return getChildren().isEmpty() ? SourceLocation.NONE : getChildren().get(0).getSourceLocation(); + } + + @Override + public final TreeType getType() { + return treeType; + } + + @Override + public final Stream tokens() { + return children.stream().flatMap(TokenTree::tokens); + } + + @Override + public String concatTokens() { + StringBuilder result = new StringBuilder(); + tokens().forEach(token -> result.append(token.getLexeme())); + return result.toString(); + } + + @Override + public final List getChildren() { + return children; + } + + @Override + public boolean isEmpty() { + return getChildren().isEmpty(); + } + + @Override + public final void appendChild(TokenTree tree) { + children.add(tree); + } + + @Override + public boolean removeChild(TokenTree tree) { + return children.removeIf(c -> c == tree); + } + + @Override + public boolean replaceChild(TokenTree find, TokenTree replacement) { + for (int i = 0; i < children.size(); i++) { + if (children.get(i) == find) { + children.set(i, replacement); + return true; + } + } + return false; + } + + @Override + public final String toString() { + StringBuilder result = new StringBuilder(); + result.append(getType()) + .append(" (").append(getStartLine()).append(", ").append(getStartColumn()) + .append(") - (") + .append(getEndLine()).append(", ").append(getEndColumn()) + .append(") {") + .append('\n'); + if (getError() != null) { + result.append(" ").append(getError()).append("\n ---\n"); + } + for (TokenTree child : children) { + result.append(" ").append(child.toString().replace("\n", "\n ")).append('\n'); + } + result.append('}'); + return result.toString(); + } + + @Override + public final int getStartPosition() { + return getChildren().isEmpty() ? 0 : getChildren().get(0).getStartPosition(); + } + + @Override + public final int getStartLine() { + return getChildren().isEmpty() ? 0 : getChildren().get(0).getStartLine(); + } + + @Override + public final int getStartColumn() { + return getChildren().isEmpty() ? 0 : getChildren().get(0).getStartColumn(); + } + + @Override + public final int getEndLine() { + return children.isEmpty() ? getStartLine() : children.get(children.size() - 1).getEndLine(); + } + + @Override + public final int getEndColumn() { + return children.isEmpty() ? getStartColumn() : children.get(children.size() - 1).getEndColumn(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (o == null || getClass() != o.getClass()) { + return false; + } else { + TokenTreeNode other = (TokenTreeNode) o; + return treeType == other.treeType && children.equals(other.children); + } + } + + @Override + public int hashCode() { + return Objects.hash(treeType, children); + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TreeCursor.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TreeCursor.java new file mode 100644 index 00000000000..b5a103e5907 --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TreeCursor.java @@ -0,0 +1,336 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.ListIterator; +import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Stream; +import software.amazon.smithy.model.FromSourceLocation; +import software.amazon.smithy.model.SourceLocation; + +/** + * Externally traverses a {@link TokenTree} to provide access to parents and siblings. + * + * @see TokenTree#zipper() + */ +public final class TreeCursor implements FromSourceLocation { + + private final TokenTree tree; + private final TreeCursor parent; + + private TreeCursor(TokenTree tree, TreeCursor parent) { + this.tree = tree; + this.parent = parent; + } + + /** + * Create a TreeCursor from the given TokenTree, treating it as the root of the tree. + * + * @param tree Tree to create a cursor from. + * @return Returns the created cursor. + */ + public static TreeCursor of(TokenTree tree) { + return new TreeCursor(tree, null); + } + + @Override + public SourceLocation getSourceLocation() { + return getTree().getSourceLocation(); + } + + /** + * Get the wrapped {@link TokenTree}. + * + * @return Return the token tree. + */ + public TokenTree getTree() { + return tree; + } + + /** + * Get the parent cursor, or null if not present. + * + * @return Nullable parent cursor. + */ + public TreeCursor getParent() { + return parent; + } + + /** + * Get the root of the tree, returning itself if the tree has no parent. + * + * @return Non-nullable root tree. + */ + public TreeCursor getRoot() { + TreeCursor current = this; + while (current.parent != null) { + current = current.parent; + } + return current; + } + + /** + * Get a list of tree cursors that lead up to the current tree, starting from the root as the first element, and + * including the current tree as the last element. + * + * @return Returns the path to the current tree from the root. + */ + public List getPathToCursor() { + List path = new ArrayList<>(); + TreeCursor current = this; + do { + path.add(current); + current = current.parent; + } while (current != null); + Collections.reverse(path); + return path; + } + + /** + * Get the previous sibling of this tree, if present. + * + * @return Return the nullable previous sibling. + */ + public TreeCursor getPreviousSibling() { + return getSibling(-1); + } + + /** + * Get the next sibling of this tree, if present. + * + * @return Return the nullable next sibling. + */ + public TreeCursor getNextSibling() { + return getSibling(1); + } + + private TreeCursor getSibling(int offset) { + if (parent == null) { + return null; + } + + List siblings = parent.getTree().getChildren(); + int myPosition = siblings.indexOf(this.tree); + + if (myPosition == -1) { + return null; + } + + int target = myPosition + offset; + if (target < 0 || target > siblings.size() - 1) { + return null; + } + + return new TreeCursor(siblings.get(target), parent); + } + + /** + * Get all children of the tree as a list of cursors. + * + * @return Return the cursors to each child. + */ + public List getChildren() { + List result = new ArrayList<>(getTree().getChildren().size()); + for (TokenTree child : tree.getChildren()) { + result.add(new TreeCursor(child, this)); + } + return result; + } + + /** + * Get a stream of child cursors. + * + * @return Returns children as a stream. + */ + Stream children() { + return getTree().getChildren().stream().map(child -> new TreeCursor(child, this)); + } + + /** + * Get direct children from the current tree of a specific type. + * + * @param types Types of children to get. + * @return Returns the collected children, or an empty list. + */ + public List getChildrenByType(TreeType... types) { + List result = new ArrayList<>(); + for (int i = 0; i < tree.getChildren().size(); i++) { + TokenTree child = tree.getChildren().get(i); + for (TreeType type : types) { + if (child.getType() == type) { + result.add(new TreeCursor(child, this)); + break; + } + } + } + return result; + } + + /** + * Get the first child of the wrapped tree. + * + * @return Return the first child, or null if the tree has no children. + */ + public TreeCursor getFirstChild() { + if (tree.getChildren().isEmpty()) { + return null; + } else { + return new TreeCursor(tree.getChildren().get(0), this); + } + } + + /** + * Get the first child of the wrapped tree with the given type. + * + * @param type Child type to get. + * @return Return the first child, or null if a matching child is not found. + */ + public TreeCursor getFirstChild(TreeType type) { + for (TokenTree child : getTree().getChildren()) { + if (child.getType() == type) { + return new TreeCursor(child, this); + } + } + return null; + } + + /** + * Get the last child of the wrapped tree. + * + * @return Return the last child, or null if the tree has no children. + */ + public TreeCursor getLastChild() { + if (tree.getChildren().isEmpty()) { + return null; + } else { + return new TreeCursor(tree.getChildren().get(tree.getChildren().size() - 1), this); + } + } + + /** + * Get the last child of the wrapped tree with the given type. + * + * @param type Child type to get. + * @return Return the last child, or null if a matching child is not found. + */ + public TreeCursor getLastChild(TreeType type) { + List children = tree.getChildren(); + ListIterator iterator = children.listIterator(children.size()); + while (iterator.hasPrevious()) { + TokenTree child = iterator.previous(); + if (child.getType() == type) { + return new TreeCursor(child, this); + } + } + return null; + } + + /** + * Recursively find every node in the tree that has the given {@code TreeType}. + * + * @param types Types of children to return. + * @return Returns the matching tree cursors. + */ + public List findChildrenByType(TreeType... types) { + return findChildren(c -> { + TreeType treeType = c.getTree().getType(); + for (TreeType type : types) { + if (treeType == type) { + return true; + } + } + return false; + }); + } + + /** + * Recursively finds every node in the tree that matches the given predicate. + * + * @param predicate Predicate to test each recursive child against. + * @return Returns the matching tree cursors, or an empty list if none are found. + */ + private List findChildren(Predicate predicate) { + List result = new ArrayList<>(); + recursiveFindChildren(this, result, predicate); + return result; + } + + private void recursiveFindChildren(TreeCursor cursor, List cursors, Predicate predicate) { + for (TreeCursor tree : cursor.getChildren()) { + if (predicate.test(tree)) { + cursors.add(tree); + } + recursiveFindChildren(tree, cursors, predicate); + } + } + + /** + * Find the innermost tree that contains the given coordinates. + * + * @param line Line to find. + * @param column Column to find. + * @return Returns the innermost tree that contains the coordinates. + */ + public TreeCursor findAt(int line, int column) { + TreeCursor current = this; + + outer: while (true) { + for (TreeCursor child : current.getChildren()) { + TokenTree childTree = child.getTree(); + int startLine = childTree.getStartLine(); + int endLine = childTree.getEndLine(); + int startColumn = childTree.getStartColumn(); + int endColumn = childTree.getEndColumn(); + boolean isMatch = false; + if (line == startLine && line == endLine) { + // Column span checks are exclusive to not match the ends of tokens. + isMatch = column >= startColumn && column < endColumn; + } else if (line == startLine && column >= startColumn) { + isMatch = true; + } else if (line == endLine && column <= endColumn) { + isMatch = true; + } else if (line > startLine && line < endLine) { + isMatch = true; + } + if (isMatch) { + current = child; + continue outer; + } + } + return current; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (o == null || getClass() != o.getClass()) { + return false; + } + TreeCursor cursor = (TreeCursor) o; + return getTree().equals(cursor.getTree()) && Objects.equals(getParent(), cursor.getParent()); + } + + @Override + public int hashCode() { + return Objects.hash(getTree(), getParent()); + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TreeType.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TreeType.java new file mode 100644 index 00000000000..c63b4d69339 --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/TreeType.java @@ -0,0 +1,1135 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.syntax; + +import software.amazon.smithy.model.loader.IdlToken; +import software.amazon.smithy.model.loader.IdlTokenizer; +import software.amazon.smithy.model.loader.ModelSyntaxException; +import software.amazon.smithy.model.shapes.ShapeType; + +/** + * Defines the tree type. + * + *

These types typically map 1:1 to a production in the Smithy IDL grammar, except that tree types bottom-out at + * {@link IdlToken}. + * + *

For example: + * + *

    + *
  • The {@code Identifier} production is combined into a single {@link #IDENTIFIER} node. + * {@code IdentifierStart} and {@code IdentifierChars} are not exposed in the token tree.
    • + *
  • The {@code Number} production is combined into a single {@link #NUMBER} node. Productions like + * {@code DecimalPoint}, {@code Exp}, etc are not exposed in the token tree.
    • + *
  • The {@code QuotedText} production is combined into a single {@link #QUOTED_TEXT} node. + *
  • The {@code TextBlock} production is combined into a single {@link #TEXT_BLOCK} node. + *
+ */ +public enum TreeType { + IDL { + @Override + void parse(CapturingTokenizer tokenizer) { + optionalWs(tokenizer); + CONTROL_SECTION.parse(tokenizer); + METADATA_SECTION.parse(tokenizer); + SHAPE_SECTION.parse(tokenizer); + tokenizer.expect(IdlToken.EOF); + } + }, + + CONTROL_SECTION { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + while (tokenizer.getCurrentToken() == IdlToken.DOLLAR) { + CONTROL_STATEMENT.parse(tokenizer); + } + }); + } + }, + + CONTROL_STATEMENT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.DOLLAR); + tokenizer.next(); // $ + NODE_OBJECT_KEY.parse(tokenizer); + optionalSpaces(tokenizer); + tokenizer.expect(IdlToken.COLON); + tokenizer.next(); + optionalSpaces(tokenizer); + NODE_VALUE.parse(tokenizer); + BR.parse(tokenizer); + }); + } + }, + + METADATA_SECTION { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + while (tokenizer.isCurrentLexeme("metadata")) { + METADATA_STATEMENT.parse(tokenizer); + } + }); + } + }, + + METADATA_STATEMENT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.next(); // append metadata + optionalSpaces(tokenizer); + NODE_OBJECT_KEY.parse(tokenizer); + optionalSpaces(tokenizer); + tokenizer.expect(IdlToken.EQUAL); + tokenizer.next(); + optionalSpaces(tokenizer); + NODE_VALUE.parse(tokenizer); + BR.parse(tokenizer); + }); + } + }, + + SHAPE_SECTION { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + NAMESPACE_STATEMENT.parse(tokenizer); + USE_SECTION.parse(tokenizer); + SHAPE_STATEMENTS.parse(tokenizer); + }); + } + }, + + NAMESPACE_STATEMENT { + @Override + void parse(CapturingTokenizer tokenizer) { + if (tokenizer.isCurrentLexeme("namespace")) { + tokenizer.withState(this, () -> { + tokenizer.next(); // skip "namespace" + SP.parse(tokenizer); + NAMESPACE.parse(tokenizer); + BR.parse(tokenizer); + }); + } else if (tokenizer.hasNext()) { + tokenizer.withState(this, () -> { + throw new ModelSyntaxException( + "Expected a namespace definition but found " + + tokenizer.getCurrentToken().getDebug(tokenizer.getCurrentTokenLexeme()), + tokenizer.getCurrentTokenLocation()); + }); + } + } + }, + + USE_SECTION { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + while (tokenizer.getCurrentToken() == IdlToken.IDENTIFIER) { + // Don't over-parse here for unions. + String keyword = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + if (!keyword.equals("use")) { + break; + } + USE_STATEMENT.parse(tokenizer); + } + }); + } + }, + + USE_STATEMENT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.next(); // Skip "use" + SP.parse(tokenizer); + ABSOLUTE_ROOT_SHAPE_ID.parse(tokenizer); + BR.parse(tokenizer); + }); + } + }, + + SHAPE_STATEMENTS { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + while (tokenizer.hasNext()) { + SHAPE_OR_APPLY_STATEMENT.parse(tokenizer); + BR.parse(tokenizer); + } + }); + } + }, + + SHAPE_OR_APPLY_STATEMENT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + if (tokenizer.expect(IdlToken.IDENTIFIER, IdlToken.AT) == IdlToken.AT) { + SHAPE_STATEMENT.parse(tokenizer); + } else if (tokenizer.isCurrentLexeme("apply")) { + APPLY_STATEMENT.parse(tokenizer); + } else { + SHAPE_STATEMENT.parse(tokenizer); + } + }); + } + }, + + SHAPE_STATEMENT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + TRAIT_STATEMENTS.parse(tokenizer); + SHAPE.parse(tokenizer); + }); + } + }, + + SHAPE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + ShapeType type = ShapeType.fromString(tokenizer.internString(tokenizer.getCurrentTokenLexeme())) + .orElseThrow(() -> new ModelSyntaxException("Expected a valid shape type", + tokenizer.getCurrentTokenLocation())); + switch (type) { + case ENUM: + case INT_ENUM: + ENUM_SHAPE.parse(tokenizer); + break; + case SERVICE: + case RESOURCE: + ENTITY_SHAPE.parse(tokenizer); + break; + case OPERATION: + OPERATION_SHAPE.parse(tokenizer); + break; + default: + switch (type.getCategory()) { + case SIMPLE: + SIMPLE_SHAPE.parse(tokenizer); + break; + case AGGREGATE: + AGGREGATE_SHAPE.parse(tokenizer); + break; + default: + throw new UnsupportedOperationException("Unexpected type: " + type); + } + } + }); + } + }, + + SIMPLE_SHAPE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + parseShapeTypeAndName(tokenizer, SIMPLE_TYPE_NAME); + parseOptionalMixins(tokenizer); + }); + } + }, + + SIMPLE_TYPE_NAME { + @Override + void parse(CapturingTokenizer tokenizer) { + // Assumes that the current token is a valid simple type name validated by SHAPE. + tokenizer.withState(this, tokenizer::next); + } + }, + + ENUM_SHAPE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + parseShapeTypeAndName(tokenizer, ENUM_TYPE_NAME); + parseOptionalMixins(tokenizer); + + optionalWs(tokenizer); + ENUM_SHAPE_MEMBERS.parse(tokenizer); + }); + } + }, + + ENUM_TYPE_NAME { + @Override + void parse(CapturingTokenizer tokenizer) { + // Assumes that the current token is a valid enum type name validated by SHAPE. + tokenizer.withState(this, tokenizer::next); + } + }, + + ENUM_SHAPE_MEMBERS { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + optionalWs(tokenizer); + + while (tokenizer.hasNext() && tokenizer.getCurrentToken() != IdlToken.RBRACE) { + ENUM_SHAPE_MEMBER.parse(tokenizer); + optionalWs(tokenizer); + } + + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); + }); + } + }, + + ENUM_SHAPE_MEMBER { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + TRAIT_STATEMENTS.parse(tokenizer); + IDENTIFIER.parse(tokenizer); + parseOptionalValueAssignment(tokenizer); + }); + } + }, + + AGGREGATE_SHAPE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + parseShapeTypeAndName(tokenizer, AGGREGATE_TYPE_NAME); + parseSharedStructureBodyWithinInline(tokenizer); + }); + } + }, + + AGGREGATE_TYPE_NAME { + @Override + void parse(CapturingTokenizer tokenizer) { + // Assumes that the current token is a valid simple type name validated by SHAPE. + tokenizer.withState(this, tokenizer::next); + } + }, + + // Don't use this directly. Instead, use parseOptionalForResource + FOR_RESOURCE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.next(); // Skip "for" + SP.parse(tokenizer); + SHAPE_ID.parse(tokenizer); + }); + } + }, + + SHAPE_MEMBERS { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + optionalWs(tokenizer); + + while (tokenizer.hasNext() && tokenizer.getCurrentToken() != IdlToken.RBRACE) { + SHAPE_MEMBER.parse(tokenizer); + optionalWs(tokenizer); + } + + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); + }); + } + }, + + SHAPE_MEMBER { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + TRAIT_STATEMENTS.parse(tokenizer); + if (tokenizer.expect(IdlToken.IDENTIFIER, IdlToken.DOLLAR) == IdlToken.DOLLAR) { + ELIDED_SHAPE_MEMBER.parse(tokenizer); + } else { + EXPLICIT_SHAPE_MEMBER.parse(tokenizer); + } + parseOptionalValueAssignment(tokenizer); + }); + } + }, + + EXPLICIT_SHAPE_MEMBER { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + IDENTIFIER.parse(tokenizer); + optionalSpaces(tokenizer); + tokenizer.expect(IdlToken.COLON); + tokenizer.next(); + optionalSpaces(tokenizer); + SHAPE_ID.parse(tokenizer); + }); + } + }, + + ELIDED_SHAPE_MEMBER { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.DOLLAR); + tokenizer.next(); + IDENTIFIER.parse(tokenizer); + }); + } + }, + + ENTITY_SHAPE { + @Override + void parse(CapturingTokenizer tokenizer) { + // Assumes that the shape type is a valid "service" or "resource". + tokenizer.withState(this, () -> { + parseShapeTypeAndName(tokenizer, ENTITY_TYPE_NAME); + + parseOptionalMixins(tokenizer); + + optionalWs(tokenizer); + tokenizer.expect(IdlToken.LBRACE); + NODE_OBJECT.parse(tokenizer); + }); + } + }, + + ENTITY_TYPE_NAME { + @Override + void parse(CapturingTokenizer tokenizer) { + // Assumes that the current token is a valid entity type name validated by SHAPE. + tokenizer.withState(this, tokenizer::next); + } + }, + + OPERATION_SHAPE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + parseShapeTypeAndName(tokenizer); + + parseOptionalMixins(tokenizer); + + optionalWs(tokenizer); + OPERATION_BODY.parse(tokenizer); + }); + } + }, + + OPERATION_BODY { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + optionalWs(tokenizer); + while (tokenizer.hasNext() && tokenizer.getCurrentToken() != IdlToken.RBRACE) { + OPERATION_PROPERTY.parse(tokenizer); + optionalWs(tokenizer); + } + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); + }); + } + }, + + OPERATION_PROPERTY { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.IDENTIFIER); + if (tokenizer.isCurrentLexeme("input")) { + OPERATION_INPUT.parse(tokenizer); + } else if (tokenizer.isCurrentLexeme("output")) { + OPERATION_OUTPUT.parse(tokenizer); + } else if (tokenizer.isCurrentLexeme("errors")) { + OPERATION_ERRORS.parse(tokenizer); + } else { + throw new ModelSyntaxException("Expected 'input', 'output', or 'errors'. Found '" + + tokenizer.getCurrentTokenLexeme() + "'", + tokenizer.getCurrentTokenLocation()); + } + }); + } + }, + + OPERATION_INPUT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.IDENTIFIER); + tokenizer.next(); // skip "input" + optionalWs(tokenizer); + operationInputOutputDefinition(tokenizer); + }); + } + }, + + OPERATION_OUTPUT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.IDENTIFIER); + tokenizer.next(); // skip "output" + optionalWs(tokenizer); + operationInputOutputDefinition(tokenizer); + }); + } + }, + + INLINE_AGGREGATE_SHAPE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.WALRUS); + tokenizer.next(); + optionalWs(tokenizer); + TRAIT_STATEMENTS.parse(tokenizer); + parseSharedStructureBodyWithinInline(tokenizer); + }); + } + }, + + OPERATION_ERRORS { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.IDENTIFIER); + tokenizer.next(); // skip "errors" + optionalWs(tokenizer); + tokenizer.expect(IdlToken.COLON); + tokenizer.next(); + optionalWs(tokenizer); + tokenizer.expect(IdlToken.LBRACKET); + tokenizer.next(); + optionalWs(tokenizer); + while (tokenizer.hasNext() && tokenizer.getCurrentToken() != IdlToken.RBRACKET) { + SHAPE_ID.parse(tokenizer); + optionalWs(tokenizer); + } + tokenizer.expect(IdlToken.RBRACKET); + tokenizer.next(); + }); + } + }, + + // Mixins = + // [SP] %s"with" [WS] "[" [WS] 1*(ShapeId [WS]) "]" + // Don't use this directly. Instead, use parseOptionalMixins + MIXINS { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.next(); // Skip "with" + optionalWs(tokenizer); + + tokenizer.expect(IdlToken.LBRACKET); + tokenizer.next(); + optionalWs(tokenizer); + + do { + SHAPE_ID.parse(tokenizer); + optionalWs(tokenizer); + } while (tokenizer.expect(IdlToken.IDENTIFIER, IdlToken.RBRACKET) == IdlToken.IDENTIFIER); + + optionalWs(tokenizer); + tokenizer.expect(IdlToken.RBRACKET); + tokenizer.next(); + }); + } + }, + + // Don't use this directly. Instead, use parseOptionalValueAssignment + VALUE_ASSIGNMENT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + optionalSpaces(tokenizer); + tokenizer.expect(IdlToken.EQUAL); + tokenizer.next(); + optionalSpaces(tokenizer); + NODE_VALUE.parse(tokenizer); + + optionalSpaces(tokenizer); + if (tokenizer.getCurrentToken() == IdlToken.COMMA) { + COMMA.parse(tokenizer); + } + + BR.parse(tokenizer); + }); + } + }, + + TRAIT_STATEMENTS { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + while (tokenizer.getCurrentToken() == IdlToken.AT) { + TRAIT.parse(tokenizer); + optionalWs(tokenizer); + } + }); + } + }, + + TRAIT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.AT); + tokenizer.next(); + SHAPE_ID.parse(tokenizer); + if (tokenizer.getCurrentToken() == IdlToken.LPAREN) { + TRAIT_BODY.parse(tokenizer); + } + }); + } + }, + + TRAIT_BODY { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.LPAREN); + tokenizer.next(); + optionalWs(tokenizer); + + if (tokenizer.getCurrentToken() != IdlToken.RPAREN) { + tokenizer.expect(IdlToken.LBRACE, IdlToken.LBRACKET, IdlToken.TEXT_BLOCK, IdlToken.STRING, + IdlToken.NUMBER, IdlToken.IDENTIFIER); + switch (tokenizer.getCurrentToken()) { + case LBRACE: + case LBRACKET: + case TEXT_BLOCK: + case NUMBER: + TRAIT_NODE.parse(tokenizer); + break; + case STRING: + case IDENTIFIER: + default: + CapturedToken nextPastWs = tokenizer.peekWhile(1, token -> + token.isWhitespace() || token == IdlToken.DOC_COMMENT); + if (nextPastWs.getIdlToken() == IdlToken.COLON) { + TRAIT_STRUCTURE.parse(tokenizer); + } else { + TRAIT_NODE.parse(tokenizer); + } + } + } + + tokenizer.expect(IdlToken.RPAREN); // Expect and skip ")" + tokenizer.next(); + }); + } + }, + + TRAIT_STRUCTURE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + do { + NODE_OBJECT_KVP.parse(tokenizer); + optionalWs(tokenizer); + } while (tokenizer.getCurrentToken() != IdlToken.RPAREN && tokenizer.hasNext()); + }); + } + }, + + TRAIT_NODE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + // parse these as NODE_VALUE. + NODE_VALUE.parse(tokenizer); + optionalWs(tokenizer); + }); + } + }, + + APPLY_STATEMENT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + // Try to see if this is a singular or block apply statement. + IdlToken peek = tokenizer + .peekWhile(1, t -> t != IdlToken.AT && t != IdlToken.LBRACE) + .getIdlToken(); + if (peek == IdlToken.LBRACE) { + APPLY_STATEMENT_BLOCK.parse(tokenizer); + } else { + APPLY_STATEMENT_SINGULAR.parse(tokenizer); + } + }); + } + }, + + APPLY_STATEMENT_SINGULAR { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.next(); // Skip "apply" + SP.parse(tokenizer); + SHAPE_ID.parse(tokenizer); + WS.parse(tokenizer); + TRAIT.parse(tokenizer); + }); + } + }, + + APPLY_STATEMENT_BLOCK { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.next(); // Skip "apply" + SP.parse(tokenizer); + SHAPE_ID.parse(tokenizer); + WS.parse(tokenizer); + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + optionalWs(tokenizer); + TRAIT_STATEMENTS.parse(tokenizer); + optionalWs(tokenizer); + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); + }); + } + }, + + NODE_VALUE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + IdlToken token = tokenizer.expect(IdlToken.STRING, IdlToken.TEXT_BLOCK, IdlToken.NUMBER, + IdlToken.IDENTIFIER, IdlToken.LBRACE, IdlToken.LBRACKET); + switch (token) { + case IDENTIFIER: + if (tokenizer.isCurrentLexeme("true") || tokenizer.isCurrentLexeme("false") + || tokenizer.isCurrentLexeme("null")) { + NODE_KEYWORD.parse(tokenizer); + } else { + NODE_STRING_VALUE.parse(tokenizer); + } + break; + case STRING: + case TEXT_BLOCK: + NODE_STRING_VALUE.parse(tokenizer); + break; + case NUMBER: + NUMBER.parse(tokenizer); + break; + case LBRACE: + NODE_OBJECT.parse(tokenizer); + break; + case LBRACKET: + default: + NODE_ARRAY.parse(tokenizer); + break; + } + }); + } + }, + + NODE_ARRAY { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.LBRACKET); + tokenizer.next(); + optionalWs(tokenizer); + do { + if (tokenizer.getCurrentToken() == IdlToken.RBRACKET) { + break; + } + NODE_VALUE.parse(tokenizer); + optionalWs(tokenizer); + } while (tokenizer.hasNext()); + tokenizer.expect(IdlToken.RBRACKET); + tokenizer.next(); + }); + } + }, + + NODE_OBJECT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + optionalWs(tokenizer); + + while (tokenizer.hasNext()) { + if (tokenizer.expect(IdlToken.RBRACE, IdlToken.STRING, IdlToken.IDENTIFIER) == IdlToken.RBRACE) { + break; + } + NODE_OBJECT_KVP.parse(tokenizer); + optionalWs(tokenizer); + } + + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); + }); + } + }, + + NODE_OBJECT_KVP { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + NODE_OBJECT_KEY.parse(tokenizer); + optionalWs(tokenizer); + tokenizer.expect(IdlToken.COLON); + tokenizer.next(); + optionalWs(tokenizer); + NODE_VALUE.parse(tokenizer); + }); + } + }, + + NODE_OBJECT_KEY { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + if (tokenizer.expect(IdlToken.IDENTIFIER, IdlToken.STRING) == IdlToken.IDENTIFIER) { + IDENTIFIER.parse(tokenizer); + } else { + QUOTED_TEXT.parse(tokenizer); + } + }); + } + }, + + NODE_KEYWORD { + @Override + void parse(CapturingTokenizer tokenizer) { + // Assumes that the tokenizer is on "true"|"false"|"null". + tokenizer.withState(this, tokenizer::next); + } + }, + + NODE_STRING_VALUE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + switch (tokenizer.expect(IdlToken.STRING, IdlToken.TEXT_BLOCK, IdlToken.IDENTIFIER)) { + case STRING: + QUOTED_TEXT.parse(tokenizer); + break; + case TEXT_BLOCK: + TEXT_BLOCK.parse(tokenizer); + break; + case IDENTIFIER: + default: + SHAPE_ID.parse(tokenizer); + } + }); + } + }, + + QUOTED_TEXT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.STRING); + tokenizer.next(); + }); + } + }, + + TEXT_BLOCK { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.TEXT_BLOCK); + tokenizer.next(); + }); + } + }, + + NUMBER { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.NUMBER); + tokenizer.next(); + }); + } + }, + + SHAPE_ID { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + ROOT_SHAPE_ID.parse(tokenizer); + if (tokenizer.getCurrentToken() == IdlToken.DOLLAR) { + SHAPE_ID_MEMBER.parse(tokenizer); + } + }); + } + }, + + ROOT_SHAPE_ID { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + IdlToken after = tokenizer + .peekWhile(0, t -> t == IdlToken.DOT || t == IdlToken.IDENTIFIER).getIdlToken(); + if (after == IdlToken.POUND) { + ABSOLUTE_ROOT_SHAPE_ID.parse(tokenizer); + } else { + IDENTIFIER.parse(tokenizer); + } + }); + } + }, + + ABSOLUTE_ROOT_SHAPE_ID { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + NAMESPACE.parse(tokenizer); + tokenizer.expect(IdlToken.POUND); + tokenizer.next(); + IDENTIFIER.parse(tokenizer); + }); + } + }, + + SHAPE_ID_MEMBER { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.DOLLAR); + tokenizer.next(); + IDENTIFIER.parse(tokenizer); + }); + } + }, + + NAMESPACE { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + IDENTIFIER.parse(tokenizer); + while (tokenizer.getCurrentToken() == IdlToken.DOT) { + tokenizer.next(); + IDENTIFIER.parse(tokenizer); + } + }); + } + }, + + IDENTIFIER { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.IDENTIFIER); + tokenizer.next(); + }); + } + }, + + SP { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.SPACE); + while (tokenizer.getCurrentToken() == IdlToken.SPACE) { + tokenizer.next(); + } + }); + } + }, + + WS { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(WS_CHARS); + do { + switch (tokenizer.getCurrentToken()) { + case SPACE: + SP.parse(tokenizer); + break; + case NEWLINE: + tokenizer.next(); + break; + case COMMA: + COMMA.parse(tokenizer); + break; + case COMMENT: + case DOC_COMMENT: + COMMENT.parse(tokenizer); + break; + default: + throw new UnsupportedOperationException("Unexpected WS token: " + + tokenizer.getCurrentToken()); + } + } while (TreeType.isToken(tokenizer, WS_CHARS)); + }); + } + }, + + COMMENT { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.COMMENT, IdlToken.DOC_COMMENT); + tokenizer.next(); + }); + } + }, + + BR { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + optionalSpaces(tokenizer); + switch (tokenizer.expect(IdlToken.NEWLINE, IdlToken.COMMENT, IdlToken.DOC_COMMENT, IdlToken.EOF)) { + case COMMENT: + case DOC_COMMENT: + COMMENT.parse(tokenizer); + optionalWs(tokenizer); + break; + case NEWLINE: + tokenizer.next(); + optionalWs(tokenizer); + break; + case EOF: + default: + break; + } + }); + } + }, + + COMMA { + @Override + void parse(CapturingTokenizer tokenizer) { + tokenizer.withState(this, () -> { + tokenizer.expect(IdlToken.COMMA); + tokenizer.next(); + }); + } + }, + + /** + * An ERROR tree is created when a parser error is encountered; that is, any parse tree that contains this node + * is an invalid model. + */ + ERROR { + @Override + void parse(CapturingTokenizer tokenizer) { + throw new UnsupportedOperationException(); + } + }, + + /** + * The innermost node of the token tree that contains an actual token returned from {@link IdlTokenizer}. + */ + TOKEN { + @Override + void parse(CapturingTokenizer tokenizer) { + throw new UnsupportedOperationException(); + } + }; + + // For now, this also skips doc comments. We may later move doc comments out of WS. + private static final IdlToken[] WS_CHARS = {IdlToken.SPACE, IdlToken.NEWLINE, IdlToken.COMMA, + IdlToken.COMMENT, IdlToken.DOC_COMMENT}; + + abstract void parse(CapturingTokenizer tokenizer); + + private static boolean isToken(CapturingTokenizer tokenizer, IdlToken... tokens) { + IdlToken currentTokenType = tokenizer.getCurrentToken(); + + for (IdlToken token : tokens) { + if (currentTokenType == token) { + return true; + } + } + + return false; + } + + protected static void optionalWs(CapturingTokenizer tokenizer) { + if (isToken(tokenizer, WS_CHARS)) { + WS.parse(tokenizer); + } + } + + protected static void optionalSpaces(CapturingTokenizer tokenizer) { + if (tokenizer.getCurrentToken() == IdlToken.SPACE) { + TreeType.SP.parse(tokenizer); + } + } + + protected static void parseShapeTypeAndName(CapturingTokenizer tokenizer) { + parseShapeTypeAndName(tokenizer, null); + } + + protected static void parseShapeTypeAndName(CapturingTokenizer tokenizer, TreeType typeName) { + if (typeName == null) { + tokenizer.next(); + } else { + typeName.parse(tokenizer); // Skip the shape type + } + optionalSpaces(tokenizer); + IDENTIFIER.parse(tokenizer); // shape name + optionalSpaces(tokenizer); + } + + protected static void parseSharedStructureBodyWithinInline(CapturingTokenizer tokenizer) { + parseOptionalForResource(tokenizer); + parseOptionalMixins(tokenizer); + + optionalWs(tokenizer); + SHAPE_MEMBERS.parse(tokenizer); + } + + protected static void parseOptionalForResource(CapturingTokenizer tokenizer) { + optionalSpaces(tokenizer); + if (tokenizer.isCurrentLexeme("for")) { + FOR_RESOURCE.parse(tokenizer); + } + } + + protected static void parseOptionalMixins(CapturingTokenizer tokenizer) { + optionalSpaces(tokenizer); + if (tokenizer.isCurrentLexeme("with")) { + MIXINS.parse(tokenizer); + } + } + + protected static void parseOptionalValueAssignment(CapturingTokenizer tokenizer) { + if (tokenizer.peekPastSpaces().getIdlToken() == IdlToken.EQUAL) { + VALUE_ASSIGNMENT.parse(tokenizer); + } + } + + protected static void operationInputOutputDefinition(CapturingTokenizer tokenizer) { + if (tokenizer.expect(IdlToken.COLON, IdlToken.WALRUS) == IdlToken.COLON) { + tokenizer.next(); + optionalWs(tokenizer); + SHAPE_ID.parse(tokenizer); + } else { + INLINE_AGGREGATE_SHAPE.parse(tokenizer); + } + } +} diff --git a/smithy-syntax/src/main/java/software/amazon/smithy/syntax/package-info.java b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/package-info.java new file mode 100644 index 00000000000..617b4a0bbdc --- /dev/null +++ b/smithy-syntax/src/main/java/software/amazon/smithy/syntax/package-info.java @@ -0,0 +1,23 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +/** + * This package is currently marked as unstable. It will be marked as stable when it is integrated with the + * Smithy LSP. + */ +@SmithyUnstableApi +package software.amazon.smithy.syntax; + +import software.amazon.smithy.utils.SmithyUnstableApi; diff --git a/smithy-syntax/src/test/java/software/amazon/smithy/syntax/ParseAndFormatTest.java b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/ParseAndFormatTest.java new file mode 100644 index 00000000000..e97c6dc604e --- /dev/null +++ b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/ParseAndFormatTest.java @@ -0,0 +1,66 @@ +package software.amazon.smithy.syntax; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.loader.IdlTokenizer; +import software.amazon.smithy.utils.IoUtils; + +// A parameterized test that finds models in corpus, parses them, skipping files that end with ".formatted.smithy". +// If there is an x.formatted.smithy file, then ensure the model when formatted is equal to the formatted version. +// If there is no formatted version, then ensure that the model when formatted is equal to itself. +public class ParseAndFormatTest { + + private static final String CORPUS_DIR = "formatter"; + + @ParameterizedTest(name = "{0}") + @MethodSource("tests") + public void testRunner(Path filename) { + Path formattedFile = Paths.get(filename.toString().replace(".smithy", ".formatted.smithy")); + if (!Files.exists(formattedFile)) { + formattedFile = filename; + } + + // Ensure that the tests can be parsed by smithy-model too. + Model.assembler().addImport(filename).disableValidation().assemble().unwrap(); + if (!formattedFile.equals(filename)) { + Model.assembler().addImport(formattedFile).disableValidation().assemble().unwrap(); + } + + String model = IoUtils.readUtf8File(filename); + IdlTokenizer tokenizer = IdlTokenizer.create(filename.toString(), model); + TokenTree tree = TokenTree.of(tokenizer); + String formatted = Formatter.format(tree, 120); + String expected = IoUtils.readUtf8File(formattedFile); + + assertThat(formatted, equalTo(expected)); + } + + public static List tests() throws Exception { + List paths = new ArrayList<>(); + + try (Stream files = Files.walk(Paths.get(ParseAndFormatTest.class.getResource(CORPUS_DIR).toURI()))) { + files + .filter(Files::isRegularFile) + .filter(file -> { + String filename = file.toString(); + return filename.endsWith(".smithy") && !filename.endsWith(".formatted.smithy"); + }) + .forEach(paths::add); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return paths; + } +} diff --git a/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TokenTreeLeafTest.java b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TokenTreeLeafTest.java new file mode 100644 index 00000000000..93d11820536 --- /dev/null +++ b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TokenTreeLeafTest.java @@ -0,0 +1,19 @@ +package software.amazon.smithy.syntax; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.loader.IdlTokenizer; + +public class TokenTreeLeafTest { + @Test + public void cannotAppendChildren() { + IdlTokenizer tokenizer = IdlTokenizer.create("foo bar"); + CapturedToken token = CapturedToken.from(tokenizer); + TokenTreeLeaf leaf = new TokenTreeLeaf(token); + + Assertions.assertThrows(UnsupportedOperationException.class, () -> { + tokenizer.next(); + leaf.appendChild(TokenTree.of(CapturedToken.from(tokenizer))); + }); + } +} diff --git a/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TokenTreeNodeTest.java b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TokenTreeNodeTest.java new file mode 100644 index 00000000000..9002125dbda --- /dev/null +++ b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TokenTreeNodeTest.java @@ -0,0 +1,55 @@ +package software.amazon.smithy.syntax; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.loader.IdlTokenizer; + +public class TokenTreeNodeTest { + @Test + public void hasChildren() { + TokenTree tree = TokenTree.of(TreeType.BR); + TokenTree a = TokenTree.of(TreeType.WS); + TokenTree b = TokenTree.of(TreeType.TOKEN); + + tree.appendChild(a); + tree.appendChild(b); + + assertThat(tree.getChildren(), contains(a, b)); + + tree.removeChild(a); + + assertThat(tree.getChildren(), contains(b)); + } + + @Test + public void hasTokens() { + IdlTokenizer tokenizer = IdlTokenizer.create("foo"); + CapturedToken capture = CapturedToken.from(tokenizer); + TokenTree tree = TokenTree.of(capture); + + assertThat(tree.tokens().collect(Collectors.toList()), contains(capture)); + assertThat(tree.getStartPosition(), equalTo(0)); + assertThat(tree.getStartLine(), equalTo(1)); + assertThat(tree.getStartColumn(), equalTo(1)); + assertThat(tree.getEndColumn(), equalTo(4)); // the column is exclusive (it goes to 3, but 4 is the _next_ col) + assertThat(tree.getEndLine(), equalTo(1)); + } + + @Test + public void equalsAndHashCodeWork() { + IdlTokenizer tokenizer = IdlTokenizer.create("foo"); + CapturedToken capture = CapturedToken.from(tokenizer); + TokenTree tree = TokenTree.of(capture); + + assertThat(tree, equalTo(tree)); + assertThat(tree, not(equalTo("hi"))); + assertThat(tree, not(equalTo(TokenTree.fromError("Foo")))); + assertThat(tree.hashCode(), is(not(0))); + } +} diff --git a/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TokenTreeTest.java b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TokenTreeTest.java new file mode 100644 index 00000000000..569886180f5 --- /dev/null +++ b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TokenTreeTest.java @@ -0,0 +1,76 @@ +package software.amazon.smithy.syntax; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.loader.IdlTokenizer; + +public class TokenTreeTest { + @Test + public void createsFromType() { + TokenTree tree = TokenTree.of(TreeType.WS); + + assertThat(tree.getType(), is(TreeType.WS)); + assertThat(tree.getChildren(), empty()); + assertThat(tree.getError(), nullValue()); + } + + @Test + public void createsFromTokenizer() { + IdlTokenizer tokenizer = IdlTokenizer.create("foo"); + TokenTree tree = TokenTree.of(tokenizer); + + assertThat(tree.getType(), is(TreeType.IDL)); + assertThat(tree.getChildren(), hasSize(3)); + assertThat(tree.getError(), nullValue()); + assertThat(tree.getChildren().get(0).getType(), equalTo(TreeType.CONTROL_SECTION)); + assertThat(tree.getChildren().get(1).getType(), equalTo(TreeType.METADATA_SECTION)); + assertThat(tree.getChildren().get(2).getType(), equalTo(TreeType.SHAPE_SECTION)); + } + + @Test + public void createsFromCapturedToken() { + IdlTokenizer tokenizer = IdlTokenizer.create("foo"); + CapturedToken token = CapturedToken.from(tokenizer); + TokenTree tree = TokenTree.of(token); + + assertThat(tree.getType(), is(TreeType.TOKEN)); + assertThat(tree.getChildren(), hasSize(0)); + } + + @Test + public void createsFromErrorString() { + TokenTree tree = TokenTree.fromError("Foo"); + + assertThat(tree.getType(), is(TreeType.ERROR)); + assertThat(tree.getError(), equalTo("Foo")); + } + + @Test + public void createsZipperForNode() { + IdlTokenizer tokenizer = IdlTokenizer.create("foo"); + CapturedToken token = CapturedToken.from(tokenizer); + TokenTree tree = TokenTree.of(token); + TreeCursor cursor = tree.zipper(); + + assertThat(tree, equalTo(cursor.getTree())); + } + + @Test + public void replacesChild() { + TokenTree tree = TokenTree.of(TreeType.WS); + TokenTree child1 = TokenTree.of(TreeType.COMMENT); + TokenTree child2 = TokenTree.of(TreeType.COMMA); + + tree.appendChild(child1); + + assertThat(tree.replaceChild(child1, child2), is(true)); + assertThat(tree.replaceChild(child1, child2), is(false)); + assertThat(tree.zipper().getFirstChild(TreeType.COMMA).getTree(), is(child2)); + } +} diff --git a/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TreeCursorTest.java b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TreeCursorTest.java new file mode 100644 index 00000000000..42ad9a4ba7f --- /dev/null +++ b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TreeCursorTest.java @@ -0,0 +1,90 @@ +package software.amazon.smithy.syntax; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; + +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.loader.IdlTokenizer; +import software.amazon.smithy.utils.IoUtils; + +public class TreeCursorTest { + @Test + public void hasChildren() { + TokenTree tree = createTree(); + TreeCursor cursor = tree.zipper(); + List children = cursor.getChildren(); + + assertThat(cursor.getFirstChild(TreeType.CONTROL_SECTION), is(not(nullValue()))); + assertThat(cursor.getFirstChild(TreeType.METADATA_SECTION), is(not(nullValue()))); + assertThat(cursor.getFirstChild(TreeType.SHAPE_SECTION), is(not(nullValue()))); + + assertThat(children, hasSize(3)); + assertThat(children.get(0).getTree().getType(), is(TreeType.CONTROL_SECTION)); + assertThat(children.get(1).getTree().getType(), is(TreeType.METADATA_SECTION)); + assertThat(children.get(2).getTree().getType(), is(TreeType.SHAPE_SECTION)); + + assertThat(cursor.getFirstChild(), equalTo(children.get(0))); + assertThat(cursor.getLastChild(), equalTo(children.get(2))); + } + + @Test + public void hasParentAndSiblings() { + TokenTree tree = createTree(); + TreeCursor cursor = tree.zipper(); + List children = cursor.getChildren(); + + assertThat(cursor.getFirstChild(TreeType.CONTROL_SECTION).getParent(), equalTo(cursor)); + assertThat(cursor.getFirstChild(TreeType.METADATA_SECTION).getParent(), equalTo(cursor)); + assertThat(cursor.getFirstChild(TreeType.SHAPE_SECTION).getParent(), equalTo(cursor)); + + assertThat(cursor.getFirstChild(TreeType.CONTROL_SECTION).getNextSibling(), equalTo(children.get(1))); + assertThat(cursor.getFirstChild(TreeType.METADATA_SECTION).getNextSibling(), equalTo(children.get(2))); + assertThat(cursor.getFirstChild(TreeType.SHAPE_SECTION).getNextSibling(), nullValue()); + + assertThat(cursor.getFirstChild(TreeType.CONTROL_SECTION).getPreviousSibling(), nullValue()); + assertThat(cursor.getFirstChild(TreeType.METADATA_SECTION).getPreviousSibling(), equalTo(children.get(0))); + assertThat(cursor.getFirstChild(TreeType.SHAPE_SECTION).getPreviousSibling(), equalTo(children.get(1))); + } + + @Test + public void findsNodeAtPosition() { + TokenTree tree = createTree(); + TreeCursor cursor = tree.zipper(); + TreeCursor click = cursor.findAt(3, 17); + + assertThat(click, notNullValue()); + assertThat(click.getTree().getType(), is(TreeType.TOKEN)); + assertThat(click.getTree().tokens().iterator().next().getLexeme().toString(), equalTo("\"hello\"")); + assertThat(click.getRoot(), equalTo(cursor)); + } + + private TokenTree createTree() { + String model = IoUtils.readUtf8Url(getClass().getResource("formatter/simple-model.smithy")); + IdlTokenizer tokenizer = IdlTokenizer.create(model); + return TokenTree.of(tokenizer); + } + + @Test + public void getLastChildOfType() { + TokenTree tree = TokenTree.of(TreeType.BR); + TokenTree child1 = TokenTree.of(TreeType.WS); + TokenTree child2 = TokenTree.of(TreeType.COMMA); + TokenTree child3 = TokenTree.of(TreeType.COMMENT); + tree.appendChild(child1); + tree.appendChild(child2); + tree.appendChild(child3); + TreeCursor cursor = tree.zipper(); + + assertThat(cursor.getLastChild(TreeType.COMMENT).getTree(), sameInstance(child3)); + assertThat(cursor.getLastChild(TreeType.COMMA).getTree(), sameInstance(child2)); + assertThat(cursor.getLastChild(TreeType.WS).getTree(), sameInstance(child1)); + assertThat(cursor.getLastChild(TreeType.APPLY_STATEMENT), nullValue()); + } +} diff --git a/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TreeTypeTest.java b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TreeTypeTest.java new file mode 100644 index 00000000000..def758ddff9 --- /dev/null +++ b/smithy-syntax/src/test/java/software/amazon/smithy/syntax/TreeTypeTest.java @@ -0,0 +1,1868 @@ +package software.amazon.smithy.syntax; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Arrays; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.loader.IdlTokenizer; + +public class TreeTypeTest { + + @Test + public void idl() { + String idl = "$version: \"2.0\"\n\nnamespace com.foo\n"; + CapturingTokenizer tokenizer = new CapturingTokenizer(IdlTokenizer.create(idl)); + TreeType.IDL.parse(tokenizer); + assertTreeIsValid(tokenizer.getRoot()); + rootAndChildTypesEqual(tokenizer.getRoot(), + TreeType.IDL, + TreeType.CONTROL_SECTION, + TreeType.METADATA_SECTION, + TreeType.SHAPE_SECTION); + } + + @Test + public void controlSection() { + String controlSection = "$version: \"2.0\"\n$foo: [\"bar\"]\n"; + TokenTree tree = getTree(TreeType.CONTROL_SECTION, controlSection); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.CONTROL_SECTION, + TreeType.CONTROL_STATEMENT, + TreeType.CONTROL_STATEMENT); + } + + @Test + public void controlStatement() { + String controlStatement = "$version: \"2.0\"\n"; + TokenTree tree = getTree(TreeType.CONTROL_STATEMENT, controlStatement); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.CONTROL_STATEMENT, + TreeType.TOKEN, + TreeType.NODE_OBJECT_KEY, + TreeType.TOKEN, + TreeType.SP, + TreeType.NODE_VALUE, + TreeType.BR); + } + + @Test + public void identifierNodeObjectKey() { + String identifier = "version"; + TokenTree tree = getTree(TreeType.NODE_OBJECT_KEY, identifier); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.NODE_OBJECT_KEY, + TreeType.IDENTIFIER); + } + + @Test + public void stringNodeObjectKey() { + String string = "\"foo bar\""; + TokenTree tree = getTree(TreeType.NODE_OBJECT_KEY, string); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.NODE_OBJECT_KEY, + TreeType.QUOTED_TEXT); + } + + @Test + public void metadataSection() { + String metadataSection = "metadata foo = bar\nmetadata bar=baz\n"; + TokenTree tree = getTree(TreeType.METADATA_SECTION, metadataSection); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.METADATA_SECTION, + TreeType.METADATA_STATEMENT, + TreeType.METADATA_STATEMENT); + } + + @Test + public void metadataStatement() { + String statement = "metadata foo = bar // Foo\n\n\t"; + TokenTree tree = getTree(TreeType.METADATA_STATEMENT, statement); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.METADATA_STATEMENT, + TreeType.TOKEN, + TreeType.SP, + TreeType.NODE_OBJECT_KEY, + TreeType.SP, + TreeType.TOKEN, + TreeType.SP, + TreeType.NODE_VALUE, + TreeType.BR); + } + + @Test + public void shapeSection() { + String shapeSection = "namespace com.foo\nuse com.bar#Baz\nstructure Foo {}"; + TokenTree tree = getTree(TreeType.SHAPE_SECTION, shapeSection); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.SHAPE_SECTION, + TreeType.NAMESPACE_STATEMENT, + TreeType.USE_SECTION, + TreeType.SHAPE_STATEMENTS); + } + + @Test + public void namespaceStatement() { + String namespaceStatement = "namespace \t com.foo// Foo\n"; + TokenTree tree = getTree(TreeType.NAMESPACE_STATEMENT, namespaceStatement); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.NAMESPACE_STATEMENT, + TreeType.TOKEN, + TreeType.SP, + TreeType.NAMESPACE, + TreeType.BR); + } + + @Test + public void namespace() { + String namespace = "foo.bar.baz.qux"; + TokenTree tree = getTree(TreeType.NAMESPACE, namespace); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.NAMESPACE, + TreeType.IDENTIFIER, + TreeType.TOKEN, + TreeType.IDENTIFIER, + TreeType.TOKEN, + TreeType.IDENTIFIER, + TreeType.TOKEN, + TreeType.IDENTIFIER); + } + + @Test + public void identifier() { + String identifier1 = "foo"; + TokenTree tree1 = getTree(TreeType.IDENTIFIER, identifier1); + assertTreeIsValid(tree1); + rootAndChildTypesEqual(tree1, + TreeType.IDENTIFIER, + TreeType.TOKEN); + + String identifier2 = "_foo"; + TokenTree tree2 = getTree(TreeType.IDENTIFIER, identifier2); + assertTreeIsValid(tree2); + rootAndChildTypesEqual(tree2, + TreeType.IDENTIFIER, + TreeType.TOKEN); + + String identifier3 = "_1foo"; + TokenTree tree3 = getTree(TreeType.IDENTIFIER, identifier3); + assertTreeIsValid(tree3); + rootAndChildTypesEqual(tree3, + TreeType.IDENTIFIER, + TreeType.TOKEN); + + String identifier4 = "1foo"; + TokenTree tree4 = getTree(TreeType.IDENTIFIER, identifier4); + rootAndChildTypesEqual(tree4, + TreeType.IDENTIFIER, + TreeType.ERROR); + } + + @Test + public void useSection() { + String useSection = "use com.foo#Foo\nuse com.bar#Bar\n"; + TokenTree tree = getTree(TreeType.USE_SECTION, useSection); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.USE_SECTION, + TreeType.USE_STATEMENT, + TreeType.USE_STATEMENT); + } + + @Test + public void useStatement() { + String useStatement = "use \tcom.foo#Bar\t\n"; + TokenTree tree = getTree(TreeType.USE_STATEMENT, useStatement); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.USE_STATEMENT, + TreeType.TOKEN, + TreeType.SP, + TreeType.ABSOLUTE_ROOT_SHAPE_ID, + TreeType.BR); + } + + @Test + public void shapeId() { + String id = "foo"; + TokenTree idTree = getTree(TreeType.SHAPE_ID, id); + assertTreeIsValid(idTree); + rootAndChildTypesEqual(idTree, + TreeType.SHAPE_ID, + TreeType.ROOT_SHAPE_ID); + + String withMember = "foo$bar"; + TokenTree withMemberTree = getTree(TreeType.SHAPE_ID, withMember); + assertTreeIsValid(withMemberTree); + rootAndChildTypesEqual(withMemberTree, + TreeType.SHAPE_ID, + TreeType.ROOT_SHAPE_ID, + TreeType.SHAPE_ID_MEMBER); + } + + @Test + public void rootShapeId() { + String absolute = "com.foo.bar#Baz"; + TokenTree absoluteTree = getTree(TreeType.ROOT_SHAPE_ID, absolute); + assertTreeIsValid(absoluteTree); + rootAndChildTypesEqual(absoluteTree, + TreeType.ROOT_SHAPE_ID, + TreeType.ABSOLUTE_ROOT_SHAPE_ID); + + String id = "foo"; + TokenTree idTree = getTree(TreeType.ROOT_SHAPE_ID, id); + assertTreeIsValid(idTree); + rootAndChildTypesEqual(idTree, + TreeType.ROOT_SHAPE_ID, + TreeType.IDENTIFIER); + } + + @Test + public void absoluteRootShapeId() { + String absolute = "com.foo.bar#Baz"; + TokenTree tree = getTree(TreeType.ABSOLUTE_ROOT_SHAPE_ID, absolute); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.ABSOLUTE_ROOT_SHAPE_ID, + TreeType.NAMESPACE, + TreeType.TOKEN, + TreeType.IDENTIFIER); + } + + @Test + public void shapeIdMember() { + String shapeIdMember = "$foo"; + TokenTree tree = getTree(TreeType.SHAPE_ID_MEMBER, shapeIdMember); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.SHAPE_ID_MEMBER, + TreeType.TOKEN, + TreeType.IDENTIFIER); + } + + @Test + public void shapeStatements() { + String statements = "structure Foo {\n\tfoo: Bar\n}\n\napply foo @bar\n"; + TokenTree tree = getTree(TreeType.SHAPE_STATEMENTS, statements); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.SHAPE_STATEMENTS, + TreeType.SHAPE_OR_APPLY_STATEMENT, + TreeType.BR, + TreeType.SHAPE_OR_APPLY_STATEMENT, + TreeType.BR); + } + + @Test + public void shapeOrApplyStatement() { + String shape = "string Foo"; + TokenTree shapeTree = getTree(TreeType.SHAPE_OR_APPLY_STATEMENT, shape); + assertTreeIsValid(shapeTree); + rootAndChildTypesEqual(shapeTree, + TreeType.SHAPE_OR_APPLY_STATEMENT, + TreeType.SHAPE_STATEMENT); + + String apply = "apply foo @bar"; + TokenTree applyTree = getTree(TreeType.SHAPE_OR_APPLY_STATEMENT, apply); + assertTreeIsValid(applyTree); + rootAndChildTypesEqual(applyTree, + TreeType.SHAPE_OR_APPLY_STATEMENT, + TreeType.APPLY_STATEMENT); + } + + @Test + public void shapeStatement() { + String statement = "@foo\nstring Foo"; + TokenTree tree = getTree(TreeType.SHAPE_STATEMENT, statement); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.SHAPE_STATEMENT, + TreeType.TRAIT_STATEMENTS, + TreeType.SHAPE); + } + + @Test + public void shape() { + String shape = "structure Foo {}\n"; + TokenTree tree = getTree(TreeType.SHAPE, shape); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.SHAPE, + TreeType.AGGREGATE_SHAPE); + } + + @Test + public void simpleShape() { + String shape = "string \tFoo"; + TokenTree shapeTree = getTree(TreeType.SIMPLE_SHAPE, shape); + assertTreeIsValid(shapeTree); + rootAndChildTypesEqual(shapeTree, + TreeType.SIMPLE_SHAPE, + TreeType.SIMPLE_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER); + + String withMixins = "string Foo with [Bar]"; + TokenTree withMixinsTree = getTree(TreeType.SIMPLE_SHAPE, withMixins); + assertTreeIsValid(withMixinsTree); + rootAndChildTypesEqual(withMixinsTree, + TreeType.SIMPLE_SHAPE, + TreeType.SIMPLE_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.MIXINS); + } + + @Test + public void simpleTypeName() { + String simpleTypeName = "string"; + TokenTree tree = getTree(TreeType.SIMPLE_TYPE_NAME, simpleTypeName); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.SIMPLE_TYPE_NAME, + TreeType.TOKEN); + } + + @Test + public void enumShape() { + String enumShape = "enum Foo\n\t{foo = 1\n}"; + TokenTree tree = getTree(TreeType.ENUM_SHAPE, enumShape); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.ENUM_SHAPE, + TreeType.ENUM_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.WS, + TreeType.ENUM_SHAPE_MEMBERS); + + String withMixins = "enum Foo with [Bar] \n{foo}"; + TokenTree withMixinsTree = getTree(TreeType.ENUM_SHAPE, withMixins); + assertTreeIsValid(withMixinsTree); + rootAndChildTypesEqual(withMixinsTree, + TreeType.ENUM_SHAPE, + TreeType.ENUM_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.MIXINS, + TreeType.WS, + TreeType.ENUM_SHAPE_MEMBERS); + } + + @Test + public void enumTypeName() { + String enumTypeName = "intEnum"; + TokenTree tree = getTree(TreeType.ENUM_TYPE_NAME, enumTypeName); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.ENUM_TYPE_NAME, + TreeType.TOKEN); + } + + @Test + public void enumShapeMembers() { + String members = "{\nfoo bar baz\n}"; + TokenTree tree = getTree(TreeType.ENUM_SHAPE_MEMBERS, members); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.ENUM_SHAPE_MEMBERS, + TreeType.TOKEN, + TreeType.WS, + TreeType.ENUM_SHAPE_MEMBER, + TreeType.WS, + TreeType.ENUM_SHAPE_MEMBER, + TreeType.WS, + TreeType.ENUM_SHAPE_MEMBER, + TreeType.WS, + TreeType.TOKEN); + } + + @Test + public void enumShapeMember() { + String member = "FOO"; + TokenTree tree = getTree(TreeType.ENUM_SHAPE_MEMBER, member); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.ENUM_SHAPE_MEMBER, + TreeType.TRAIT_STATEMENTS, + TreeType.IDENTIFIER); + + String withValue = "FOO = BAR\n"; + TokenTree withValueTree = getTree(TreeType.ENUM_SHAPE_MEMBER, withValue); + assertTreeIsValid(withValueTree); + rootAndChildTypesEqual(withValueTree, + TreeType.ENUM_SHAPE_MEMBER, + TreeType.TRAIT_STATEMENTS, + TreeType.IDENTIFIER, + TreeType.VALUE_ASSIGNMENT); + + String noWs = "FOO=BAR\n"; + TokenTree noWsTree = getTree(TreeType.ENUM_SHAPE_MEMBER, noWs); + assertTreeIsValid(noWsTree); + rootAndChildTypesEqual(noWsTree, + TreeType.ENUM_SHAPE_MEMBER, + TreeType.TRAIT_STATEMENTS, + TreeType.IDENTIFIER, + TreeType.VALUE_ASSIGNMENT); + + String withTraits = "@foo\n@bar\nFOO"; + TokenTree withTraitsTree = getTree(TreeType.ENUM_SHAPE_MEMBER, withTraits); + assertTreeIsValid(withTraitsTree); + rootAndChildTypesEqual(withTraitsTree, + TreeType.ENUM_SHAPE_MEMBER, + TreeType.TRAIT_STATEMENTS, + TreeType.IDENTIFIER); + } + + @Test + public void aggregateShape() { + String shape = "structure Foo {}"; + TokenTree tree = getTree(TreeType.AGGREGATE_SHAPE, shape); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.AGGREGATE_SHAPE, + TreeType.AGGREGATE_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.SHAPE_MEMBERS); + } + + @Test + public void aggregateTypeName() { + String typeName = "structure"; + TokenTree tree = getTree(TreeType.AGGREGATE_TYPE_NAME, typeName); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.AGGREGATE_TYPE_NAME, + TreeType.TOKEN); + } + + @Test + public void aggregateShapeForResource() { + String shape = "structure Foo for Bar {}"; + TokenTree tree = getTree(TreeType.AGGREGATE_SHAPE, shape); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.AGGREGATE_SHAPE, + TreeType.AGGREGATE_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.FOR_RESOURCE, + TreeType.SP, + TreeType.SHAPE_MEMBERS); + } + + @Test + public void aggregateShapeMixins() { + String shape = "structure Foo with [Bar, Baz] {}"; + TokenTree tree = getTree(TreeType.AGGREGATE_SHAPE, shape); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.AGGREGATE_SHAPE, + TreeType.AGGREGATE_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.MIXINS, + TreeType.WS, + TreeType.SHAPE_MEMBERS); + } + + @Test + public void aggregateShapeForResourceAndMixins() { + String shape = "structure Foo for Bar with [Baz] {}"; + TokenTree tree = getTree(TreeType.AGGREGATE_SHAPE, shape); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.AGGREGATE_SHAPE, + TreeType.AGGREGATE_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.FOR_RESOURCE, + TreeType.SP, + TreeType.MIXINS, + TreeType.WS, + TreeType.SHAPE_MEMBERS); + } + + @Test + public void forResource() { + String forResource = "for Foo"; + TokenTree tree = getTree(TreeType.FOR_RESOURCE, forResource); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.FOR_RESOURCE, + TreeType.TOKEN, + TreeType.SP, + TreeType.SHAPE_ID); + } + + @Test + public void mixins() { + String mixins = "with [Foo, Bar]"; + TokenTree tree = getTree(TreeType.MIXINS, mixins); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.MIXINS, + TreeType.TOKEN, + TreeType.WS, + TreeType.TOKEN, + TreeType.SHAPE_ID, + TreeType.WS, + TreeType.SHAPE_ID, + TreeType.TOKEN); + } + + @Test + public void shapeMembers() { + String shapeMembers = "{\n\n\tfoo: Bar\n\tbar: Baz\n}"; + TokenTree tree = getTree(TreeType.SHAPE_MEMBERS, shapeMembers); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.SHAPE_MEMBERS, + TreeType.TOKEN, + TreeType.WS, + TreeType.SHAPE_MEMBER, + TreeType.WS, + TreeType.SHAPE_MEMBER, + TreeType.WS, + TreeType.TOKEN); + } + + @Test + public void shapeMember() { + String shapeMember = "@foo\n/// Foo\n\tfoo: Bar"; + TokenTree tree = getTree(TreeType.SHAPE_MEMBER, shapeMember); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.SHAPE_MEMBER, + TreeType.TRAIT_STATEMENTS, + TreeType.EXPLICIT_SHAPE_MEMBER); + } + + @Test + public void traitStatements() { + String traitStatements = "@foo\t// Foo\n\t/// Foo\n@bar\n"; + TokenTree tree = getTree(TreeType.TRAIT_STATEMENTS, traitStatements); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.TRAIT_STATEMENTS, + TreeType.TRAIT, + TreeType.WS, + TreeType.TRAIT, + TreeType.WS); + } + + @Test + public void trait() { + String trait = "@com.foo#Bar"; + TokenTree tree = getTree(TreeType.TRAIT, trait); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.TRAIT, + TreeType.TOKEN, + TreeType.SHAPE_ID); + } + + @Test + public void traitWithEmptyBody() { + String trait = "@abc()"; + TokenTree tree = getTree(TreeType.TRAIT, trait); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.TRAIT, + TreeType.TOKEN, + TreeType.SHAPE_ID, + TreeType.TRAIT_BODY); + } + + @Test + public void traitWithNonEmptyBody() { + String trait = "@abc(hi)"; + TokenTree tree = getTree(TreeType.TRAIT, trait); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.TRAIT, + TreeType.TOKEN, + TreeType.SHAPE_ID, + TreeType.TRAIT_BODY); + } + + @Test + public void traitBody() { + String traitBody = "(\nfoo: Bar\n)"; + TokenTree tree = getTree(TreeType.TRAIT_BODY, traitBody); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.TRAIT_BODY, + TreeType.TOKEN, + TreeType.WS, + TreeType.TRAIT_STRUCTURE, + TreeType.TOKEN); + } + + @Test + public void traitBodyTraitStructure() { + String traitBody = "foo: bar"; + TokenTree tree = getTree(TreeType.TRAIT_STRUCTURE, traitBody); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, TreeType.TRAIT_STRUCTURE, TreeType.NODE_OBJECT_KVP); + } + + @Test + public void traitBodyTraitNodeString() { + String traitBody = "(\"foo\")"; + TokenTree tree = getTree(TreeType.TRAIT_BODY, traitBody); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, TreeType.TRAIT_BODY, TreeType.TOKEN, TreeType.TRAIT_NODE, TreeType.TOKEN); + } + + @Test + public void traitBodyWithWs() { + String traitBody = "( \"foo\" )"; + TokenTree tree = getTree(TreeType.TRAIT_BODY, traitBody); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.TRAIT_BODY, + TreeType.TOKEN, + TreeType.WS, + TreeType.TRAIT_NODE, + TreeType.TOKEN); + } + + @Test + public void traitBodyTraitNodeStructure() { + String traitBody = "({ foo: bar })"; + TokenTree tree = getTree(TreeType.TRAIT_BODY, traitBody); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, TreeType.TRAIT_BODY, TreeType.TOKEN, TreeType.TRAIT_NODE, TreeType.TOKEN); + } + + @Test + public void traitNode() { + String traitBody = "\"foo\""; + TokenTree tree = getTree(TreeType.TRAIT_NODE, traitBody); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, TreeType.TRAIT_NODE, TreeType.NODE_VALUE); + } + + @Test + public void traitStructure() { + String traitStructure = "foo: bar, bar: baz\n\n// Baz\nbaz: qux\n"; + TokenTree tree = getTree(TreeType.TRAIT_STRUCTURE, traitStructure); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.TRAIT_STRUCTURE, + TreeType.NODE_OBJECT_KVP, + TreeType.WS, + TreeType.NODE_OBJECT_KVP, + TreeType.WS, + TreeType.NODE_OBJECT_KVP, + TreeType.WS); + } + + @Test + public void explicitShapeMember() { + String explicitShapeMember = "foo \t: \t com.foo#Bar"; + TokenTree tree = getTree(TreeType.EXPLICIT_SHAPE_MEMBER, explicitShapeMember); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.EXPLICIT_SHAPE_MEMBER, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.TOKEN, + TreeType.SP, + TreeType.SHAPE_ID); + } + + @Test + public void elidedShapeMember() { + String elidedShapeMember = "$foo"; + TokenTree tree = getTree(TreeType.ELIDED_SHAPE_MEMBER, elidedShapeMember); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.ELIDED_SHAPE_MEMBER, + TreeType.TOKEN, + TreeType.IDENTIFIER); + } + + @Test + public void valueAssignment() { + String valueAssignment = "\t = \t foo , \n"; + TokenTree tree = getTree(TreeType.VALUE_ASSIGNMENT, valueAssignment); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.VALUE_ASSIGNMENT, + TreeType.SP, + TreeType.TOKEN, + TreeType.SP, + TreeType.NODE_VALUE, + TreeType.SP, + TreeType.COMMA, + TreeType.BR); + } + + @Test + public void entityShape() { + String entityShape = "service Foo {}"; + TokenTree entityShapeTree = getTree(TreeType.ENTITY_SHAPE, entityShape); + assertTreeIsValid(entityShapeTree); + rootAndChildTypesEqual(entityShapeTree, + TreeType.ENTITY_SHAPE, + TreeType.ENTITY_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.NODE_OBJECT); + + String noWs = "service Foo{}"; + TokenTree noWsTree = getTree(TreeType.ENTITY_SHAPE, noWs); + assertTreeIsValid(noWsTree); + rootAndChildTypesEqual(noWsTree, + TreeType.ENTITY_SHAPE, + TreeType.ENTITY_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.NODE_OBJECT); + + String withMixins = "service Foo with [Bar] {}"; + TokenTree withMixinsTree = getTree(TreeType.ENTITY_SHAPE, withMixins); + assertTreeIsValid(withMixinsTree); + rootAndChildTypesEqual(withMixinsTree, + TreeType.ENTITY_SHAPE, + TreeType.ENTITY_TYPE_NAME, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.MIXINS, + TreeType.WS, + TreeType.NODE_OBJECT); + } + + @Test + public void entityTypeName() { + String typeName = "service"; + TokenTree tree = getTree(TreeType.ENTITY_TYPE_NAME, typeName); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.ENTITY_TYPE_NAME, + TreeType.TOKEN); + } + + @Test + public void operationShape() { + String shape = "operation Foo {input: Foo output: Bar}"; + TokenTree shapeTree = getTree(TreeType.OPERATION_SHAPE, shape); + assertTreeIsValid(shapeTree); + rootAndChildTypesEqual(shapeTree, + TreeType.OPERATION_SHAPE, + TreeType.TOKEN, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.OPERATION_BODY); + + String noWs = "operation Foo{input: Foo output: Bar}"; + TokenTree noWsTree = getTree(TreeType.OPERATION_SHAPE, noWs); + assertTreeIsValid(noWsTree); + rootAndChildTypesEqual(noWsTree, + TreeType.OPERATION_SHAPE, + TreeType.TOKEN, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.OPERATION_BODY); + + String withMixins = "operation Foo with [Bar] {input: Foo output: Bar}"; + TokenTree withMixinsTree = getTree(TreeType.OPERATION_SHAPE, withMixins); + assertTreeIsValid(withMixinsTree); + rootAndChildTypesEqual(withMixinsTree, + TreeType.OPERATION_SHAPE, + TreeType.TOKEN, + TreeType.SP, + TreeType.IDENTIFIER, + TreeType.SP, + TreeType.MIXINS, + TreeType.WS, + TreeType.OPERATION_BODY); + } + + @Test + public void operationBody() { + String operationBody = "{ input: foo\noutput: bar }"; + TokenTree operationBodyTree = getTree(TreeType.OPERATION_BODY, operationBody); + assertTreeIsValid(operationBodyTree); + rootAndChildTypesEqual(operationBodyTree, + TreeType.OPERATION_BODY, + TreeType.TOKEN, + TreeType.WS, + TreeType.OPERATION_PROPERTY, + TreeType.WS, + TreeType.OPERATION_PROPERTY, + TreeType.WS, + TreeType.TOKEN); + + String noWs = "{input:foo}"; + TokenTree noWsTree = getTree(TreeType.OPERATION_BODY, noWs); + assertTreeIsValid(noWsTree); + rootAndChildTypesEqual(noWsTree, + TreeType.OPERATION_BODY, + TreeType.TOKEN, + TreeType.OPERATION_PROPERTY, + TreeType.TOKEN); + + String onlyWs = "{\n// Foo\n \n}"; + TokenTree onlyWsTree = getTree(TreeType.OPERATION_BODY, onlyWs); + assertTreeIsValid(onlyWsTree); + rootAndChildTypesEqual(onlyWsTree, + TreeType.OPERATION_BODY, + TreeType.TOKEN, + TreeType.WS, + TreeType.TOKEN); + } + + @Test + public void operationProperty() { + String input = "input: foo"; + TokenTree inputTree = getTree(TreeType.OPERATION_PROPERTY, input); + assertTreeIsValid(inputTree); + rootAndChildTypesEqual(inputTree, + TreeType.OPERATION_PROPERTY, + TreeType.OPERATION_INPUT); + + String output = "output: foo"; + TokenTree outputTree = getTree(TreeType.OPERATION_PROPERTY, output); + assertTreeIsValid(outputTree); + rootAndChildTypesEqual(outputTree, + TreeType.OPERATION_PROPERTY, + TreeType.OPERATION_OUTPUT); + + String errors = "errors: []"; + TokenTree errorsTree = getTree(TreeType.OPERATION_PROPERTY, errors); + assertTreeIsValid(errorsTree); + rootAndChildTypesEqual(errorsTree, + TreeType.OPERATION_PROPERTY, + TreeType.OPERATION_ERRORS); + } + + @Test + public void operationInput() { + String withWs = "input\n//Foo\n : foo"; + TokenTree withWsTree = getTree(TreeType.OPERATION_INPUT, withWs); + assertTreeIsValid(withWsTree); + rootAndChildTypesEqual(withWsTree, + TreeType.OPERATION_INPUT, + TreeType.TOKEN, + TreeType.WS, + TreeType.TOKEN, + TreeType.WS, + TreeType.SHAPE_ID); + + String noWs = "input:foo"; + TokenTree noWsTree = getTree(TreeType.OPERATION_INPUT, noWs); + assertTreeIsValid(noWsTree); + rootAndChildTypesEqual(noWsTree, + TreeType.OPERATION_INPUT, + TreeType.TOKEN, + TreeType.TOKEN, + TreeType.SHAPE_ID); + + String shapeId = "input: foo.bar#Baz"; + TokenTree shapeIdTree = getTree(TreeType.OPERATION_INPUT, shapeId); + assertTreeIsValid(shapeIdTree); + rootAndChildTypesEqual(shapeIdTree, + TreeType.OPERATION_INPUT, + TreeType.TOKEN, + TreeType.TOKEN, + TreeType.WS, + TreeType.SHAPE_ID); + + String inline = "input := {}"; + TokenTree inlineTree = getTree(TreeType.OPERATION_INPUT, inline); + assertTreeIsValid(inlineTree); + rootAndChildTypesEqual(inlineTree, + TreeType.OPERATION_INPUT, + TreeType.TOKEN, + TreeType.WS, + TreeType.INLINE_AGGREGATE_SHAPE); + } + + @Test + public void operationOutput() { + String withWs = "output\n//Foo\n : foo"; + TokenTree withWsTree = getTree(TreeType.OPERATION_OUTPUT, withWs); + assertTreeIsValid(withWsTree); + rootAndChildTypesEqual(withWsTree, + TreeType.OPERATION_OUTPUT, + TreeType.TOKEN, + TreeType.WS, + TreeType.TOKEN, + TreeType.WS, + TreeType.SHAPE_ID); + + String noWs = "output:foo"; + TokenTree noWsTree = getTree(TreeType.OPERATION_OUTPUT, noWs); + assertTreeIsValid(noWsTree); + rootAndChildTypesEqual(noWsTree, + TreeType.OPERATION_OUTPUT, + TreeType.TOKEN, + TreeType.TOKEN, + TreeType.SHAPE_ID); + + String shapeId = "output: foo.bar#Baz"; + TokenTree shapeIdTree = getTree(TreeType.OPERATION_OUTPUT, shapeId); + assertTreeIsValid(shapeIdTree); + rootAndChildTypesEqual(shapeIdTree, + TreeType.OPERATION_OUTPUT, + TreeType.TOKEN, + TreeType.TOKEN, + TreeType.WS, + TreeType.SHAPE_ID); + + String inline = "output := {}"; + TokenTree inlineTree = getTree(TreeType.OPERATION_OUTPUT, inline); + assertTreeIsValid(inlineTree); + rootAndChildTypesEqual(inlineTree, + TreeType.OPERATION_OUTPUT, + TreeType.TOKEN, + TreeType.WS, + TreeType.INLINE_AGGREGATE_SHAPE); + } + + @Test + public void inlineAggregateShape() { + String basic = ":= { foo: bar }"; + TokenTree basicTree = getTree(TreeType.INLINE_AGGREGATE_SHAPE, basic); + assertTreeIsValid(basicTree); + rootAndChildTypesEqual(basicTree, + TreeType.INLINE_AGGREGATE_SHAPE, + TreeType.TOKEN, + TreeType.WS, + TreeType.TRAIT_STATEMENTS, + TreeType.SHAPE_MEMBERS); + + String withTraits = ":= @foo\n\n@bar\n\n{ foo: bar }"; + TokenTree withTraitsTree = getTree(TreeType.INLINE_AGGREGATE_SHAPE, withTraits); + assertTreeIsValid(withTraitsTree); + rootAndChildTypesEqual(withTraitsTree, + TreeType.INLINE_AGGREGATE_SHAPE, + TreeType.TOKEN, + TreeType.WS, + TreeType.TRAIT_STATEMENTS, + TreeType.SHAPE_MEMBERS); + } + + @Test + public void operationErrors() { + String empty = "errors: []"; + TokenTree emptyTree = getTree(TreeType.OPERATION_ERRORS, empty); + assertTreeIsValid(emptyTree); + rootAndChildTypesEqual(emptyTree, + TreeType.OPERATION_ERRORS, + TreeType.TOKEN, + TreeType.TOKEN, + TreeType.WS, + TreeType.TOKEN, + TreeType.TOKEN); + + String noWs = "errors:[foo]"; + TokenTree noWsTree = getTree(TreeType.OPERATION_ERRORS, noWs); + assertTreeIsValid(noWsTree); + rootAndChildTypesEqual(noWsTree, + TreeType.OPERATION_ERRORS, + TreeType.TOKEN, + TreeType.TOKEN, + TreeType.TOKEN, + TreeType.SHAPE_ID, + TreeType.TOKEN); + + String withWs = "errors\n//Foo\n: \n[\nfoo // Foo\nbar ]"; + TokenTree withWsTree = getTree(TreeType.OPERATION_ERRORS, withWs); + assertTreeIsValid(withWsTree); + rootAndChildTypesEqual(withWsTree, + TreeType.OPERATION_ERRORS, + TreeType.TOKEN, + TreeType.WS, + TreeType.TOKEN, + TreeType.WS, + TreeType.TOKEN, + TreeType.WS, + TreeType.SHAPE_ID, + TreeType.WS, + TreeType.SHAPE_ID, + TreeType.WS, + TreeType.TOKEN); + } + + @Test + public void applyStatement() { + String applyStatement = "apply foo @bar"; + TokenTree tree = getTree(TreeType.APPLY_STATEMENT, applyStatement); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.APPLY_STATEMENT, + TreeType.APPLY_STATEMENT_SINGULAR); + } + + @Test + public void applyStatementSingular() { + String singular = "apply foo\n\t@bar"; + TokenTree tree = getTree(TreeType.APPLY_STATEMENT_SINGULAR, singular); + assertTreeIsValid(tree); + rootAndChildTypesEqual(tree, + TreeType.APPLY_STATEMENT_SINGULAR, + TreeType.TOKEN, + TreeType.SP, + TreeType.SHAPE_ID, + TreeType.WS, + TreeType.TRAIT); + } + + @Test + public void applyStatementBlock() { + String block = "apply foo\n\t{@foo\n@bar\n}"; + TokenTree blockTree = getTree(TreeType.APPLY_STATEMENT_BLOCK, block); + assertTreeIsValid(blockTree); + rootAndChildTypesEqual(blockTree, + TreeType.APPLY_STATEMENT_BLOCK, + TreeType.TOKEN, + TreeType.SP, + TreeType.SHAPE_ID, + TreeType.WS, + TreeType.TOKEN, + TreeType.TRAIT_STATEMENTS, + TreeType.TOKEN); + + String withLeadingWsInBlock = "apply foo\n{// Bar\n@foo\n}"; + TokenTree withLeadingWsInBlockTree = getTree(TreeType.APPLY_STATEMENT_BLOCK, withLeadingWsInBlock); + assertTreeIsValid(withLeadingWsInBlockTree); + rootAndChildTypesEqual(withLeadingWsInBlockTree, + TreeType.APPLY_STATEMENT_BLOCK, + TreeType.TOKEN, + TreeType.SP, + TreeType.SHAPE_ID, + TreeType.WS, + TreeType.TOKEN, + TreeType.WS, + TreeType.TRAIT_STATEMENTS, + TreeType.TOKEN); + + String minimalWs = "apply foo {@bar}"; + TokenTree minimalWsTree = getTree(TreeType.APPLY_STATEMENT_BLOCK, minimalWs); + assertTreeIsValid(minimalWsTree); + rootAndChildTypesEqual(minimalWsTree, + TreeType.APPLY_STATEMENT_BLOCK, + TreeType.TOKEN, + TreeType.SP, + TreeType.SHAPE_ID, + TreeType.WS, + TreeType.TOKEN, + TreeType.TRAIT_STATEMENTS, + TreeType.TOKEN); + } + + @Test + public void nodeValue() { + String array = "[]"; + TokenTree arrayTree = getTree(TreeType.NODE_VALUE, array); + assertTreeIsValid(arrayTree); + rootAndChildTypesEqual(arrayTree, + TreeType.NODE_VALUE, + TreeType.NODE_ARRAY); + + String object = "{}"; + TokenTree objectTree = getTree(TreeType.NODE_VALUE, object); + assertTreeIsValid(objectTree); + rootAndChildTypesEqual(objectTree, + TreeType.NODE_VALUE, + TreeType.NODE_OBJECT); + + String number = "1"; + TokenTree numberTree = getTree(TreeType.NODE_VALUE, number); + assertTreeIsValid(numberTree); + rootAndChildTypesEqual(numberTree, + TreeType.NODE_VALUE, + TreeType.NUMBER); + + String trueKeyword = "true"; + TokenTree trueTree = getTree(TreeType.NODE_VALUE, trueKeyword); + assertTreeIsValid(trueTree); + rootAndChildTypesEqual(trueTree, + TreeType.NODE_VALUE, + TreeType.NODE_KEYWORD); + + String falseKeyword = "false"; + TokenTree falseTree = getTree(TreeType.NODE_VALUE, falseKeyword); + assertTreeIsValid(falseTree); + rootAndChildTypesEqual(falseTree, + TreeType.NODE_VALUE, + TreeType.NODE_KEYWORD); + + String nullKeyword = "null"; + TokenTree nullTree = getTree(TreeType.NODE_VALUE, nullKeyword); + assertTreeIsValid(nullTree); + rootAndChildTypesEqual(nullTree, + TreeType.NODE_VALUE, + TreeType.NODE_KEYWORD); + + String shapeId = "foo"; + TokenTree shapeIdTree = getTree(TreeType.NODE_VALUE, shapeId); + assertTreeIsValid(shapeIdTree); + rootAndChildTypesEqual(shapeIdTree, + TreeType.NODE_VALUE, + TreeType.NODE_STRING_VALUE); + + String quotedText = "\"foo\""; + TokenTree quotedTextTree = getTree(TreeType.NODE_VALUE, quotedText); + assertTreeIsValid(quotedTextTree); + rootAndChildTypesEqual(quotedTextTree, + TreeType.NODE_VALUE, + TreeType.NODE_STRING_VALUE); + + String textBlock = "\"\"\"\nfoo\"\"\""; + TokenTree textBlockTree = getTree(TreeType.NODE_VALUE, textBlock); + assertTreeIsValid(textBlockTree); + rootAndChildTypesEqual(textBlockTree, + TreeType.NODE_VALUE, + TreeType.NODE_STRING_VALUE); + } + + @Test + public void nodeArray() { + String empty = "[]"; + TokenTree emptyTree = getTree(TreeType.NODE_ARRAY, empty); + assertTreeIsValid(emptyTree); + rootAndChildTypesEqual(emptyTree, + TreeType.NODE_ARRAY, + TreeType.TOKEN, + TreeType.TOKEN); + + String emptyWs = "[\n\n\t//Foo\n]"; + TokenTree emptyWsTree = getTree(TreeType.NODE_ARRAY, emptyWs); + assertTreeIsValid(emptyWsTree); + rootAndChildTypesEqual(emptyWsTree, + TreeType.NODE_ARRAY, + TreeType.TOKEN, + TreeType.WS, + TreeType.TOKEN); + + String withElements = "[ foo\n bar, baz // foo\n\tqux ]"; + TokenTree withElementsTree = getTree(TreeType.NODE_ARRAY, withElements); + assertTreeIsValid(withElementsTree); + rootAndChildTypesEqual(withElementsTree, + TreeType.NODE_ARRAY, + TreeType.TOKEN, + TreeType.WS, + TreeType.NODE_VALUE, + TreeType.WS, + TreeType.NODE_VALUE, + TreeType.WS, + TreeType.NODE_VALUE, + TreeType.WS, + TreeType.NODE_VALUE, + TreeType.WS, + TreeType.TOKEN); + + String noWs = "[foo]"; + TokenTree noWsTree = getTree(TreeType.NODE_ARRAY, noWs); + assertTreeIsValid(noWsTree); + rootAndChildTypesEqual(noWsTree, + TreeType.NODE_ARRAY, + TreeType.TOKEN, + TreeType.NODE_VALUE, + TreeType.TOKEN); + } + + @Test + public void nodeObject() { + String empty = "{}"; + TokenTree emptyTree = getTree(TreeType.NODE_OBJECT, empty); + assertTreeIsValid(emptyTree); + rootAndChildTypesEqual(emptyTree, + TreeType.NODE_OBJECT, + TreeType.TOKEN, + TreeType.TOKEN); + + String emptyWs = "{// Foo\n\n\t}"; + TokenTree emptyWsTree = getTree(TreeType.NODE_OBJECT, emptyWs); + assertTreeIsValid(emptyWsTree); + rootAndChildTypesEqual(emptyWsTree, + TreeType.NODE_OBJECT, + TreeType.TOKEN, + TreeType.WS, + TreeType.TOKEN); + + String withElements = "{ foo: bar\n// Foo\n\tbaz: qux\n}"; + TokenTree withElementsTree = getTree(TreeType.NODE_OBJECT, withElements); + assertTreeIsValid(withElementsTree); + rootAndChildTypesEqual(withElementsTree, + TreeType.NODE_OBJECT, + TreeType.TOKEN, + TreeType.WS, + TreeType.NODE_OBJECT_KVP, + TreeType.WS, + TreeType.NODE_OBJECT_KVP, + TreeType.WS, + TreeType.TOKEN); + + String noWs = "{foo:bar}"; + TokenTree noWsTree = getTree(TreeType.NODE_OBJECT, noWs); + assertTreeIsValid(noWsTree); + rootAndChildTypesEqual(noWsTree, + TreeType.NODE_OBJECT, + TreeType.TOKEN, + TreeType.NODE_OBJECT_KVP, + TreeType.TOKEN); + } + + @Test + public void nodeObjectKvp() { + String kvp = "foo\n:\nbar"; + TokenTree kvpTree = getTree(TreeType.NODE_OBJECT_KVP, kvp); + assertTreeIsValid(kvpTree); + rootAndChildTypesEqual(kvpTree, + TreeType.NODE_OBJECT_KVP, + TreeType.NODE_OBJECT_KEY, + TreeType.WS, + TreeType.TOKEN, + TreeType.WS, + TreeType.NODE_VALUE); + + String noWs = "foo:bar"; + TokenTree noWsTree = getTree(TreeType.NODE_OBJECT_KVP, noWs); + assertTreeIsValid(noWsTree); + rootAndChildTypesEqual(noWsTree, + TreeType.NODE_OBJECT_KVP, + TreeType.NODE_OBJECT_KEY, + TreeType.TOKEN, + TreeType.NODE_VALUE); + } + + @Test + public void nodeObjectKey() { + String quoted = "\"foo bar\""; + TokenTree quotedTree = getTree(TreeType.NODE_OBJECT_KEY, quoted); + assertTreeIsValid(quotedTree); + rootAndChildTypesEqual(quotedTree, TreeType.NODE_OBJECT_KEY, TreeType.QUOTED_TEXT); + + String identifier = "foo"; + TokenTree idTree = getTree(TreeType.NODE_OBJECT_KEY, identifier); + assertTreeIsValid(idTree); + rootAndChildTypesEqual(idTree, TreeType.NODE_OBJECT_KEY, TreeType.IDENTIFIER); + } + + @Test + public void nodeStringValue() { + String identifier = "foo"; + TokenTree idTree = getTree(TreeType.NODE_STRING_VALUE, identifier); + assertTreeIsValid(idTree); + rootAndChildTypesEqual(idTree, TreeType.NODE_STRING_VALUE, TreeType.SHAPE_ID); + + /* + TODO: Right now grammar is ambiguous, no way to tell if its an identifier or shape id. + String shapeId = "com.foo#Bar$baz"; + TokenTree shapeIdTree = getTree(TreeType.NODE_STRING_VALUE, shapeId); + rootAndChildTypesEqual(shapeIdTree, TreeType.NODE_STRING_VALUE, TreeType.SHAPE_ID); + */ + + String quoted = "\"foo bar\""; + TokenTree quotedTree = getTree(TreeType.NODE_STRING_VALUE, quoted); + assertTreeIsValid(quotedTree); + rootAndChildTypesEqual(quotedTree, TreeType.NODE_STRING_VALUE, TreeType.QUOTED_TEXT); + + String block = "\"\"\"\nfoo\"\"\""; + TokenTree blockTree = getTree(TreeType.NODE_STRING_VALUE, block); + assertTreeIsValid(blockTree); + rootAndChildTypesEqual(blockTree, TreeType.NODE_STRING_VALUE, TreeType.TEXT_BLOCK); + } + + @Test + public void textBlock() { + String empty = "\"\"\"\n\"\"\""; + TokenTree emptyTree = getTree(TreeType.TEXT_BLOCK, empty); + assertTreeIsValid(emptyTree); + rootAndChildTypesEqual(emptyTree, TreeType.TEXT_BLOCK, TreeType.TOKEN); + + String withQuotes = "\"\"\"\n\"\"foo\"\n\"\"bar\"\"\""; + TokenTree withQuotesTree = getTree(TreeType.TEXT_BLOCK, withQuotes); + assertTreeIsValid(withQuotesTree); + rootAndChildTypesEqual(withQuotesTree, TreeType.TEXT_BLOCK, TreeType.TOKEN); + } + + @Test + public void invalidControlSection() { + String invalidTrailing = "$foo: bar\n$"; + TokenTree invalidTrailingTree = getTree(TreeType.CONTROL_SECTION, invalidTrailing); + assertTreeIsInvalid(invalidTrailingTree); + + String invalidLeading = "$foo: \nbar\n$foo: bar\n"; + TokenTree invalidLeadingTree = getTree(TreeType.CONTROL_SECTION, invalidLeading); + assertTreeIsInvalid(invalidLeadingTree); + + String multipleInvalid = "$foo: bar\n$1\n#foo: bar\n$foo = bar\n$foo: bar\n"; + TokenTree multipleInvalidTree = getTree(TreeType.CONTROL_SECTION, multipleInvalid); + assertTreeIsInvalid(multipleInvalidTree); + } + + @Test + public void invalidControlStatement() { + String missingDollar = "version: 2\n"; + TokenTree missingDollarTree = getTree(TreeType.CONTROL_STATEMENT, missingDollar); + assertTreeIsInvalid(missingDollarTree); + + String missingColon = "$version 2\n"; + TokenTree missingColonTree = getTree(TreeType.CONTROL_STATEMENT, missingColon); + assertTreeIsInvalid(missingColonTree); + + String notDollar = "=version: 2\n"; + TokenTree notDollarTree = getTree(TreeType.CONTROL_STATEMENT, notDollar); + assertTreeIsInvalid(notDollarTree); + + String notColon = "$version = 2\n"; + TokenTree notColonTree = getTree(TreeType.CONTROL_STATEMENT, notColon); + assertTreeIsInvalid(notColonTree); + } + + @Test + public void invalidMetadataSection() { + String invalidTrailing = "metadata foo = bar\nmetadata = bar\n"; + TokenTree invalidTrailingTree = getTree(TreeType.METADATA_SECTION, invalidTrailing); + assertTreeIsInvalid(invalidTrailingTree); + + String invalidLeading = "metadata foo =\nbar\nmetadata = bar\n"; + TokenTree invalidLeadingTree = getTree(TreeType.METADATA_SECTION, invalidLeading); + assertTreeIsInvalid(invalidLeadingTree); + + String multipleInvalid = "metadata foo = bar\nmetadata = \nfoo\nmetadata bar: baz\nmetadata baz = qux\n"; + TokenTree multipleInvalidTree = getTree(TreeType.METADATA_SECTION, multipleInvalid); + assertTreeIsInvalid(multipleInvalidTree); + } + + @Test + public void invalidMetadataStatement() { + String missingKeyword = "foo = bar\n"; + TokenTree missingKeywordTree = getTree(TreeType.METADATA_STATEMENT, missingKeyword); + assertTreeIsInvalid(missingKeywordTree); + + String missingEquals = "metadata foo bar\n"; + TokenTree missingEqualsTree = getTree(TreeType.METADATA_STATEMENT, missingEquals); + assertTreeIsInvalid(missingEqualsTree); + + String notEquals = "metadata foo: bar\n"; + TokenTree notEqualsTree = getTree(TreeType.METADATA_STATEMENT, notEquals); + assertTreeIsInvalid(notEqualsTree); + } + + @Test + public void invalidNodeValue() { + String notNodeValue = "$"; + TokenTree tree = getTree(TreeType.NODE_VALUE, notNodeValue); + assertTreeIsInvalid(tree); + } + + @Test + public void invalidNodeArray() { + String missingOpenBracket = "foo, bar]"; + TokenTree missingOpenBracketTree = getTree(TreeType.NODE_ARRAY, missingOpenBracket); + assertTreeIsInvalid(missingOpenBracketTree); + + String missingCloseBracket = "[foo, bar"; + TokenTree missingCloseBracketTree = getTree(TreeType.NODE_ARRAY, missingCloseBracket); + assertTreeIsInvalid(missingCloseBracketTree); + + String missingBrackets = "foo, bar"; + TokenTree missingBracketsTree = getTree(TreeType.NODE_OBJECT, missingBrackets); + assertTreeIsInvalid(missingBracketsTree); + + String notBrackets = "{foo, bar}"; + TokenTree notBracketsTree = getTree(TreeType.NODE_OBJECT, notBrackets); + assertTreeIsInvalid(notBracketsTree); + } + + @Test + public void invalidNodeObject() { + String missingOpenBrace = "foo: bar}"; + TokenTree missingOpenBraceTree = getTree(TreeType.NODE_OBJECT, missingOpenBrace); + assertTreeIsInvalid(missingOpenBraceTree); + + String missingCloseBrace = "{foo: bar"; + TokenTree missingCloseBraceTree = getTree(TreeType.NODE_OBJECT, missingCloseBrace); + assertTreeIsInvalid(missingCloseBraceTree); + + String missingBraces = "foo: bar"; + TokenTree missingBracesTree = getTree(TreeType.NODE_OBJECT, missingBraces); + assertTreeIsInvalid(missingBracesTree); + + String notBraces = "[foo: bar]"; + TokenTree notBracesTree = getTree(TreeType.NODE_ARRAY, notBraces); + assertTreeIsInvalid(notBracesTree); + } + + @Test + public void invalidNodeObjectKvp() { + String notColon = "foo = bar"; + TokenTree notColonTree = getTree(TreeType.NODE_OBJECT_KVP, notColon); + assertTreeIsInvalid(notColonTree); + + String missingColon = "foo bar"; + TokenTree missingColonTree = getTree(TreeType.NODE_OBJECT_KVP, missingColon); + assertTreeIsInvalid(missingColonTree); + } + + @Test + public void invalidNodeObjectKey() { + String invalid = "1"; + TokenTree tree = getTree(TreeType.NODE_OBJECT_KEY, invalid); + assertTreeIsInvalid(tree); + } + + @Test + public void invalidNamespaceStatement() { + String missingKeyword = " foo.bar\n"; + TokenTree missingKeywordTree = getTree(TreeType.NAMESPACE_STATEMENT, missingKeyword); + assertTreeIsInvalid(missingKeywordTree); + } + + @Test + public void invalidUseSection() { + String invalidTrailing = "use com.foo#Bar\nuse \ncom.foo#Bar\n"; + TokenTree invalidTrailingTree = getTree(TreeType.USE_SECTION, invalidTrailing); + assertTreeIsInvalid(invalidTrailingTree); + + String invalidLeading = "use\n com.foo#Bar\nuse com.foo#Bar\n"; + TokenTree invalidLeadingTree = getTree(TreeType.USE_SECTION, invalidLeading); + assertTreeIsInvalid(invalidLeadingTree); + + String multipleInvalid = "use com.foo#Bar\nuse\ncom.foo#Bar\nuse #Bar\nuse com.foo#Bar\n"; + TokenTree multipleInvalidTree = getTree(TreeType.USE_SECTION, multipleInvalid); + assertTreeIsInvalid(multipleInvalidTree); + } + + @Test + public void invalidUseStatement() { + String missingKeyword = " foo.bar#Baz\n"; + TokenTree missingKeywordTree = getTree(TreeType.USE_STATEMENT, missingKeyword); + assertTreeIsInvalid(missingKeywordTree); + } + + @Test + public void invalidShapeStatements() { + String incompleteShape = "string Foo\nstructure Bar {\nfoo: Foo\n"; + TokenTree incompleteShapeTree = getTree(TreeType.SHAPE_STATEMENTS, incompleteShape); + assertTreeIsInvalid(incompleteShapeTree); + + String firstInvalid = "string Foo {}\nstructure Bar{}\n"; + TokenTree firstInvalidTree = getTree(TreeType.SHAPE_STATEMENTS, firstInvalid); + assertTreeIsInvalid(firstInvalidTree); + + String trailingTraits = "structure Foo {}\n@bar\n@baz(foo: bar)\n"; + TokenTree trailingTraitsTree = getTree(TreeType.SHAPE_STATEMENTS, trailingTraits); + assertTreeIsInvalid(trailingTraitsTree); + } + + @Test + public void invalidShapeOrApplyStatement() { + // SHAPE_OR_APPLY_STATEMENT checks whether it's an apply or shape statement, + // and shape types are checked before that type is parsed, so this test + // covers all invalid shape type name/apply cases. + + String missingTypeName = " Foo with [Bar] {}"; + TokenTree missingTypeNameTree = getTree(TreeType.SHAPE_STATEMENT, missingTypeName); + assertTreeIsInvalid(missingTypeNameTree); + + String wrongTypeName = "unknown Foo with [Bar] {}"; + TokenTree wrongTypeNameTree = getTree(TreeType.SHAPE_STATEMENT, wrongTypeName); + assertTreeIsInvalid(wrongTypeNameTree); + } + @Test + public void invalidMixins() { + String missingOpenBracket = "with foo, bar]"; + TokenTree missingOpenBracketTree = getTree(TreeType.MIXINS, missingOpenBracket); + assertTreeIsInvalid(missingOpenBracketTree); + + String missingCloseBracket = "with [foo, bar"; + TokenTree missingCloseBracketTree = getTree(TreeType.MIXINS, missingCloseBracket); + assertTreeIsInvalid(missingCloseBracketTree); + + String missingBrackets = "with foo, bar"; + TokenTree missingBracketsTree = getTree(TreeType.MIXINS, missingBrackets); + assertTreeIsInvalid(missingBracketsTree); + + String notBrackets = "with {foo, bar}"; + TokenTree notBracketsTree = getTree(TreeType.MIXINS, notBrackets); + assertTreeIsInvalid(notBracketsTree); + } + + @Test + public void invalidEnumShapeMembers() { + String missingOpenBrace = "FOO }"; + TokenTree missingOpenBraceTree = getTree(TreeType.ENUM_SHAPE_MEMBERS, missingOpenBrace); + assertTreeIsInvalid(missingOpenBraceTree); + + String missingCloseBrace = "{FOO"; + TokenTree missingCloseBraceTree = getTree(TreeType.ENUM_SHAPE_MEMBERS, missingCloseBrace); + assertTreeIsInvalid(missingCloseBraceTree); + + String missingBraces = "FOO"; + TokenTree missingBracesTree = getTree(TreeType.ENUM_SHAPE_MEMBERS, missingBraces); + assertTreeIsInvalid(missingBracesTree); + + String notBraces = "[FOO]"; + TokenTree notBracesTree = getTree(TreeType.ENUM_SHAPE_MEMBERS, notBraces); + assertTreeIsInvalid(notBracesTree); + + String leadingInvalidMember = "{\n1\nFOO\n}"; + TokenTree leadingInvalidMemberTree = getTree(TreeType.ENUM_SHAPE_MEMBERS, leadingInvalidMember); + assertTreeIsInvalid(leadingInvalidMemberTree); + + String trailingInvalidMember = "{\nFOO\n1\n}"; + TokenTree trailingInvalidMemberTree = getTree(TreeType.ENUM_SHAPE_MEMBERS, trailingInvalidMember); + assertTreeIsInvalid(trailingInvalidMemberTree); + + String multipleInvalidMembers = "{\nFOO\nBAR = \n @foo(\nBAR\n = 2\nFOO\n}"; + TokenTree multipleInvalidMembersTree = getTree(TreeType.ENUM_SHAPE_MEMBERS, multipleInvalidMembers); + assertTreeIsInvalid(multipleInvalidMembersTree); + + String trailingTraits = "{\nFOO\n@bar\n@baz\n}"; + TokenTree trailingTraitsTree = getTree(TreeType.ENUM_SHAPE_MEMBERS, trailingTraits); + assertTreeIsInvalid(trailingTraitsTree); + } + + @Test + public void invalidValueAssignment() { + String missingEquals = "1\n"; + TokenTree missingEqualsTree = getTree(TreeType.VALUE_ASSIGNMENT, missingEquals); + assertTreeIsInvalid(missingEqualsTree); + + String notEquals = " + 1\n"; + TokenTree notEqualsTree = getTree(TreeType.VALUE_ASSIGNMENT, notEquals); + assertTreeIsInvalid(notEqualsTree); + } + + @Test + public void invalidForResource() { + String missingFor = " foo.bar#Baz"; + TokenTree missingForTree = getTree(TreeType.FOR_RESOURCE, missingFor); + assertTreeIsInvalid(missingForTree); + } + + @Test + public void invalidShapeMembers() { + String missingOpenBrace = "foo: bar }"; + TokenTree missingOpenBraceTree = getTree(TreeType.SHAPE_MEMBERS, missingOpenBrace); + assertTreeIsInvalid(missingOpenBraceTree); + + String missingCloseBrace = "{foo: bar"; + TokenTree missingCloseBraceTree = getTree(TreeType.SHAPE_MEMBERS, missingCloseBrace); + assertTreeIsInvalid(missingCloseBraceTree); + + String missingBraces = "foo: bar"; + TokenTree missingBracesTree = getTree(TreeType.SHAPE_MEMBERS, missingBraces); + assertTreeIsInvalid(missingBracesTree); + + String notBraces = "[foo: bar]"; + TokenTree notBracesTree = getTree(TreeType.SHAPE_MEMBERS, notBraces); + assertTreeIsInvalid(notBracesTree); + + String leadingInvalidMember = "{\nfoo: 1\nbar: baz\n}"; + TokenTree leadingInvalidMemberTree = getTree(TreeType.SHAPE_MEMBERS, leadingInvalidMember); + assertTreeIsInvalid(leadingInvalidMemberTree); + + String trailingInvalidMember = "{\nfoo: bar\nbaz:\n}"; + TokenTree trailingInvalidMemberTree = getTree(TreeType.SHAPE_MEMBERS, trailingInvalidMember); + assertTreeIsInvalid(trailingInvalidMemberTree); + + String multipleInvalidMembers = "{\nfoo: @foo({})\nfoo = com.foo#Bar\n}"; + TokenTree multipleInvalidMembersTree = getTree(TreeType.SHAPE_MEMBERS, multipleInvalidMembers); + assertTreeIsInvalid(multipleInvalidMembersTree); + + String trailingTraits = "{\nfoo: bar\n@foo\n@bar\n}"; + TokenTree trailingTraitsTree = getTree(TreeType.SHAPE_MEMBERS, trailingTraits); + assertTreeIsInvalid(trailingTraitsTree); + } + + @Test + public void invalidExplicitShapeMember() { + String missingColon = "foo Bar"; + TokenTree missingColonTree = getTree(TreeType.EXPLICIT_SHAPE_MEMBER, missingColon); + assertTreeIsInvalid(missingColonTree); + + String notColon = "foo = Bar"; + TokenTree notColonTree = getTree(TreeType.EXPLICIT_SHAPE_MEMBER, notColon); + assertTreeIsInvalid(notColonTree); + } + + @Test + public void invalidElidedShapeMember() { + String missingDollar = "Foo"; + TokenTree missingDollarTree = getTree(TreeType.ELIDED_SHAPE_MEMBER, missingDollar); + assertTreeIsInvalid(missingDollarTree); + + String notDollar = "#Foo"; + TokenTree notDollarTree = getTree(TreeType.ELIDED_SHAPE_MEMBER, notDollar); + assertTreeIsInvalid(notDollarTree); + } + + @Test + public void invalidOperationProperty() { + // This production determines which kind of operation property is present, + // so these cases cover unexpected/invalid property names. + + String missingPropertyName = " : FooInput"; + TokenTree missingPropertyNameTree = getTree(TreeType.OPERATION_PROPERTY, missingPropertyName); + assertTreeIsInvalid(missingPropertyNameTree); + + String wrongPropertyName = "unknown: FooInput"; + TokenTree wrongPropertyNameTree = getTree(TreeType.OPERATION_PROPERTY, wrongPropertyName); + assertTreeIsInvalid(wrongPropertyNameTree); + } + + @Test + public void invalidOperationInput() { + String missingColon = "input Foo"; + TokenTree missingColonTree = getTree(TreeType.OPERATION_INPUT, missingColon); + assertTreeIsInvalid(missingColonTree); + + String notColon = "input = Foo"; + TokenTree notColonTree = getTree(TreeType.OPERATION_INPUT, notColon); + assertTreeIsInvalid(notColonTree); + + String missingValue = "input: "; + TokenTree missingValueTree = getTree(TreeType.OPERATION_INPUT, missingValue); + assertTreeIsInvalid(missingValueTree); + } + + @Test + public void invalidOperationOutput() { + String missingColon = "output Foo"; + TokenTree missingColonTree = getTree(TreeType.OPERATION_OUTPUT, missingColon); + assertTreeIsInvalid(missingColonTree); + + String notColon = "output = Foo"; + TokenTree notColonTree = getTree(TreeType.OPERATION_OUTPUT, notColon); + assertTreeIsInvalid(notColonTree); + + String missingValue = "output: "; + TokenTree missingValueTree = getTree(TreeType.OPERATION_OUTPUT, missingValue); + assertTreeIsInvalid(missingValueTree); + } + + @Test + public void invalidOperationErrors() { + String missingColon = "errors Foo"; + TokenTree missingColonTree = getTree(TreeType.OPERATION_ERRORS, missingColon); + assertTreeIsInvalid(missingColonTree); + + + String notColon = "errors = Foo"; + TokenTree notColonTree = getTree(TreeType.OPERATION_ERRORS, notColon); + assertTreeIsInvalid(notColonTree); + + String missingValue = "errors: "; + TokenTree missingValueTree = getTree(TreeType.OPERATION_ERRORS, missingValue); + assertTreeIsInvalid(missingValueTree); + + String missingOpenBracket = "errors: Foo, Bar]"; + TokenTree missingOpenBracketTree = getTree(TreeType.OPERATION_ERRORS, missingOpenBracket); + assertTreeIsInvalid(missingOpenBracketTree); + + String missingCloseBracket = "errors: [Foo, Bar"; + TokenTree missingCloseBracketTree = getTree(TreeType.OPERATION_ERRORS, missingCloseBracket); + assertTreeIsInvalid(missingCloseBracketTree); + + String missingBrackets = "errors: Foo, Bar"; + TokenTree missingBracketsTree = getTree(TreeType.OPERATION_ERRORS, missingBrackets); + assertTreeIsInvalid(missingBracketsTree); + + String notBrackets = "errors: {Foo, Bar}"; + TokenTree notBracketsTree = getTree(TreeType.OPERATION_ERRORS, notBrackets); + assertTreeIsInvalid(notBracketsTree); + } + + @Test + public void invalidInlineAggregateShape() { + String missingWalrus = " @foo\n for Bar with [Baz] {}"; + TokenTree missingWalrusTree = getTree(TreeType.INLINE_AGGREGATE_SHAPE, missingWalrus); + assertTreeIsInvalid(missingWalrusTree); + + String notWalrus = "= @foo\n for bar with [Baz] {}"; + TokenTree notWalrusTree = getTree(TreeType.INLINE_AGGREGATE_SHAPE, notWalrus); + assertTreeIsInvalid(notWalrusTree); + } + + @Test + public void invalidTraitStatements() { + String incomplete = "@foo({bar: baz}\n"; + TokenTree incompleteTree = getTree(TreeType.TRAIT_STATEMENTS, incomplete); + assertTreeIsInvalid(incompleteTree); + + String leadingInvalid = "@foo(:)\n@bar\n"; + TokenTree leadingInvalidTree = getTree(TreeType.TRAIT_STATEMENTS, leadingInvalid); + assertTreeIsInvalid(leadingInvalidTree); + + String trailingInvalid = "@foo\n@bar(\n"; + TokenTree trailingInvalidTree = getTree(TreeType.TRAIT_STATEMENTS, trailingInvalid); + assertTreeIsInvalid(trailingInvalidTree); + + String multipleInvalid = "@foo\n@\nbaz\n@foo({\n@bar\n"; + TokenTree multipleInvalidTree = getTree(TreeType.TRAIT_STATEMENTS, multipleInvalid); + assertTreeIsInvalid(multipleInvalidTree); + } + + @Test + public void invalidTrait() { + String missingAt = "foo(bar: baz)"; + TokenTree missingAtTree = getTree(TreeType.TRAIT, missingAt); + assertTreeIsInvalid(missingAtTree); + } + + @Test + public void invalidTraitBody() { + String missingOpenParen = "foo: bar)"; + TokenTree missingOpenParenTree = getTree(TreeType.TRAIT_BODY, missingOpenParen); + assertTreeIsInvalid(missingOpenParenTree); + + String missingCloseParen = "(foo: bar"; + TokenTree missingCloseParenTree = getTree(TreeType.TRAIT_BODY, missingCloseParen); + assertTreeIsInvalid(missingCloseParenTree); + + String missingParens = "foo: bar"; + TokenTree missingParensTree = getTree(TreeType.TRAIT_BODY, missingParens); + assertTreeIsInvalid(missingParensTree); + + String notParens = "{foo: bar}"; + TokenTree notParensTree = getTree(TreeType.TRAIT_BODY, notParens); + assertTreeIsInvalid(notParensTree); + } + + @Test + public void invalidApplyStatementBlock() { + String missingOpenBrace = "apply foo @bar }"; + TokenTree missingOpenBraceTree = getTree(TreeType.APPLY_STATEMENT_BLOCK, missingOpenBrace); + assertTreeIsInvalid(missingOpenBraceTree); + + String missingCloseBrace = "apply foo { @bar"; + TokenTree missingCloseBraceTree = getTree(TreeType.APPLY_STATEMENT_BLOCK, missingCloseBrace); + assertTreeIsInvalid(missingCloseBraceTree); + + String missingBraces = "apply foo @bar"; + TokenTree missingBracesTree = getTree(TreeType.APPLY_STATEMENT_BLOCK, missingBraces); + assertTreeIsInvalid(missingBracesTree); + + String notBraces = "apply foo [@bar]"; + TokenTree notBracesTree = getTree(TreeType.APPLY_STATEMENT_BLOCK, notBraces); + assertTreeIsInvalid(notBracesTree); + + String invalidTraits = "apply foo {\n@bar(\n@ baz\n}"; + TokenTree invalidTraitsTree = getTree(TreeType.APPLY_STATEMENT_BLOCK, invalidTraits); + assertTreeIsInvalid(invalidTraitsTree); + } + + @Test + public void invalidAbsoluteRootShapeId() { + String notPound = "com.foo$Bar"; + TokenTree notPoundTree = getTree(TreeType.ABSOLUTE_ROOT_SHAPE_ID, notPound); + assertTreeIsInvalid(notPoundTree); + + String trailingPound = "com.foo#"; + TokenTree trailingPoundTree = getTree(TreeType.ABSOLUTE_ROOT_SHAPE_ID, trailingPound); + assertTreeIsInvalid(trailingPoundTree); + + String multiPound = "com.foo##Bar"; + TokenTree multiPoundTree = getTree(TreeType.ABSOLUTE_ROOT_SHAPE_ID, multiPound); + assertTreeIsInvalid(multiPoundTree); + } + + @Test + public void invalidShapeIdMember() { + String missingIdentifier = "$"; + TokenTree missingIdentifierTree = getTree(TreeType.SHAPE_ID_MEMBER, missingIdentifier); + assertTreeIsInvalid(missingIdentifierTree); + + String notDollar = "#Foo"; + TokenTree notDollarTree = getTree(TreeType.SHAPE_ID_MEMBER, notDollar); + assertTreeIsInvalid(notDollarTree); + } + + @Test + public void invalidNamespace() { + String trailingDot = "com.foo."; + TokenTree trailingDotTree = getTree(TreeType.NAMESPACE, trailingDot); + assertTreeIsInvalid(trailingDotTree); + + String multiDot = "com.foo..bar"; + TokenTree multiDotTree = getTree(TreeType.NAMESPACE, multiDot); + assertTreeIsInvalid(multiDotTree); + } + + @Test + public void invalidIdentifier() { + String leadingNumber = "1foo"; + TokenTree leadingNumberTree = getTree(TreeType.IDENTIFIER, leadingNumber); + assertTreeIsInvalid(leadingNumberTree); + + String leadingSymbol = "@foo"; + TokenTree leadingSymbolTree = getTree(TreeType.IDENTIFIER, leadingSymbol); + assertTreeIsInvalid(leadingSymbolTree); + } + + @Test + public void invalidBr() { + // Need the "foo" at the end because EOF is a valid BR. + String missingNewline = "\tfoo"; + TokenTree missingNewlineTree = getTree(TreeType.BR, missingNewline); + assertTreeIsInvalid(missingNewlineTree); + } + + @Test + public void invalidComment() { + String invalidSlashes = "/ / Foo"; + TokenTree invalidSlashesTree = getTree(TreeType.COMMENT, invalidSlashes); + assertTreeIsInvalid(invalidSlashesTree); + } + + private static void rootAndChildTypesEqual(TokenTree actualTree, TreeType expectedRoot, TreeType... expectedChildren) { + assertEquals(expectedRoot, actualTree.getType()); + String actual = actualTree.getChildren().stream().map(t -> t.getType().toString()).collect(Collectors.joining(",")); + String expected = Arrays.stream(expectedChildren).map(Object::toString).collect(Collectors.joining(",")); + assertEquals(expected, actual); + } + + private static void assertTreeIsValid(TokenTree tree) { + if (tree.getType() == TreeType.ERROR) { + Assertions.fail(() -> "Expected tree to be valid, but found error: " + tree); + } else { + for (TokenTree child : tree.getChildren()) { + assertTreeIsValid(child); + } + } + } + + private static void assertTreeIsInvalid(TokenTree tree) { + TreeCursor cursor = tree.zipper(); + if (cursor.findChildrenByType(TreeType.ERROR).isEmpty()) { + Assertions.fail(() -> "Expected tree to be invalid, but found no errors.\nFull tree:\n" + tree); + } + } + + private static TokenTree getTree(TreeType type, String forText) { + CapturingTokenizer tokenizer = new CapturingTokenizer(IdlTokenizer.create(forText)); + type.parse(tokenizer); + // The root of the tree is always IDL with children appended, so the first child is the one we want. + return tokenizer.getRoot().getChildren().get(0); + } +} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/aggregate-shapes.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/aggregate-shapes.smithy new file mode 100644 index 00000000000..fbfaa66374b --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/aggregate-shapes.smithy @@ -0,0 +1,97 @@ +$version: "2.0" + +namespace smithy.example + +list MyList { + member: String +} + +list MyList2 { + /// Hello + member: String +} + +list MyList3 { + @length(min: 1) + member: String +} + +list MyList4 { + // Comment + member: String +} + +map MyMap1 { + key: String + value: MyList3 +} + +map MyMap2 { + /// Docs 1 + key: String + + /// Docs 2 + value: MyList3 +} + +structure Empty {} + +structure OneMember { + foo: String +} + +structure OneMemberWithDefault { + foo: String = "" +} + +structure TwoMembers { + foo: String + bar: String +} + +structure TwoMembersWithOneDefault { + foo: String = "" + bar: String +} + +structure TwoMembersWithDefaults { + foo: String = "" + bar: String = "" +} + +structure ThreeMembersWithTraits { + /// foo + foo: String = "" + + @required + @deprecated + bar: String = "" + + /// Docs 1 + /// Docs 2 + @required + baz: String +} + +@mixin +structure MyMixin1 {} + +@mixin +structure MyMixin2 {} + +structure HasMixins with [MyMixin1, MyMixin2] { + greeting: String +} + +structure HasForResource for MyResource with [MyMixin1] {} + +resource MyResource { + operations: [PutMyResource] +} + +operation PutMyResource { + input := {} + output: PutMyResourceOutput +} + +structure PutMyResourceOutput {} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/awkward-comments.formatted.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/awkward-comments.formatted.smithy new file mode 100644 index 00000000000..c15c7a5faff --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/awkward-comments.formatted.smithy @@ -0,0 +1,129 @@ +$version: "2.0" + +metadata a = [ + // a +] + +metadata b = [] // b + +metadata c = [ + // c + 100 +] + +metadata d = { + // d +} + +metadata e = {} // e + +metadata f = { + // f + f: 100 +} + +metadata g = { + // g + g: 100 +} + +metadata h = { + h: 100 + // h +} + +metadata i = { + i: 100 + // i1 + // i2 +} + +metadata j = { + j1: 100 + // j1a + // j1b + j2: 100 + // j2a + // j2b +} + +metadata k = { + // k1a + k1: 100 + // k1b + k2: 100 + // k2a + // k2b +} + +// na +namespace smithy.example // nb + +// nc +use smithy.api#Integer // i + +// l1 +use smithy.api#Long // l2 + +// l3 +use smithy.api#String // s + +/// Docs 1 +/// Docs 2 +@required +// trailing trait comment +@length( + // Leading + // Pre-colon + // Post colon 1 + // Post-colon 2 + min: 1 + // post value + // Pre-close +) +// trailing +string MyString // Trailing comment + +/// Docs for next +integer MyInteger + +@tags( + // a + [ + // b + "foo" + // c + ] + // d +) +// e +integer MyInteger2 + +// Hello 1 +// Hello 2 +// a +structure MyStruct { + // b + // c + a: String + + // d + // e + // f + b: String + + // g + // h +} // i + +// j +structure MyStructure2 { + // k + // l + i: Integer + + l: Long + + // m +} // n +// o diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/awkward-comments.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/awkward-comments.smithy new file mode 100644 index 00000000000..4974beaeb0d --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/awkward-comments.smithy @@ -0,0 +1,93 @@ +$version: "2.0" + +metadata a = [ // a +] +metadata b = [ +] // b +metadata c = [ + // c + 100 +] +metadata d = { // d +} +metadata e = {} // e +metadata f = { + f // f + : 100 +} +metadata g = { + g: // g + 100 +} +metadata h = { + h: 100 // h +} +metadata i = { + i: 100 // i1 + // i2 +} +metadata j = { + j1: 100 // j1a + // j1b + j2: 100 // j2a + // j2b +} +metadata k = { + k1: // k1a + 100 // k1b + k2: 100 // k2a + // k2b +} + +// na +namespace smithy.example // nb +// nc + +use smithy.api#Integer // i +// l1 +use smithy.api#Long // l2 +// l3 +use smithy.api#String // s + +/// Docs 1 +/// Docs 2 +@required // trailing trait comment +@length( // Leading + min + // Pre-colon + : // Post colon 1 + // Post-colon 2 + 1 // post value + // Pre-close +) // trailing +string MyString // Trailing comment +/// Docs for next + +integer MyInteger + +@tags(// a + [ // b + "foo" // c + ] // d +) // e +integer MyInteger2 + +// Hello 1 +// Hello 2 +structure MyStruct // a +{ // b + // c + a: String // d + // e + // f + b: String // g + // h +} // i +// j + +structure MyStructure2 { //k + // l + i: Integer + l: Long // m +} // n +// o diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/br-comments-no-shapes.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/br-comments-no-shapes.smithy new file mode 100644 index 00000000000..70124858ca1 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/br-comments-no-shapes.smithy @@ -0,0 +1,9 @@ +$version: "2.0" + +// Comment 1 +metadata foo1 = "bar" + +// Comment 2 +metadata foo2 = "bam" + +// Comment 3 diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/br-comments-with-use.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/br-comments-with-use.smithy new file mode 100644 index 00000000000..d641aa21042 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/br-comments-with-use.smithy @@ -0,0 +1,10 @@ +$version: "2.0" + +namespace smithy.example + +use smithy.api#sensitive + +// Comment +/// A +@sensitive +string A diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/br-comments.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/br-comments.smithy new file mode 100644 index 00000000000..e674d7f6b7a --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/br-comments.smithy @@ -0,0 +1,16 @@ +$version: "2.0" + +namespace smithy.example + +/// A +string A + +/// B1 +/// B2 +structure B {} + +// Regular comment +/// C +@sensitive +integer C +// Trailing comments are not omitted. diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/comment-correct-spacing.formatted.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/comment-correct-spacing.formatted.smithy new file mode 100644 index 00000000000..5c56fb81318 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/comment-correct-spacing.formatted.smithy @@ -0,0 +1,9 @@ +$version: "2.0" + +// This needs a space +namespace smithy.example + +/// This needs a space too. +/// Also this one. +/// Don't remove extra spaces. +string MyString diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/comment-correct-spacing.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/comment-correct-spacing.smithy new file mode 100644 index 00000000000..3386b919694 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/comment-correct-spacing.smithy @@ -0,0 +1,9 @@ +$version: "2.0" + +//This needs a space +namespace smithy.example + +///This needs a space too. +///Also this one. +/// Don't remove extra spaces. +string MyString diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/entity-shapes.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/entity-shapes.smithy new file mode 100644 index 00000000000..8018e75f584 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/entity-shapes.smithy @@ -0,0 +1,26 @@ +$version: "2.0" + +namespace smithy.example + +/// Documentation +@auth([httpBasicAuth]) +service Foo { + version: "2" + operations: [GetTime1, GetTime2] + resources: [Sprocket1, Sprocket2] +} + +operation GetTime1 {} + +operation GetTime2 { + input := {} +} + +resource Sprocket1 { + identifiers: {username: String} +} + +@http(method: "X", uri: "/foo", code: 200) +resource Sprocket2 { + identifiers: {username: String, id: String, otherId: String} +} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/fixes-invalid-doc-comments.formatted.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/fixes-invalid-doc-comments.formatted.smithy new file mode 100644 index 00000000000..b6984e10c44 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/fixes-invalid-doc-comments.formatted.smithy @@ -0,0 +1,48 @@ +// 1 (change) +// 2 (change) +$version: "2.0" // 3 (change) + +// 4 (change) +// 5 (change) +metadata x = 123 // 6 (change) + +// 7 (change) +metadata y = [ + // 8 (change) + 123 +] + +// 9 (change) +namespace smithy.example + +// 10 (change) +use smithy.api#Integer + +// 11 (change) +use smithy.api#String + +/// 12 (keep) +@deprecated +/// 13 (change) +structure Foo { + /// 14 (keep) + @length( + // 15 (change) + min: 1 + ) + /// 16 (change) + @since("1.x") + /// 17 (TODO: change) + bar: String + + // 18 (change) +} + +// 19 (change) +apply Foo @tags(["a"]) + +/// 20 (keep) +list Baz { + member: Integer +} +// 21 (change) diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/fixes-invalid-doc-comments.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/fixes-invalid-doc-comments.smithy new file mode 100644 index 00000000000..e16e23e1cc9 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/fixes-invalid-doc-comments.smithy @@ -0,0 +1,48 @@ +/// 1 (change) +/// 2 (change) +$version: "2.0" /// 3 (change) +/// 4 (change) + +/// 5 (change) +metadata x = 123 /// 6 (change) + +/// 7 (change) +metadata y = [ + /// 8 (change) + 123 +] + +/// 9 (change) +namespace smithy.example + +/// 10 (change) +use smithy.api#Integer + +/// 11 (change) +use smithy.api#String + +/// 12 (keep) +@deprecated +/// 13 (change) +structure Foo { + /// 14 (keep) + @length( + /// 15 (change) + min: 1 + ) + /// 16 (change) + @since("1.x") + /// 17 (TODO: change) + bar: String + /// 18 (change) +} + +/// 19 (change) +apply Foo @tags(["a"]) + +/// 20 (keep) +list Baz { + member: Integer +} + +/// 21 (change) diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/metadata-with-quoted-key.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/metadata-with-quoted-key.smithy new file mode 100644 index 00000000000..88252362806 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/metadata-with-quoted-key.smithy @@ -0,0 +1,3 @@ +$version: "2.0" + +metadata "com.foo.bar" = "Hello" diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/mixins-too-long.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/mixins-too-long.smithy new file mode 100644 index 00000000000..0ed9cfbf8a0 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/mixins-too-long.smithy @@ -0,0 +1,67 @@ +$version: "2.0" + +namespace smithy.example + +string MyString1 with [ThisIsReallyLongStringName1, ThisIsReallyLongStringName2, ThisIsReallyLongStringName3] + +string MyString2 with [ + ThisIsReallyLongStringName1 + ThisIsReallyLongStringName2 + ThisIsReallyLongStringName3 + ThisIsReallyLongStringName4 +] + +structure MyStruct1 with [ThisIsReallyLongStructName1, ThisIsReallyLongStructName2, ThisIsReallyLongStructName3] {} + +structure MyStruct2 with [ + ThisIsReallyLongStructName1 + ThisIsReallyLongStructName2 + ThisIsReallyLongStructName3 + ThisIsReallyLongStructName4 +] {} + +structure MyStruct3 with [ + ThisIsReallyLongStructName1 + ThisIsReallyLongStructName2 + ThisIsReallyLongStructName3 + ThisIsReallyLongStructName4 +] { + foo: String + bar: String +} + +structure MyStruct11111111 with [ThisIsReallyLongStructName1, ThisIsReallyLongStructName2, ThisIsReallyLongStructName3] +{} + +structure MyStruct111112 with [ThisIsReallyLongStructName1, ThisIsReallyLongStructName2, ThisIsReallyLongStructName3] { + foo: Bar +} + +structure MyStruct111111113 with [ThisIsReallyLongStructName1, ThisIsReallyLongStructName2, ThisIsReallyLongStructName3] +{ + foo: Bar +} + +@mixin +string ThisIsReallyLongStringName1 + +@mixin +string ThisIsReallyLongStringName2 + +@mixin +string ThisIsReallyLongStringName3 + +@mixin +string ThisIsReallyLongStringName4 + +@mixin +structure ThisIsReallyLongStructName1 {} + +@mixin +structure ThisIsReallyLongStructName2 {} + +@mixin +structure ThisIsReallyLongStructName3 {} + +@mixin +structure ThisIsReallyLongStructName4 {} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/operation-test.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/operation-test.smithy new file mode 100644 index 00000000000..cf384f033ad --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/operation-test.smithy @@ -0,0 +1,79 @@ +$version: "2.0" + +namespace smithy.example + +operation Empty {} + +operation OnlyInput { + input: Struct +} + +operation OnlyOutput { + output: Struct +} + +operation InputAndOutput { + input: Struct + output: Struct +} + +operation InputAndOutputWithComments { + // A + input: Struct + + // B + output: Struct +} + +operation InlineEmptyInput { + input := {} +} + +operation InlineEmptyOutput { + output := {} +} + +operation InlineEmptyInputOutput { + input := {} + output := {} +} + +operation InlineEmptyInputOutputWithResource { + input := for Foo {} + output := for Foo {} +} + +operation InlineEmptyInputOutputWithResourceAndMixins { + input := for Foo with [X] {} + output := for Foo with [X, Y] {} +} + +operation InlineEmptyInputOutputWithTraits { + input := + @since("1.0") + for Foo {} + + output := + @since("1.0") + for Foo {} +} + +operation InlineEmptyInputOutputWithComments { + input := + /// Docs 1 + for Foo {} + + output := + /// Docs 2 + for Foo {} +} + +structure Struct {} + +resource Foo {} + +@mixin +structure X {} + +@mixin +structure Y {} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/remove-unusued-use.formatted.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/remove-unusued-use.formatted.smithy new file mode 100644 index 00000000000..1c6da282dc8 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/remove-unusued-use.formatted.smithy @@ -0,0 +1,19 @@ +$version: "2.0" + +namespace smithy.example + +use smithy.api#Integer +use smithy.api#String +use smithy.api#range +use smithy.api#required +use smithy.api#sensitive + +structure Foo { + @required + // this is strange, but resolve to the full ID + @documentation(Integer) + a: String + + @tags([sensitive, range$min, smithy.api#Long, smithy.api#Short]) + b: String +} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/remove-unusued-use.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/remove-unusued-use.smithy new file mode 100644 index 00000000000..536f7923b06 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/remove-unusued-use.smithy @@ -0,0 +1,27 @@ +$version: "2.0" + +namespace smithy.example + +use smithy.api#Boolean +use smithy.api#Document +use smithy.api#Integer +use smithy.api#Long +use smithy.api#String +use smithy.api#range +use smithy.api#required +use smithy.api#sensitive + +structure Foo { + @required + // this is strange, but resolve to the full ID + @documentation(Integer) + a: String + + @tags([ + sensitive + range$min + smithy.api#Long + smithy.api#Short + ]) + b: String +} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/section-statement-blank-lines.formatted.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/section-statement-blank-lines.formatted.smithy new file mode 100644 index 00000000000..ff3b73ab84a --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/section-statement-blank-lines.formatted.smithy @@ -0,0 +1,42 @@ +// Blank lines are added before and after control, metadata, and use statements if the statement contains any +// newline. +$version: "2.0" +$foo: 100 +$baz: {abc: 123} + +$bar: { + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1" + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2" +} + +metadata a = "a" +metadata b = "b" + +// A comment makes a blank line here, and beneath too. +metadata c = "c" + +metadata d = "d" + +metadata e = { + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1" + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2" +} + +metadata f = 100 + +namespace smithy.example + +use smithy.api#Boolean + +// This comment separates this use statement from it's leading and trailing lines. +use smithy.api#Integer + +// This comment separates this use statement from it's leading and trailing lines. +// It does not add an trailing line though since it's the last statement in the section. +use smithy.api#String + +structure Shape { + a: String + b: Boolean + c: Integer +} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/section-statement-blank-lines.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/section-statement-blank-lines.smithy new file mode 100644 index 00000000000..31a00cc7dd3 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/section-statement-blank-lines.smithy @@ -0,0 +1,25 @@ +// Blank lines are added before and after control, metadata, and use statements if the statement contains any +// newline. +$version: "2.0" +$foo: 100 +$baz: {abc: 123} +$bar: {aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2"} + +metadata a = "a" +metadata b = "b" +// A comment makes a blank line here, and beneath too. +metadata c = "c" +metadata d = "d" +metadata e = {aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1" + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa2"} +metadata f = 100 + +namespace smithy.example +use smithy.api#Boolean +// This comment separates this use statement from it's leading and trailing lines. +use smithy.api#Integer +// This comment separates this use statement from it's leading and trailing lines. +// It does not add an trailing line though since it's the last statement in the section. +use smithy.api#String + +structure Shape {a: String b: Boolean c: Integer} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/simple-model.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/simple-model.smithy new file mode 100644 index 00000000000..2b63f95c449 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/simple-model.smithy @@ -0,0 +1,10 @@ +$version: "2.0" + +metadata foo = "hello" + +namespace smithy.example + +/// Documentation +@sensitive +@length(min: 10, max: 100) +string MyString diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/simple-shapes.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/simple-shapes.smithy new file mode 100644 index 00000000000..fb4f2dd99aa --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/simple-shapes.smithy @@ -0,0 +1,10 @@ +$version: "2.0" + +namespace smithy.example + +string String + +integer Integer + +@deprecated +boolean MyBoolean diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements-with-comments.formatted.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements-with-comments.formatted.smithy new file mode 100644 index 00000000000..14fc247b46e --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements-with-comments.formatted.smithy @@ -0,0 +1,34 @@ +// Comments that appear above USE_STATEMENTs are found in the WS tree before the statement. This gets tricky with +// the first use statement and the last use statement. The first use statement actually gets comments from the +// preceding NAMESPACE_STATEMENT. The last use statement actually provides comments for any subsequent shapes. +$version: "2.0" + +namespace smithy.example + +// Comes before Integer 1 +// Comes before Integer 2 +use smithy.api#Integer // Trailing smithy.api#Integer + +// Comes before Long 1 +// Comes before Long 2 +use smithy.api#Long // Trailing smithy.api#Long + +// This set of comments is actually in the BR->WS of the above NAMESPACE_STATEMENT. +// When moved, it needs to be removed from NAMESPACE_STATEMENT, and added to the USE_STATEMENT prior to where +// smithy.api#String is reorderd. +use smithy.api#String // Trailing smithy.api#String + +// Comes before required 1 +// Comes before required 2 +use smithy.api#required // Trailing smithy.api#required + +// These comments are actually part of the last USE_STATEMENT. When the last statement is moved, this comment must +// be removed from that statement and added to the updated last statement. +structure UseThem { + @required + string: String + + integer: Integer + + long: Long +} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements-with-comments.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements-with-comments.smithy new file mode 100644 index 00000000000..e99078579f1 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements-with-comments.smithy @@ -0,0 +1,34 @@ +// Comments that appear above USE_STATEMENTs are found in the WS tree before the statement. This gets tricky with +// the first use statement and the last use statement. The first use statement actually gets comments from the +// preceding NAMESPACE_STATEMENT. The last use statement actually provides comments for any subsequent shapes. +$version: "2.0" + +namespace smithy.example + +// This set of comments is actually in the BR->WS of the above NAMESPACE_STATEMENT. +// When moved, it needs to be removed from NAMESPACE_STATEMENT, and added to the USE_STATEMENT prior to where +// smithy.api#String is reorderd. +use smithy.api#String // Trailing smithy.api#String + +// Comes before Integer 1 +// Comes before Integer 2 +use smithy.api#Integer // Trailing smithy.api#Integer + +// Comes before required 1 +// Comes before required 2 +use smithy.api#required // Trailing smithy.api#required + +// Comes before Long 1 +// Comes before Long 2 +use smithy.api#Long // Trailing smithy.api#Long + +// These comments are actually part of the last USE_STATEMENT. When the last statement is moved, this comment must +// be removed from that statement and added to the updated last statement. +structure UseThem { + @required + string: String + + integer: Integer + + long: Long +} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements.formatted.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements.formatted.smithy new file mode 100644 index 00000000000..2134be80f99 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements.formatted.smithy @@ -0,0 +1,17 @@ +$version: "2.0" + +namespace smithy.example + +use smithy.api#Integer +use smithy.api#Long +use smithy.api#String +use smithy.api#required + +structure UseThem { + @required + string: String + + integer: Integer + + long: Long +} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements.smithy new file mode 100644 index 00000000000..1ab1765ad99 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/sorts-use-statements.smithy @@ -0,0 +1,17 @@ +$version: "2.0" + +namespace smithy.example + +use smithy.api#String +use smithy.api#Integer +use smithy.api#Long +use smithy.api#required + +structure UseThem { + @required + string: String + + integer: Integer + + long: Long +} diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-string-single-line.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-string-single-line.smithy new file mode 100644 index 00000000000..95d88fb8812 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-string-single-line.smithy @@ -0,0 +1,6 @@ +$version: "2.0" + +namespace smithy.example + +@documentation("Hello!") +string Example diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-string-span-lines.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-string-span-lines.smithy new file mode 100644 index 00000000000..68b029f11d2 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-string-span-lines.smithy @@ -0,0 +1,9 @@ +$version: "2.0" + +namespace smithy.example + +@documentation( + "Hello! +This string spans lines!" +) +string Example diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-structured-multi-line.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-structured-multi-line.smithy new file mode 100644 index 00000000000..0a3dfb4a567 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-structured-multi-line.smithy @@ -0,0 +1,17 @@ +$version: "2.0" + +namespace smithy.example + +@externalDocumentation( + a: "http://foo.com" + b: "http://baz.com" + c: "http://baz.com" + d: "http://baz.com" + e: "http://baz.com" + f: "http://baz.com" + g: "http://baz.com" + h: "http://baz.com" + i: "http://baz.com" + j: "http://baz.com" +) +string Foo diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-structured-one-line.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-structured-one-line.smithy new file mode 100644 index 00000000000..750844162f9 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-structured-one-line.smithy @@ -0,0 +1,6 @@ +$version: "2.0" + +namespace smithy.example + +@externalDocumentation(a: "http://foo.com", b: "http://baz.com") +string Foo diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-textblock.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-textblock.smithy new file mode 100644 index 00000000000..f586af1aaa9 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/trait-textblock.smithy @@ -0,0 +1,18 @@ +$version: "2.0" + +namespace smithy.example + +@documentation( + """ + This is the documentation for Foo. + Lorem ipsum dolor.""" +) +string Foo + +@documentation( + """ + This is the documentation for Bar. + Lorem ipsum dolor. + """ +) +string Bar diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/traits-few-tags.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/traits-few-tags.smithy new file mode 100644 index 00000000000..e72eb6ff584 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/traits-few-tags.smithy @@ -0,0 +1,6 @@ +$version: "2.0" + +namespace smithy.example + +@tags(["abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz"]) +string Foo diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/traits-lots-of-tags.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/traits-lots-of-tags.smithy new file mode 100644 index 00000000000..ed06893c2a3 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/traits-lots-of-tags.smithy @@ -0,0 +1,22 @@ +$version: "2.0" + +namespace smithy.example + +@tags( + [ + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz" + ] +) +string Foo diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/version-only-with-comments.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/version-only-with-comments.smithy new file mode 100644 index 00000000000..a943125db5e --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/version-only-with-comments.smithy @@ -0,0 +1,4 @@ +// Leading comment +$version: "1.0" + +// Trailing comment diff --git a/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/version-only.smithy b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/version-only.smithy new file mode 100644 index 00000000000..adccc0c5981 --- /dev/null +++ b/smithy-syntax/src/test/resources/software/amazon/smithy/syntax/formatter/version-only.smithy @@ -0,0 +1 @@ +$version: "1.0"