Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for 1.0 downgrades and serialization #1403

Merged
merged 1 commit into from
Sep 14, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,11 @@ public Builder id(ShapeId shapeId) {
return this;
}

@Override
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The fluent interface here needed this override for it to work

public Builder id(String shapeId) {
return id(ShapeId.from(shapeId));
}

/**
* Replaces the members of the builder.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,16 @@
import software.amazon.smithy.model.node.Node;
import software.amazon.smithy.model.node.ObjectNode;
import software.amazon.smithy.model.node.StringNode;
import software.amazon.smithy.model.traits.AddedDefaultTrait;
import software.amazon.smithy.model.traits.BoxTrait;
import software.amazon.smithy.model.traits.ClientOptionalTrait;
import software.amazon.smithy.model.traits.DefaultTrait;
import software.amazon.smithy.model.traits.NotPropertyTrait;
import software.amazon.smithy.model.traits.PropertyTrait;
import software.amazon.smithy.model.traits.Trait;
import software.amazon.smithy.model.transform.ModelTransformer;
import software.amazon.smithy.utils.FunctionalUtils;
import software.amazon.smithy.utils.SetUtils;
import software.amazon.smithy.utils.SmithyBuilder;
import software.amazon.smithy.utils.StringUtils;

Expand All @@ -50,26 +58,62 @@
* to formats like JSON, YAML, Ion, etc.
*/
public final class ModelSerializer {

// Explicitly filter out these traits. While some of these are automatically removed in the downgradeToV1
// model transformation, calling them out here explicitly is a defense in depth. This also has to remove all
// default traits from the output, whereas the downgradeToV1 transform only removes unnecessary default traits
// that don't correlate to boxing in V1 models.
private static final Set<ShapeId> V2_TRAITS_TO_FILTER_FROM_V1 = SetUtils.of(
DefaultTrait.ID,
AddedDefaultTrait.ID,
ClientOptionalTrait.ID,
PropertyTrait.ID,
NotPropertyTrait.ID);

private final Predicate<String> metadataFilter;
private final Predicate<Shape> shapeFilter;
private final Predicate<Trait> traitFilter;
private final String version;

private ModelSerializer(Builder builder) {
metadataFilter = builder.metadataFilter;
version = builder.version;

if (!builder.includePrelude) {
shapeFilter = builder.shapeFilter.and(FunctionalUtils.not(Prelude::isPreludeShape));
} else if (version.equals("1.0")) {
throw new UnsupportedOperationException("Cannot serialize prelude and set model version to 1.0");
} else {
shapeFilter = builder.shapeFilter;
}
// Never serialize synthetic traits.
traitFilter = builder.traitFilter.and(FunctionalUtils.not(Trait::isSynthetic));

if (version.equals("1.0")) {
traitFilter = builder.traitFilter.and(trait -> {
if (trait.toShapeId().equals(BoxTrait.ID)) {
// Include the box trait in 1.0 models.
return true;
} else if (V2_TRAITS_TO_FILTER_FROM_V1.contains(trait.toShapeId())) {
// Exclude V2 specific traits.
return false;
} else {
return !trait.isSynthetic();
}
});
} else {
// 2.0 models just need to filter out synthetic traits, including box.
traitFilter = builder.traitFilter.and(FunctionalUtils.not(Trait::isSynthetic));
}
}

public ObjectNode serialize(Model model) {
ShapeSerializer shapeSerializer = new ShapeSerializer();

if (version.equals("1.0")) {
model = ModelTransformer.create().downgradeToV1(model);
}

ObjectNode.Builder builder = Node.objectNodeBuilder()
.withMember("smithy", Node.from(Model.MODEL_VERSION))
.withMember("smithy", Node.from(version))
.withOptionalMember("metadata", createMetadata(model).map(Node::withDeepSortedKeys));

// Sort shapes by ID.
Expand Down Expand Up @@ -124,6 +168,7 @@ public static final class Builder implements SmithyBuilder<ModelSerializer> {
private Predicate<Shape> shapeFilter = FunctionalUtils.alwaysTrue();
private boolean includePrelude = false;
private Predicate<Trait> traitFilter = FunctionalUtils.alwaysTrue();
private String version = "2.0";

private Builder() {}

Expand Down Expand Up @@ -181,6 +226,31 @@ public Builder traitFilter(Predicate<Trait> traitFilter) {
return this;
}

/**
* Sets the IDL version to serialize. Defaults to 2.0.
*
* <p>Version "1.0" serialization cannot be used with {@link #includePrelude}.
*
* @param version IDL version to set. Can be "1", "1.0", "2", or "2.0".
* "1" and "2" are normalized to "1.0" and "2.0".
* @return Returns the builder.
*/
public Builder version(String version) {
switch (version) {
case "2":
case "2.0":
this.version = "2.0";
break;
case "1":
case "1.0":
this.version = "1.0";
break;
default:
throw new IllegalArgumentException("Unsupported Smithy model version: " + version);
}
return this;
}

@Override
public ModelSerializer build() {
return new ModelSerializer(this);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/*
* Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package software.amazon.smithy.model.transform;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import software.amazon.smithy.model.Model;
import software.amazon.smithy.model.knowledge.NullableIndex;
import software.amazon.smithy.model.shapes.MemberShape;
import software.amazon.smithy.model.shapes.ResourceShape;
import software.amazon.smithy.model.shapes.Shape;
import software.amazon.smithy.model.shapes.ShapeId;
import software.amazon.smithy.model.shapes.ShapeType;
import software.amazon.smithy.model.shapes.StructureShape;
import software.amazon.smithy.model.traits.AddedDefaultTrait;
import software.amazon.smithy.model.traits.ClientOptionalTrait;
import software.amazon.smithy.model.traits.DefaultTrait;
import software.amazon.smithy.model.traits.NotPropertyTrait;
import software.amazon.smithy.model.traits.PropertyTrait;

final class DowngradeToV1 {

Model transform(ModelTransformer transformer, Model model) {
// Flatten and remove mixins since they aren't part of IDL 1.0
model = transformer.flattenAndRemoveMixins(model);

// Change enums to string shapes and intEnums to integers.
model = downgradeEnums(transformer, model);

// Remove resource properties
model = removeResourceProperties(transformer, model);

// Remove default traits that do not correlate to box traits from v1.
model = removeUnnecessaryDefaults(transformer, model);

return removeOtherV2Traits(transformer, model);
}

private Model downgradeEnums(ModelTransformer transformer, Model model) {
Map<ShapeId, ShapeType> typeChanges = new HashMap<>();

for (Shape shape : model.getEnumShapes()) {
typeChanges.put(shape.getId(), ShapeType.STRING);
}

for (Shape shape : model.getIntEnumShapes()) {
typeChanges.put(shape.getId(), ShapeType.INTEGER);
}

return transformer.changeShapeType(model, typeChanges);
}

private Model removeResourceProperties(ModelTransformer transformer, Model model) {
List<Shape> updates = new ArrayList<>();

// Remove the "properties" key from resources.
for (ResourceShape shape : model.getResourceShapes()) {
if (shape.hasProperties()) {
updates.add(shape.toBuilder().properties(Collections.emptyMap()).build());
}
}

// Remove @notProperty.
for (Shape shape : model.getShapesWithTrait(NotPropertyTrait.class)) {
updates.add(Shape.shapeToBuilder(shape).removeTrait(NotPropertyTrait.ID).build());
}

// Remove @property.
for (Shape shape : model.getShapesWithTrait(PropertyTrait.class)) {
updates.add(Shape.shapeToBuilder(shape).removeTrait(PropertyTrait.ID).build());
}

return transformer.replaceShapes(model, updates);
}

private Model removeUnnecessaryDefaults(ModelTransformer transformer, Model model) {
Set<Shape> updates = new HashSet<>();

// Remove addedDefault traits, and any found default trait if present.
for (MemberShape shape : model.getMemberShapesWithTrait(AddedDefaultTrait.class)) {
updates.add(shape.toBuilder().removeTrait(DefaultTrait.ID).removeTrait(AddedDefaultTrait.ID).build());
}

for (Shape shape : model.getShapesWithTrait(DefaultTrait.class)) {
DefaultTrait trait = shape.expectTrait(DefaultTrait.class);
// Members with a null default are considered boxed. Keep the trait to retain consistency with other
// indexes and checks.
if (!trait.toNode().isNullNode()) {
if (!NullableIndex.isDefaultZeroValueOfTypeInV1(trait.toNode(), shape.getType())) {
gosar marked this conversation as resolved.
Show resolved Hide resolved
updates.add(Shape.shapeToBuilder(shape)
.removeTrait(DefaultTrait.ID)
.removeTrait(AddedDefaultTrait.ID)
.build());
}
}
}

return transformer.replaceShapes(model, updates);
}

private Model removeOtherV2Traits(ModelTransformer transformer, Model model) {
Set<Shape> updates = new HashSet<>();

for (StructureShape structure : model.getStructureShapes()) {
for (MemberShape member : structure.getAllMembers().values()) {
Comment on lines +122 to +123
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could have been getMemberShapesWithTrait like above?

if (member.hasTrait(ClientOptionalTrait.class)) {
updates.add(member.toBuilder().removeTrait(ClientOptionalTrait.ID).build());
}
}
}

return transformer.replaceShapes(model, updates);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -636,4 +636,20 @@ public Model flattenAndRemoveMixins(Model model) {
public Model addClientOptional(Model model, boolean applyWhenNoDefaultValue) {
return new AddClientOptional(applyWhenNoDefaultValue).transform(this, model);
}

/**
* Removes Smithy IDL 2.0 features from a model that are not strictly necessary to keep for consistency with the
* rest of Smithy.
*
* <p>This transformer converts enum shapes to string shapes with the enum trait, intEnum shapes to integer shapes,
* flattens and removes mixins, removes properties from resources, and removes default traits that have no impact
* on IDL 1.0 semantics (i.e., default traits on structure members set to something other than null, or default
* traits on any other shape that are not the zero value of the shape of a 1.0 model).
*
* @param model Model to downgrade.
* @return Returns the downgraded model.
*/
public Model downgradeToV1(Model model) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The javadoc does say 'remove' a bunch, but probably worth an explicit callout that this is lossy.

return new DowngradeToV1().transform(this, model);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.startsWith;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.IOException;
Expand All @@ -34,13 +33,13 @@
import java.util.Optional;
import java.util.TreeMap;
import java.util.stream.Stream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.DynamicTest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestFactory;
import software.amazon.smithy.model.Model;
import software.amazon.smithy.model.SourceLocation;
import software.amazon.smithy.model.node.Node;
import software.amazon.smithy.model.node.NodeMapper;
import software.amazon.smithy.model.node.NodePointer;
import software.amazon.smithy.model.node.ObjectNode;
import software.amazon.smithy.model.traits.DocumentationTrait;
Expand All @@ -53,6 +52,7 @@ public class ModelSerializerTest {
public Stream<DynamicTest> generateTests() throws IOException, URISyntaxException {
return Files.list(Paths.get(
SmithyIdlModelSerializer.class.getResource("ast-serialization/cases").toURI()))
.filter(path -> !path.toString().endsWith(".1.0.json"))
.map(path -> DynamicTest.dynamicTest(path.getFileName().toString(), () -> testRoundTrip(path)));
}

Expand All @@ -63,6 +63,14 @@ public void testRoundTrip(Path path) {
ObjectNode expected = Node.parse(IoUtils.readUtf8File(path)).expectObjectNode();

Node.assertEquals(actual, expected);

// Now validate the file is serialized correctly when downgraded to 1.0.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would we be able to have a test that starts a 1.0 model, loads it and serializes it as 1.0 and compares?

Path downgradeFile = Paths.get(path.toString().replace(".json", ".1.0.json"));
ObjectNode expectedDowngrade = Node.parse(IoUtils.readUtf8File(downgradeFile)).expectObjectNode();
ModelSerializer serializer1 = ModelSerializer.builder().version("1.0").build();
ObjectNode model1 = serializer1.serialize(model);

Node.assertEquals(model1, expectedDowngrade);
}

@Test
Expand Down Expand Up @@ -252,4 +260,18 @@ public void serializesResourceProperties() {
"{\"type\":\"resource\",\"properties\":{\"fooProperty\":{\"target\":\"ns.foo#Shape\"}}}}}");
Node.assertEquals(node, expectedNode);
}

@Test
public void failsOnInvalidVersion() {
Assertions.assertThrows(IllegalArgumentException.class, () -> {
ModelSerializer.builder().version("1.5").build();
});
}

@Test
public void failsWhenUsingV1WithPrelude() {
Assertions.assertThrows(UnsupportedOperationException.class, () -> {
ModelSerializer.builder().version("1.0").includePrelude(true).build();
});
}
}
Loading