From 8a624c1886360a8a354fea6658047c9597b0ea1f Mon Sep 17 00:00:00 2001 From: Michael Dowling Date: Sat, 22 Aug 2020 20:26:51 -0700 Subject: [PATCH] Add :topdown selector to match hierarchically The topdown selector is used to match shapes hierarchically based on a "qualifier" selector and optional "disqualifier" selector. This can be used to match shapes that are marked as controlPlane or dataPlane using inheritance from resource/service bindings, match shapes that use httpBasic auth, etc. --- docs/source/1.0/spec/core/selectors.rst | 87 +++++++++++++++ .../smithy/model/selector/SelectorParser.java | 7 ++ .../model/selector/TopDownSelector.java | 85 ++++++++++++++ .../model/selector/TopDownSelectorTest.java | 104 ++++++++++++++++++ .../model/selector/recursive-resources.smithy | 12 ++ .../smithy/model/selector/topdown-auth.smithy | 39 +++++++ .../selector/topdown-exclusive-traits.smithy | 34 ++++++ 7 files changed, 368 insertions(+) create mode 100644 smithy-model/src/main/java/software/amazon/smithy/model/selector/TopDownSelector.java create mode 100644 smithy-model/src/test/java/software/amazon/smithy/model/selector/TopDownSelectorTest.java create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/selector/recursive-resources.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/selector/topdown-auth.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/selector/topdown-exclusive-traits.smithy diff --git a/docs/source/1.0/spec/core/selectors.rst b/docs/source/1.0/spec/core/selectors.rst index 3da01959151..df62baee15e 100644 --- a/docs/source/1.0/spec/core/selectors.rst +++ b/docs/source/1.0/spec/core/selectors.rst @@ -1359,6 +1359,93 @@ trait applied to it: service :not(-[trait]-> [trait|protocolDefinition]) +``:topdown`` +------------ + +The ``:topdown`` function matches service, resource, and operation shapes +and resource and operation shapes within their containment hierarchy. The +``:topdown`` function starts at each given shape and forward-traverses +the containment hierarchy of the shape by following ``operation`` and +``resource`` :ref:`relationships ` from the shape +to its neighbors; this function *does not* traverse *up* the containment +hierarchy of a given shape to check if the shape is within the containment +hierarchy of a qualified service or resource shape. This function essentially +allows shapes to be matched by inheriting from the resource or service they +are bound to. + +.. rubric:: Selector arguments + +Exactly one or two selectors MUST be provided to the ``:topdown`` selector: + +1. The first selector is the "qualifier". It is used to mark a shape as a + match. If the selector yields any results, then it is considered a match. +2. If provided, the second selector is called the "disqualifier". It is used + to remove the match flag for the current shape before traversing any + resource and operation bindings of the current shape. If this selector + yields any results, then the shape is not considered a match, and bound + resources and operations are not considered a match until the qualifier + selector matches again. Resource and operation binding traversal continues + regardless of if the second selector removes the match flag for the current + shape because resource and operation shapes bound to the current shape + could yield matching results. + +.. rubric:: Examples + +The following selector finds all service, resource, and operation shapes that +are marked with the ``aws.api#dataPlane`` trait or that are bound within the +containment hierarchy of resource and service shapes that are marked as such: + +.. code-block:: none + + :topdown([trait|aws.api#dataPlane]) + +The following selector finds all service, resource, and operation shapes that +are marked with the ``aws.api#dataPlane`` trait, but does not match shapes +where the ``aws.api#controlPlane`` trait is used to override the +``aws.api#dataPlane`` trait. For example, if a service is marked with the +``aws.api#dataPlane`` trait to provide a default setting for all resources and +operations within the service, the ``aws.api#controlPlane`` trait can be used +to override the default. + +.. code-block:: none + + :topdown([trait|aws.api#dataPlane], [trait|aws.api#controlPlane]) + +The above selector applied to the following model matches ``Example``, +``OperationA``, and ``OperationB``. It does not match ``Foo`` because ``Foo`` +matches the disqualifier selector. + +.. code-block:: smithy + + namespace smithy.example + + @aws.api#dataPlane + service Example { + version: "2020-09-08", + resources: [Foo], + operations: [OperationA], + } + + operation OperationA {} + + @aws.api#controlPlane + resource Foo { + operations: [OperationB] + } + + @aws.api#dataPlane + operation OperationB {} + +In the following example, the ``:topdown`` function does not inherit any +matches from service shapes because the selector only sends resource shapes +to the function. When applied to the previous example model, the following +selector matches only ``OperationB``. + +.. code-block:: none + + resource :topdown([trait|aws.api#dataPlane], [trait|aws.api#controlPlane]) + + .. _selector-variables: Variables diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java index 5bf377c4bae..b86461f994d 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java @@ -230,6 +230,13 @@ private InternalSelector parseSelectorFunction() { return new TestSelector(selectors); case "is": return IsSelector.of(selectors); + case "topdown": + if (selectors.size() > 2) { + throw new SelectorSyntaxException( + "The :topdown function accepts 1 or 2 selectors, but found " + selectors.size(), + expression(), functionPosition, line(), column()); + } + return new TopDownSelector(selectors); case "each": LOGGER.warning("The `:each` selector function has been renamed to `:is`: " + expression()); return IsSelector.of(selectors); diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/selector/TopDownSelector.java b/smithy-model/src/main/java/software/amazon/smithy/model/selector/TopDownSelector.java new file mode 100644 index 00000000000..a192540b7c8 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/selector/TopDownSelector.java @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.selector; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import software.amazon.smithy.model.neighbor.Relationship; +import software.amazon.smithy.model.neighbor.RelationshipType; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; + +final class TopDownSelector implements InternalSelector { + private final InternalSelector qualifier; + private final InternalSelector disqualifier; + + TopDownSelector(List selectors) { + this.qualifier = selectors.get(0); + disqualifier = selectors.size() > 1 ? selectors.get(1) : null; + } + + @Override + public boolean push(Context context, Shape shape, Receiver next) { + if (shape.isServiceShape() || shape.isResourceShape() || shape.isOperationShape()) { + return pushMatch(false, context, shape, next, new HashSet<>()); + } + + return true; + } + + // While a model can't contain recursive resource references, a custom + // validator might use the :topdown selector function on a model with + // recursive references. Custom validators are applied before resource + // cycles are detected, meaning this function needs to protect against + // recursion. + private boolean pushMatch(boolean qualified, Context context, Shape shape, Receiver next, Set visited) { + if (visited.contains(shape.getId())) { + return true; + } + + visited.add(shape.getId()); + + // If the flag isn't set, then check if this shape sets it to true. + if (!qualified && context.receivedShapes(shape, qualifier)) { + qualified = true; + } + + // If the flag is set, then check if any predicates unset it. + if (qualified && disqualifier != null && context.receivedShapes(shape, disqualifier)) { + qualified = false; + } + + // If the shape is matched, then it's sent to the next receiver. + if (qualified && !next.apply(context, shape)) { + return false; // fast-fail if the receiver fast-fails. + } + + // Recursively check each nested resource/operation. + for (Relationship rel : context.neighborIndex.getProvider().getNeighbors(shape)) { + if (rel.getNeighborShape().isPresent() && !rel.getNeighborShapeId().equals(shape.getId())) { + if (rel.getRelationshipType() == RelationshipType.RESOURCE + || rel.getRelationshipType() == RelationshipType.OPERATION) { + if (!pushMatch(qualified, context, rel.getNeighborShape().get(), next, visited)) { + return false; + } + } + } + } + + return true; + } +} diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/selector/TopDownSelectorTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/selector/TopDownSelectorTest.java new file mode 100644 index 00000000000..06751598351 --- /dev/null +++ b/smithy-model/src/test/java/software/amazon/smithy/model/selector/TopDownSelectorTest.java @@ -0,0 +1,104 @@ +package software.amazon.smithy.model.selector; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ShapeId; + +public class TopDownSelectorTest { + + private static Model model1; + private static Model model2; + + @BeforeAll + public static void before() { + model1 = Model.assembler().addImport(SelectorTest.class.getResource("topdown-auth.smithy")) + .assemble() + .unwrap(); + + model2 = Model.assembler().addImport(SelectorTest.class.getResource("topdown-exclusive-traits.smithy")) + .assemble() + .unwrap(); + } + + @Test + public void requiresAtLeastOneSelector() { + Assertions.assertThrows(SelectorSyntaxException.class, () -> Selector.parse(":topdown()")); + } + + @Test + public void doesNotAllowMoreThanTwoSelectors() { + Assertions.assertThrows(SelectorSyntaxException.class, () -> Selector.parse(":topdown(*, *, *)")); + } + + @Test + public void findsByAuthScheme() { + Set basic = SelectorTest.ids( + model1, ":topdown([trait|auth|(values)='smithy.api#httpBasicAuth'],\n" + + " [trait|auth]:not([trait|auth|(values)='smithy.api#httpBasicAuth']))"); + Set digest = SelectorTest.ids( + model1, ":topdown([trait|auth|(values)='smithy.api#httpDigestAuth'],\n" + + " [trait|auth]:not([trait|auth|(values)='smithy.api#httpDigestAuth']))"); + + assertThat(basic, containsInAnyOrder("smithy.example#RA", "smithy.example#ServiceWithAuthTrait", + "smithy.example#OperationWithNoAuthTrait")); + assertThat(digest, containsInAnyOrder("smithy.example#ServiceWithAuthTrait", + "smithy.example#OperationWithNoAuthTrait", + "smithy.example#RA", "smithy.example#OperationWithAuthTrait")); + } + + @Test + public void findsExclusiveTraits() { + Set a = SelectorTest.ids(model2, ":topdown([trait|smithy.example#a], [trait|smithy.example#b])"); + Set b = SelectorTest.ids(model2, ":topdown([trait|smithy.example#b], [trait|smithy.example#a])"); + + assertThat(a, containsInAnyOrder("smithy.example#Service1", "smithy.example#R1", "smithy.example#O2")); + assertThat(b, containsInAnyOrder("smithy.example#R2", "smithy.example#O1", "smithy.example#O3", + "smithy.example#O4")); + } + + @Test + public void topDownWithNoDisqualifiers() { + Set a = SelectorTest.ids(model2, ":topdown([trait|smithy.example#a])"); + + assertThat(a, containsInAnyOrder("smithy.example#Service1", "smithy.example#R1", + "smithy.example#O1", "smithy.example#O2", "smithy.example#R2", + "smithy.example#O3", "smithy.example#O4")); + } + + @Test + public void topDownWithNoDisqualifiersWithServiceVariableFollowedByFilter() { + Map matches = new HashMap<>(); + Selector.parse("service $service(*) :topdown([trait|smithy.example#a]) resource") + .runner() + .model(model2) + .selectMatches((s, vars) -> matches.put(s.getId(), vars.get("service").iterator().next().getId())); + + assertThat(matches, hasKey(ShapeId.from("smithy.example#R1"))); + assertThat(matches.get(ShapeId.from("smithy.example#R1")), equalTo(ShapeId.from("smithy.example#Service1"))); + assertThat(matches, hasKey(ShapeId.from("smithy.example#R2"))); + assertThat(matches.get(ShapeId.from("smithy.example#R2")), equalTo(ShapeId.from("smithy.example#Service1"))); + } + + @Test + public void doesNotOverflowOnBrokenResourceCycles() { + Model recursiveModel = Model.assembler() + .addImport(getClass().getResource("recursive-resources.smithy")) + .assemble() + .getResult() + .get(); // we know it's invalid. + + // The result isn't really important here. We just don't want it to + // cause a stack overflow. + Selector.parse(":topdown(*)").select(recursiveModel); + } +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/selector/recursive-resources.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/selector/recursive-resources.smithy new file mode 100644 index 00000000000..b2c1e9aaf04 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/selector/recursive-resources.smithy @@ -0,0 +1,12 @@ +// This model is broken. It's just used to test whether the topdown function +// blows up or not. + +namespace smithy.example + +resource A { + resources: [B], +} + +resource B { + resources: [A], +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/selector/topdown-auth.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/selector/topdown-auth.smithy new file mode 100644 index 00000000000..76de436caf3 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/selector/topdown-auth.smithy @@ -0,0 +1,39 @@ +namespace smithy.example + +@httpBasicAuth +@httpDigestAuth +@httpBearerAuth +service ServiceWithNoAuthTrait { + version: "2020-01-29", + operations: [ + OperationWithNoAuthTrait, + OperationWithAuthTrait + ] +} + +@httpBasicAuth +@httpDigestAuth +@httpBearerAuth +@auth([httpBasicAuth, httpDigestAuth]) +service ServiceWithAuthTrait { + version: "2020-01-29", + operations: [ + OperationWithNoAuthTrait, + OperationWithAuthTrait + ], + resources: [ + RA + ] +} + +operation OperationWithNoAuthTrait {} + +resource RA { + operations: [OperationWithNoAuthTrait2] +} + +@auth([]) +operation OperationWithNoAuthTrait2 {} + +@auth([httpDigestAuth]) +operation OperationWithAuthTrait {} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/selector/topdown-exclusive-traits.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/selector/topdown-exclusive-traits.smithy new file mode 100644 index 00000000000..52024421765 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/selector/topdown-exclusive-traits.smithy @@ -0,0 +1,34 @@ +namespace smithy.example + +@trait +structure a {} + +@trait +structure b {} + +@a +service Service1 { + version: "2020-08-22", + operations: [O1, O2], + resources: [R1] +} + +@b +operation O1 {} + +operation O2 {} + +resource R1 { + resources: [R2], + operations: [O3] +} + +@b +operation O3 {} + +@b +resource R2 { + operations: [O4] +} + +operation O4 {}