From c01f22b1637e9d2767cadfa58f5e31fa3f0829a4 Mon Sep 17 00:00:00 2001 From: "Wilkins, Emily (Counterpointe Solutions)" <80470879+ewilkins-csi@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:06:15 -0500 Subject: [PATCH] [#442] account for lists in `YamlUtils#loadYaml` Also fixes a bug with the `hasDouble`, `hasInt`, and `hasBoolean` methods of `YamlObject`. Adds tests for `YamlUtils` including one to catch the root list issue. Finally, also updates the POM migration to account for POMs without dependcy management sections. --- .../v1_10_0/DeltaSparkPomMigration.java | 12 ++- .../aissemble/upgrade/util/YamlUtils.java | 31 ++++-- .../src/main/resources/migrations.json | 3 +- .../migration/utils/YamlUtilsSteps.java | 96 +++++++++++++++++++ .../v1_10_0/DeltaSparkMigration.java | 5 + .../v1_10_0/delta-spark-migration.feature | 5 + .../specifications/yaml-utils.feature | 35 +++++++ .../migration/no-mgmt.xml | 22 +++++ .../validation/no-mgmt.xml | 23 +++++ 9 files changed, 221 insertions(+), 11 deletions(-) create mode 100644 foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/utils/YamlUtilsSteps.java create mode 100644 foundation/foundation-upgrade/src/test/resources/specifications/yaml-utils.feature create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/DeltaSparkPomMigration/migration/no-mgmt.xml create mode 100644 foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/DeltaSparkPomMigration/validation/no-mgmt.xml diff --git a/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/DeltaSparkPomMigration.java b/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/DeltaSparkPomMigration.java index 02cce25a8..2822c3bc1 100644 --- a/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/DeltaSparkPomMigration.java +++ b/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/DeltaSparkPomMigration.java @@ -13,6 +13,7 @@ import com.boozallen.aissemble.upgrade.migration.AbstractAissembleMigration; import org.apache.commons.lang3.StringUtils; import org.apache.maven.model.Dependency; +import org.apache.maven.model.DependencyManagement; import org.apache.maven.model.InputLocation; import org.apache.maven.model.InputLocationTracker; import org.apache.maven.model.Model; @@ -86,10 +87,13 @@ private List getDeltaCoreDependenciesForProject(Model model) { private List getDeltaCoreDependencies(ModelBase model) { List deltaCoreDependencies = new ArrayList<>(); - model.getDependencyManagement().getDependencies() - .stream() - .filter(DeltaSparkPomMigration::isDeltaCore) - .forEach(deltaCoreDependencies::add); + DependencyManagement dependencyManagement = model.getDependencyManagement(); + if (dependencyManagement != null) { + dependencyManagement.getDependencies() + .stream() + .filter(DeltaSparkPomMigration::isDeltaCore) + .forEach(deltaCoreDependencies::add); + } model.getDependencies() .stream() .filter(DeltaSparkPomMigration::isDeltaCore) diff --git a/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/util/YamlUtils.java b/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/util/YamlUtils.java index 1913a306b..ca11c2c04 100644 --- a/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/util/YamlUtils.java +++ b/foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/util/YamlUtils.java @@ -32,6 +32,7 @@ import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.error.YAMLException; import org.yaml.snakeyaml.introspector.Property; import org.yaml.snakeyaml.introspector.PropertyUtils; import org.yaml.snakeyaml.representer.Representer; @@ -40,13 +41,31 @@ public class YamlUtils { private static final String SPACE = " "; private static final int TAB = 2; + /** + * Reads the YAML file into a {@link YamlObject}. If the file contains a list at the root level, then the returned + * YamlObject will have one entry under the key "items" that is the resulting list. Multiple objects separated by + * dashes is not supported. + * + * @param file a YAML file to parse + * @return the {@link YamlObject} represented by the file + * @throws IOException if the file cannot be read + * @throws YAMLException if the file fails to parse to a valid YamlObject + */ public static YamlObject loadYaml(File file) throws IOException { Yaml yaml = new Yaml(); try (InputStream fileStream = Files.asByteSource(file).openStream()) { - Map contents = yaml.load(fileStream); - //If the file is empty or only contains comments, contents will be null - if (contents == null) { + Object data = yaml.load(fileStream); + Map contents; + //If the file is empty or only contains comments, data will be null + if (data == null) { contents = Map.of(); + } else if (data instanceof Map) { + //noinspection unchecked + contents = (Map) data; + } else if (data instanceof List) { + contents = Map.of("items", data); + } else { + throw new YAMLException("Unrecognized root data type [" + data.getClass().getName() + "] in file: " + file.getAbsolutePath()); } return new YamlObject(contents); } @@ -78,7 +97,7 @@ public String getString(String... path) { } public boolean hasInt(String... path) { - return hasValue(int.class, path); + return hasValue(Integer.class, path); } public int getInt(String... path) { @@ -86,7 +105,7 @@ public int getInt(String... path) { } public boolean hasDouble(String... path) { - return hasValue(double.class, path); + return hasValue(Double.class, path); } public double getDouble(String... path) { @@ -94,7 +113,7 @@ public double getDouble(String... path) { } public boolean hasBoolean(String... path) { - return hasValue(boolean.class, path); + return hasValue(Boolean.class, path); } public boolean getBoolean(String... path) { diff --git a/foundation/foundation-upgrade/src/main/resources/migrations.json b/foundation/foundation-upgrade/src/main/resources/migrations.json index d5e95776c..4dce7ca84 100644 --- a/foundation/foundation-upgrade/src/main/resources/migrations.json +++ b/foundation/foundation-upgrade/src/main/resources/migrations.json @@ -73,7 +73,8 @@ "implementation": "com.boozallen.aissemble.upgrade.migration.v1_10_0.DeltaSparkYamlMigration", "fileSets": [ { - "includes": ["**/*.yaml"] + "includes": ["**/*.yaml"], + "excludes": ["**/target/**", "**/templates/**"] } ] } diff --git a/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/utils/YamlUtilsSteps.java b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/utils/YamlUtilsSteps.java new file mode 100644 index 000000000..b277819c1 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/utils/YamlUtilsSteps.java @@ -0,0 +1,96 @@ +package com.boozallen.aissemble.upgrade.migration.utils; + +/*- + * #%L + * aiSSEMBLE::Foundation::Upgrade + * %% + * Copyright (C) 2021 Booz Allen + * %% + * This software package is licensed under the Booz Allen Public License. All Rights Reserved. + * #L% + */ + +import com.boozallen.aissemble.upgrade.util.YamlUtils; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.When; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class YamlUtilsSteps { + + private Path testFile; + private YamlUtils.YamlObject yaml; + + @Given("a YAML file:") + public void aYamlFile(String contents) throws IOException { + testFile = Files.createTempFile(Path.of("target"), "yaml-util-test", ""); + Files.writeString(testFile, contents); + } + + @When("the file is loaded") + public void theFileIsLoaded() throws IOException { + yaml = YamlUtils.loadYaml(testFile.toFile()); + } + + @Then("the string value of the property {string} is {string}") + public void theStringValueOfThePropertyIs(String name, String value) { + String type = yaml.get(name).getClass().getSimpleName(); + assertTrue("Property was not loaded as string: " + name + " (was " + type + ")", yaml.hasString(name)); + assertEquals("Property value does not match for: " + name, value, yaml.getString(name)); + } + + @Then("the decimal value of the property {string} is {double}") + public void theDecimalValueOfThePropertyIs(String name, double value) { + String type = yaml.get(name).getClass().getSimpleName(); + assertTrue("Property was not loaded as double: " + name + " (was " + type + ")", yaml.hasDouble(name)); + assertEquals("Property value does not match for: " + name, value, yaml.getDouble(name), 0.01); + } + + @Then("the integer value of the property {string} is {int}") + public void theIntegerValueOfThePropertyIs(String name, int value) { + String type = yaml.get(name).getClass().getSimpleName(); + assertTrue("Property was not loaded as int: " + name + " (was " + type + ")", yaml.hasInt(name)); + assertEquals("Property value does not match for: " + name, value, yaml.getInt(name)); + } + + @Then("the boolean value of the property {string} is {string}") + public void theBooleanValueOfThePropertyIs(String name, String value) { + String type = yaml.get(name).getClass().getSimpleName(); + assertTrue("Property was not loaded as boolean: " + name + " (was " + type + ")", yaml.hasBoolean(name)); + assertEquals("Property value does not match for: " + name, Boolean.valueOf(value), yaml.getBoolean(name)); + } + + @Then("the property {string} is an object") + public void thePropertyIsAnObject(String name) { + String type = yaml.get(name).getClass().getSimpleName(); + assertTrue("Property was not loaded as object: " + name + " (was " + type + ")", yaml.hasObject(name)); + } + + @Then("the decimal value of the property {string} of the Object {string} is {double}") + public void theDecimalValueOfThePropertyOfTheObjectIs(String prop, String obj, double value) { + assertTrue("Property was not loaded as double: " + obj + "." + prop, yaml.hasDouble(obj, prop)); + assertEquals("Property value does not match for: " + obj + "." + prop, value, yaml.getDouble(obj, prop), 0.01); + } + + @Then("the size of the list {string} is {int}") + public void theSizeOfTheListIs(String name, int size) { + String type = yaml.get(name).getClass().getSimpleName(); + assertTrue("Property was not loaded as list: " + name + " (was " + type + ")", yaml.hasList(name)); + assertEquals("Property value does not match for: " + name, size, yaml.getList(name).size()); + } + + @Then("the {string} of item {int} of the list {string} is {int}") + public void theIntegerValueOfItemOfTheListIs(String prop, int index, String list, int value) { + List objects = yaml.getListOfObjects(list); + YamlUtils.YamlObject object = objects.get(index); + String key = list + "[" + index + "]." + prop; + assertEquals("Property value does not match for: " + key, value, object.getInt(prop)); + } +} diff --git a/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/DeltaSparkMigration.java b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/DeltaSparkMigration.java index 4c968b6a3..898e0464f 100644 --- a/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/DeltaSparkMigration.java +++ b/foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/DeltaSparkMigration.java @@ -31,6 +31,11 @@ public void aPOMThatReferencesTheDeltaCorePackage() { testFile = getTestFile("v1_10_0/DeltaSparkPomMigration/migration/pom.xml"); } + @Given("a POM that references the delta-core package that does not have dependencyManagement") + public void aPOMThatReferencesTheDeltaCorePackageThatDoesNotHaveDependencyManagement() { + testFile = getTestFile("v1_10_0/DeltaSparkPomMigration/migration/no-mgmt.xml"); + } + @When("the 1.10.0 DeltaSpark yaml migration executes") public void theDeltaSparkYamlMigrationExecutes() { DeltaSparkYamlMigration migration = new DeltaSparkYamlMigration(); diff --git a/foundation/foundation-upgrade/src/test/resources/specifications/v1_10_0/delta-spark-migration.feature b/foundation/foundation-upgrade/src/test/resources/specifications/v1_10_0/delta-spark-migration.feature index a2c696aa4..503d176fb 100644 --- a/foundation/foundation-upgrade/src/test/resources/specifications/v1_10_0/delta-spark-migration.feature +++ b/foundation/foundation-upgrade/src/test/resources/specifications/v1_10_0/delta-spark-migration.feature @@ -17,3 +17,8 @@ Feature: Migrate Delta Lake Spark dependencies Given a POM that references the delta-core package When the 1.10.0 DeltaSpark POM migration executes Then delta-core is updated to delta-spark and the version is set to the version.delta property + + Scenario: POM dependencies are migrated without dependency management + Given a POM that references the delta-core package that does not have dependencyManagement + When the 1.10.0 DeltaSpark POM migration executes + Then delta-core is updated to delta-spark and the version is set to the version.delta property diff --git a/foundation/foundation-upgrade/src/test/resources/specifications/yaml-utils.feature b/foundation/foundation-upgrade/src/test/resources/specifications/yaml-utils.feature new file mode 100644 index 000000000..535e29e48 --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/specifications/yaml-utils.feature @@ -0,0 +1,35 @@ +Feature: Utilities -> YAML + + Scenario: I can load a YAML file with an object + Given a YAML file: + """yaml + name: test + size: 7.5 + count: 12 + complete: True + nested: { + size: 2.0 + } + multiple: + - intvalue: 5 + - intvalue: 7 + """ + When the file is loaded + Then the string value of the property "name" is "test" + And the decimal value of the property "size" is 7.5 + And the integer value of the property "count" is 12 + And the boolean value of the property "complete" is "true" + And the property "nested" is an object + And the decimal value of the property "size" of the Object "nested" is 2.0 + And the size of the list "multiple" is 2 + And the "intvalue" of item 1 of the list "multiple" is 7 + + Scenario: I can load a YAML file with a list + Given a YAML file: + """yaml + - red + - blue + - yellow + """ + When the file is loaded + Then the size of the list "items" is 3 \ No newline at end of file diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/DeltaSparkPomMigration/migration/no-mgmt.xml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/DeltaSparkPomMigration/migration/no-mgmt.xml new file mode 100644 index 000000000..43da58daf --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/DeltaSparkPomMigration/migration/no-mgmt.xml @@ -0,0 +1,22 @@ + + + 4.0.0 + org.test + test-pom + 1.0.0-SNAPSHOT + + + + io.delta + delta-core_2.13 + + + diff --git a/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/DeltaSparkPomMigration/validation/no-mgmt.xml b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/DeltaSparkPomMigration/validation/no-mgmt.xml new file mode 100644 index 000000000..1f4d63e5b --- /dev/null +++ b/foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/DeltaSparkPomMigration/validation/no-mgmt.xml @@ -0,0 +1,23 @@ + + + 4.0.0 + org.test + test-pom + 1.0.0-SNAPSHOT + + + + io.delta + delta-spark_2.13 + ${version.delta} + + +