From cddb822fb2d487f46bc411bbe8577ec831b1ecb5 Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Thu, 26 Jan 2023 11:32:11 -0800 Subject: [PATCH 1/4] Copy non-service integration tests into SDK root tests directory For tests in `aws/sdk/integration-tests` that are not named after a service module, copy them into the SDK's root `tests/` directory so that they are run as part of CI on the entire SDK. --- aws/sdk/build.gradle.kts | 27 +++++++++ .../src/main/kotlin/aws/sdk/ServiceLoader.kt | 55 +++++++++++++------ 2 files changed, 66 insertions(+), 16 deletions(-) diff --git a/aws/sdk/build.gradle.kts b/aws/sdk/build.gradle.kts index a894601809..91812da6eb 100644 --- a/aws/sdk/build.gradle.kts +++ b/aws/sdk/build.gradle.kts @@ -198,6 +198,28 @@ tasks.register("relocateExamples") { outputs.dir(outputDir) } +tasks.register("relocateTests") { + description = "relocate the root integration tests and rewrite path dependencies" + doLast { + if (awsServices.rootTests.isNotEmpty()) { + copy { + val testDir = projectDir.resolve("integration-tests") + from(testDir) + awsServices.rootTests.forEach { test -> + include(test.path.toRelativeString(testDir) + "/**") + } + into(outputDir.resolve("tests")) + exclude("**/target") + filter { line -> line.replace("build/aws-sdk/sdk/", "sdk/") } + } + } + } + for (test in awsServices.rootTests) { + inputs.dir(test.path) + } + outputs.dir(outputDir) +} + tasks.register("fixExampleManifests") { description = "Adds dependency path and corrects version number of examples after relocation" enabled = awsServices.examples.isNotEmpty() @@ -287,6 +309,9 @@ tasks.register("generateCargoWorkspace") { if (awsServices.examples.isNotEmpty()) { inputs.dir(projectDir.resolve("examples")) } + for (test in awsServices.rootTests) { + inputs.dir(test.path) + } outputs.file(outputDir.resolve("Cargo.toml")) outputs.upToDateWhen { false } } @@ -310,6 +335,7 @@ tasks.register("fixManifests") { dependsOn("relocateRuntime") dependsOn("relocateAwsRuntime") dependsOn("relocateExamples") + dependsOn("relocateTests") } tasks.register("hydrateReadme") { @@ -371,6 +397,7 @@ tasks.register("finalizeSdk") { "relocateRuntime", "relocateAwsRuntime", "relocateExamples", + "relocateTests", "generateIndexMd", "fixManifests", "generateVersionManifest", diff --git a/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt b/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt index bb99ff0251..8b3a8db932 100644 --- a/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt @@ -14,6 +14,11 @@ import software.amazon.smithy.model.traits.TitleTrait import java.io.File import kotlin.streams.toList +data class RootTest( + val path: File, + val manifestName: String, +) + class AwsServices( private val project: Project, services: List, @@ -23,37 +28,55 @@ class AwsServices( val services: List val moduleNames: Set by lazy { services.map { it.module }.toSortedSet() } + init { + this.services = services.sortedBy { it.module } + } + val allModules: Set by lazy { ( services.map(AwsService::module).map { "sdk/$it" } + CrateSet.AWS_SDK_SMITHY_RUNTIME.map { "sdk/$it" } + CrateSet.AWS_SDK_RUNTIME.map { "sdk/$it" } + - examples + examples + + rootTests.map(RootTest::manifestName) ).toSortedSet() } val examples: List by lazy { project.projectDir.resolve("examples") .listFiles { file -> !file.name.startsWith(".") }.orEmpty().toList() - .filter { file -> - val cargoToml = File(file, "Cargo.toml") - if (cargoToml.exists()) { - val usedModules = cargoToml.readLines() - .map { line -> line.substringBefore('=').trim() } - .filter { line -> line.startsWith("aws-sdk-") } - .map { line -> line.substringAfter("aws-sdk-") } - .toSet() - moduleNames.containsAll(usedModules) - } else { - false - } - } + .filter { file -> manifestCompatibleWithGeneratedServices(file) } .map { "examples/${it.name}" } } - init { - this.services = services.sortedBy { it.module } + /** + * Tests in `aws/sdk/integration-tests` that are not named after a service module, and therefore, + * are not included in a service's `tests/` directory. These are to be included at the SDK root + * `tests/` directory for inclusion in CI. + */ + val rootTests: List by lazy { + project.projectDir.resolve("integration-tests") + .listFiles { file -> !file.name.startsWith(".") }.orEmpty().toList() + .filter { file -> !moduleNames.contains(file.name) && manifestCompatibleWithGeneratedServices(file) } + .map { file -> RootTest(file, "tests/${file.name}") } } + + /** + * Returns true if the Cargo manifest in the given path is compatible with the set of generated services. + */ + private fun manifestCompatibleWithGeneratedServices(path: File) = + File(path, "Cargo.toml").let { cargoToml -> + if (cargoToml.exists()) { + val usedModules = cargoToml.readLines() + .map { line -> line.substringBefore('=').trim() } + .filter { line -> line.startsWith("aws-sdk-") } + .map { line -> line.substringAfter("aws-sdk-") } + .toSet() + moduleNames.containsAll(usedModules) + } else { + false + } + } } /** From fb70d12db2967a4b79f8fe3515441e9199b1edf2 Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Thu, 26 Jan 2023 13:48:50 -0800 Subject: [PATCH 2/4] Add missing fields to integration test manifests --- aws/sdk/integration-tests/dynamodb/Cargo.toml | 3 +++ aws/sdk/integration-tests/ec2/Cargo.toml | 3 +++ aws/sdk/integration-tests/glacier/Cargo.toml | 3 +++ aws/sdk/integration-tests/iam/Cargo.toml | 3 +++ aws/sdk/integration-tests/kms/Cargo.toml | 3 +++ aws/sdk/integration-tests/lambda/Cargo.toml | 3 +++ aws/sdk/integration-tests/no-default-features/Cargo.toml | 4 +++- aws/sdk/integration-tests/polly/Cargo.toml | 3 +++ aws/sdk/integration-tests/qldbsession/Cargo.toml | 3 +++ aws/sdk/integration-tests/s3/Cargo.toml | 3 +++ aws/sdk/integration-tests/s3control/Cargo.toml | 3 +++ aws/sdk/integration-tests/sts/Cargo.toml | 3 +++ aws/sdk/integration-tests/transcribestreaming/Cargo.toml | 4 ++++ 13 files changed, 40 insertions(+), 1 deletion(-) diff --git a/aws/sdk/integration-tests/dynamodb/Cargo.toml b/aws/sdk/integration-tests/dynamodb/Cargo.toml index da55395838..bf47eddcca 100644 --- a/aws/sdk/integration-tests/dynamodb/Cargo.toml +++ b/aws/sdk/integration-tests/dynamodb/Cargo.toml @@ -4,6 +4,9 @@ name = "dynamo-tests" version = "0.1.0" authors = ["AWS Rust SDK Team ", "Russell Cohen "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/ec2/Cargo.toml b/aws/sdk/integration-tests/ec2/Cargo.toml index ee6c83b094..d7580d6b89 100644 --- a/aws/sdk/integration-tests/ec2/Cargo.toml +++ b/aws/sdk/integration-tests/ec2/Cargo.toml @@ -2,6 +2,9 @@ name = "ec2-tests" version = "0.1.0" edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false [dev-dependencies] aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } diff --git a/aws/sdk/integration-tests/glacier/Cargo.toml b/aws/sdk/integration-tests/glacier/Cargo.toml index ea1d13b695..88e20751d2 100644 --- a/aws/sdk/integration-tests/glacier/Cargo.toml +++ b/aws/sdk/integration-tests/glacier/Cargo.toml @@ -4,6 +4,9 @@ name = "glacier-tests" version = "0.1.0" authors = ["AWS Rust SDK Team "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/iam/Cargo.toml b/aws/sdk/integration-tests/iam/Cargo.toml index b9061aa85f..ea3ff36ea0 100644 --- a/aws/sdk/integration-tests/iam/Cargo.toml +++ b/aws/sdk/integration-tests/iam/Cargo.toml @@ -4,6 +4,9 @@ name = "iam-tests" version = "0.1.0" authors = ["AWS Rust SDK Team "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/kms/Cargo.toml b/aws/sdk/integration-tests/kms/Cargo.toml index 4e6611d4a4..bfe5ed5cd9 100644 --- a/aws/sdk/integration-tests/kms/Cargo.toml +++ b/aws/sdk/integration-tests/kms/Cargo.toml @@ -4,6 +4,9 @@ name = "kms-tests" version = "0.1.0" authors = ["Russell Cohen "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/lambda/Cargo.toml b/aws/sdk/integration-tests/lambda/Cargo.toml index 830e3eaf75..5492fbc68a 100644 --- a/aws/sdk/integration-tests/lambda/Cargo.toml +++ b/aws/sdk/integration-tests/lambda/Cargo.toml @@ -3,6 +3,9 @@ name = "lambda" version = "0.1.0" authors = ["AWS Rust SDK Team ", "Zelda Hessler "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false [dev-dependencies] async-stream = "0.3.0" diff --git a/aws/sdk/integration-tests/no-default-features/Cargo.toml b/aws/sdk/integration-tests/no-default-features/Cargo.toml index 45980e2b90..bb0ba9d02d 100644 --- a/aws/sdk/integration-tests/no-default-features/Cargo.toml +++ b/aws/sdk/integration-tests/no-default-features/Cargo.toml @@ -1,4 +1,3 @@ -# This Cargo.toml is unused in generated code. It exists solely to enable these tests to compile in-situ [package] name = "no-default-features" version = "0.1.0" @@ -8,6 +7,9 @@ These tests ensure that things will fail (or not fail) as expected when default features are disabled for all SDK and runtime crates. """ edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/polly/Cargo.toml b/aws/sdk/integration-tests/polly/Cargo.toml index c5ea81ebec..b7b1a0b6f8 100644 --- a/aws/sdk/integration-tests/polly/Cargo.toml +++ b/aws/sdk/integration-tests/polly/Cargo.toml @@ -4,6 +4,9 @@ name = "polly-tests" version = "0.1.0" authors = ["John DiSanti "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/qldbsession/Cargo.toml b/aws/sdk/integration-tests/qldbsession/Cargo.toml index fe28618f59..f6710cb575 100644 --- a/aws/sdk/integration-tests/qldbsession/Cargo.toml +++ b/aws/sdk/integration-tests/qldbsession/Cargo.toml @@ -4,6 +4,9 @@ name = "qldb-tests" version = "0.1.0" authors = ["Russell Cohen ", "Shing Lyu "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/s3control/Cargo.toml b/aws/sdk/integration-tests/s3control/Cargo.toml index 53b4c48c73..ada6d95b41 100644 --- a/aws/sdk/integration-tests/s3control/Cargo.toml +++ b/aws/sdk/integration-tests/s3control/Cargo.toml @@ -4,6 +4,9 @@ name = "s3control" version = "0.1.0" authors = ["AWS Rust SDK Team "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/sts/Cargo.toml b/aws/sdk/integration-tests/sts/Cargo.toml index 874ce80c89..63c91c4639 100644 --- a/aws/sdk/integration-tests/sts/Cargo.toml +++ b/aws/sdk/integration-tests/sts/Cargo.toml @@ -4,6 +4,9 @@ name = "sts-tests" version = "0.1.0" authors = ["Russell Cohen "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/transcribestreaming/Cargo.toml b/aws/sdk/integration-tests/transcribestreaming/Cargo.toml index b398e3554d..1c19e73f9f 100644 --- a/aws/sdk/integration-tests/transcribestreaming/Cargo.toml +++ b/aws/sdk/integration-tests/transcribestreaming/Cargo.toml @@ -1,8 +1,12 @@ +# This Cargo.toml is unused in generated code. It exists solely to enable these tests to compile in-situ [package] name = "transcribestreaming" version = "0.1.0" authors = ["AWS Rust SDK Team ", "John DiSanti "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false [dev-dependencies] async-stream = "0.3.0" From 2c56a057b229c4bdf6465c4d8238c72e440bd97e Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Thu, 26 Jan 2023 14:00:08 -0800 Subject: [PATCH 3/4] Remove tests from root workspace --- buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt | 5 +++-- tools/ci-build/scripts/check-aws-sdk-services | 7 +++++++ tools/ci-build/scripts/check-only-aws-sdk-services | 7 +++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt b/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt index 8b3a8db932..fdc6448b0c 100644 --- a/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt @@ -37,8 +37,9 @@ class AwsServices( services.map(AwsService::module).map { "sdk/$it" } + CrateSet.AWS_SDK_SMITHY_RUNTIME.map { "sdk/$it" } + CrateSet.AWS_SDK_RUNTIME.map { "sdk/$it" } + - examples + - rootTests.map(RootTest::manifestName) + examples + // Root tests should not be included since they can't be part of the root Cargo workspace + // in order to test differences in Cargo features. ).toSortedSet() } diff --git a/tools/ci-build/scripts/check-aws-sdk-services b/tools/ci-build/scripts/check-aws-sdk-services index 9d25ba0e29..a440c98f83 100755 --- a/tools/ci-build/scripts/check-aws-sdk-services +++ b/tools/ci-build/scripts/check-aws-sdk-services @@ -11,3 +11,10 @@ cd aws-sdk sed -i '/"examples\//d' Cargo.toml cargo test --all-features + +for test_dir in tests/*; do + if [ -f "${test_dir}/Cargo.toml" ]; then + echo "#### Testing ${test_dir}..." + cargo test --all-features --manifest-path "${test_dir}/Cargo.toml" + fi +done diff --git a/tools/ci-build/scripts/check-only-aws-sdk-services b/tools/ci-build/scripts/check-only-aws-sdk-services index 494f47f13c..27c93ea479 100755 --- a/tools/ci-build/scripts/check-only-aws-sdk-services +++ b/tools/ci-build/scripts/check-only-aws-sdk-services @@ -13,3 +13,10 @@ cd aws-sdk sed -i '/"examples\//d' Cargo.toml cargo check --all-targets --all-features + +for test_dir in tests/*; do + if [ -f "${test_dir}/Cargo.toml" ]; then + echo "#### Checking ${test_dir}..." + cargo check --all-targets --all-features --manifest-path "${test_dir}/Cargo.toml" + fi +done From 89a6ce08858eafdfa09bfa2622206ceec85f4497 Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Thu, 26 Jan 2023 17:00:24 -0800 Subject: [PATCH 4/4] Explicitly exclude the root tests from the root workspace --- aws/sdk/build.gradle.kts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aws/sdk/build.gradle.kts b/aws/sdk/build.gradle.kts index 91812da6eb..31a407964b 100644 --- a/aws/sdk/build.gradle.kts +++ b/aws/sdk/build.gradle.kts @@ -5,6 +5,7 @@ import aws.sdk.AwsServices import aws.sdk.Membership +import aws.sdk.RootTest import aws.sdk.discoverServices import aws.sdk.docsLandingPage import aws.sdk.parseMembership @@ -294,6 +295,8 @@ tasks.register("relocateChangelog") { fun generateCargoWorkspace(services: AwsServices): String { return """ |[workspace] + |exclude = [${"\n"}${services.rootTests.map(RootTest::manifestName).joinToString(",\n") { "| \"$it\"" }} + |] |members = [${"\n"}${services.allModules.joinToString(",\n") { "| \"$it\"" }} |] """.trimMargin()