From 92d02922c23e9445c438b69017634415e05d2d98 Mon Sep 17 00:00:00 2001 From: Mridula <66699525+mpeddada1@users.noreply.github.com> Date: Wed, 25 May 2022 19:31:40 -0400 Subject: [PATCH] fix: add native image configurations for Spanner classes (#1858) * fix: add native image configurations for Spanner classes --- .github/workflows/ci.yaml | 14 ++- .../integration-tests-against-emulator.yaml | 21 +++- .github/workflows/samples.yaml | 27 +++- .kokoro/build.sh | 18 ++- .kokoro/common.sh | 7 +- .kokoro/dependencies.sh | 8 ++ google-cloud-spanner/pom.xml | 35 +++--- .../spanner/nativeimage/SpannerFeature.java | 116 ++++++++++++++++++ .../native-image/native-image.properties | 4 + .../native-image/resource-config.json | 8 ++ ...lientTest.java => SessionClientTests.java} | 2 +- ...DdlClientTest.java => DdlClientTests.java} | 2 +- .../spanner/connection/SqlScriptVerifier.java | 21 ++-- owlbot.py | 2 + 14 files changed, 242 insertions(+), 43 deletions(-) create mode 100644 google-cloud-spanner/src/main/java/com/google/cloud/spanner/nativeimage/SpannerFeature.java create mode 100644 google-cloud-spanner/src/main/resources/META-INF/native-image/native-image.properties create mode 100644 google-cloud-spanner/src/main/resources/META-INF/native-image/resource-config.json rename google-cloud-spanner/src/test/java/com/google/cloud/spanner/{SessionClientTest.java => SessionClientTests.java} (99%) rename google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/{DdlClientTest.java => DdlClientTests.java} (99%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0ef324c71e3..857028b88b8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -83,10 +83,22 @@ jobs: java: [8, 11, 17] steps: - uses: actions/checkout@v3 + # For Java 8 tests, use JDK 11 to compile + - if: ${{matrix.java}} == '8' + uses: actions/setup-java@v3 + with: + java-version: 11 + distribution: zulu + - if: ${{matrix.java}} == '8' + run: echo "JAVA11_HOME=${JAVA_HOME}" >> $GITHUB_ENV + shell: bash - uses: actions/setup-java@v3 with: distribution: zulu java-version: ${{matrix.java}} + - if: ${{matrix.java}} == '8' + run: echo "JAVA8_HOME=${JAVA_HOME}" >> $GITHUB_ENV + shell: bash - run: java -version - run: .kokoro/dependencies.sh lint: @@ -108,7 +120,7 @@ jobs: - uses: actions/setup-java@v3 with: distribution: zulu - java-version: 8 + java-version: 11 - run: java -version - run: .kokoro/build.sh env: diff --git a/.github/workflows/integration-tests-against-emulator.yaml b/.github/workflows/integration-tests-against-emulator.yaml index 7d82960228a..da05e8d3d75 100644 --- a/.github/workflows/integration-tests-against-emulator.yaml +++ b/.github/workflows/integration-tests-against-emulator.yaml @@ -20,12 +20,27 @@ jobs: - uses: stCarolas/setup-maven@v4 with: maven-version: 3.8.1 - - uses: actions/setup-java@v1 + # Build with JDK 11 and run tests with JDK 8 + - uses: actions/setup-java@v3 + with: + java-version: 11 + distribution: zulu + - run: echo "JAVA11_HOME=${JAVA_HOME}" >> $GITHUB_ENV + shell: bash + - uses: actions/setup-java@v3 with: java-version: 8 + distribution: zulu + - run: echo "JAVA8_HOME=${JAVA_HOME}" >> $GITHUB_ENV + shell: bash - run: java -version - - run: .kokoro/build.sh - - run: mvn -B -Dspanner.testenv.instance="" -Penable-integration-tests -DtrimStackTrace=false -Dclirr.skip=true -Denforcer.skip=true -fae verify + - name: Compiling main library + run: .kokoro/build.sh + - name: Running tests + run: | + mvn -B -Dspanner.testenv.instance="" -Penable-integration-tests \ + -DtrimStackTrace=false -Dclirr.skip=true -Denforcer.skip=true \ + -Dmaven.main.skip=true -fae verify env: JOB_TYPE: test SPANNER_EMULATOR_HOST: localhost:9010 diff --git a/.github/workflows/samples.yaml b/.github/workflows/samples.yaml index ab983e5d492..d0eb0cb5def 100644 --- a/.github/workflows/samples.yaml +++ b/.github/workflows/samples.yaml @@ -12,18 +12,39 @@ jobs: - name: Run checkstyle run: mvn -P lint --quiet --batch-mode checkstyle:check working-directory: samples/snippets + compile-java8: + name: "compile (8)" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v3 + with: + # Java 11 to generate class file targeting Java 8 + java-version: 11 + distribution: zulu + - name: Compile Spanner + run: mvn clean install + - uses: actions/setup-java@v3 + with: + java-version: 8 + distribution: zulu + - name: Compile samples + run: mvn compile + working-directory: samples + compile: runs-on: ubuntu-latest strategy: matrix: - java: [8, 11] + java: [11, 17] steps: - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: java-version: ${{matrix.java}} + distribution: zulu - name: Compile Spanner run: mvn clean install - name: Compile samples run: mvn compile - working-directory: samples + working-directory: samples \ No newline at end of file diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 1b8cfc5620f..750b4915792 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -23,11 +23,6 @@ cd ${scriptDir}/.. # include common functions source ${scriptDir}/common.sh -function setJava() { - export JAVA_HOME=$1 - export PATH=${JAVA_HOME}/bin:$PATH -} - # units-java8 uses both JDK 11 and JDK 8. GraalVM dependencies require JDK 11 to # compile the classes touching GraalVM classes. if [ ! -z "${JAVA11_HOME}" ]; then @@ -57,7 +52,6 @@ fi # are compatible with Java 8 when running tests. if [ ! -z "${JAVA8_HOME}" ]; then setJava "${JAVA8_HOME}" - mvn -version fi RETURN_CODE=0 @@ -87,10 +81,21 @@ integration) -DtrimStackTrace=false \ -Dclirr.skip=true \ -Denforcer.skip=true \ + -Dmaven.main.skip=true \ -fae \ verify RETURN_CODE=$? ;; +graalvm) + # Run Unit and Integration Tests with Native Image + mvn test -Pnative -Penable-integration-tests + RETURN_CODE=$? + ;; +graalvm17) + # Run Unit and Integration Tests with Native Image + mvn test -Pnative -Penable-integration-tests + RETURN_CODE=$? + ;; slowtests) mvn -B ${INTEGRATION_TEST_ARGS} \ -ntp \ @@ -100,6 +105,7 @@ slowtests) -DtrimStackTrace=false \ -Dclirr.skip=true \ -Denforcer.skip=true \ + -Dmaven.main.skip=true \ -fae \ verify RETURN_CODE=$? diff --git a/.kokoro/common.sh b/.kokoro/common.sh index ace89f45a9d..6c8089ffddd 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -55,4 +55,9 @@ function retry_with_backoff { ## Helper functionss function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; } function msg() { println "$*" >&2; } -function println() { printf '%s\n' "$(now) $*"; } \ No newline at end of file +function println() { printf '%s\n' "$(now) $*"; } + +function setJava() { + export JAVA_HOME=$1 + export PATH=${JAVA_HOME}/bin:$PATH +} diff --git a/.kokoro/dependencies.sh b/.kokoro/dependencies.sh index d7476cfe972..6a9f719a5e6 100755 --- a/.kokoro/dependencies.sh +++ b/.kokoro/dependencies.sh @@ -49,6 +49,10 @@ function determineMavenOpts() { export MAVEN_OPTS=$(determineMavenOpts) +if [ ! -z "${JAVA11_HOME}" ]; then + setJava "${JAVA11_HOME}" +fi + # this should run maven enforcer retry_with_backoff 3 10 \ mvn install -B -V -ntp \ @@ -56,6 +60,10 @@ retry_with_backoff 3 10 \ -Dmaven.javadoc.skip=true \ -Dclirr.skip=true +if [ ! -z "${JAVA8_HOME}" ]; then + setJava "${JAVA8_HOME}" +fi + mvn -B dependency:analyze -DfailOnWarning=true echo "****************** DEPENDENCY LIST COMPLETENESS CHECK *******************" diff --git a/google-cloud-spanner/pom.xml b/google-cloud-spanner/pom.xml index 14efa78ecfe..63cde83b3ad 100644 --- a/google-cloud-spanner/pom.xml +++ b/google-cloud-spanner/pom.xml @@ -16,6 +16,7 @@ google-cloud-spanner 0.31.1 + 22.1.0 com.google.cloud.spanner.GceTestEnvConfig projects/gcloud-devel/instances/spanner-testing-east1 gcloud-devel @@ -56,6 +57,16 @@ default-test com.google.cloud.spanner.TracerTest,com.google.cloud.spanner.IntegrationTest + + + + ${spanner.testenv.config.class} + ${spanner.testenv.instance} + ${spanner.gce.config.project_id} + ${spanner.testenv.kms_key.name} + + @@ -105,20 +116,6 @@ - - - org.apache.maven.plugins - maven-surefire-plugin - - - ${spanner.testenv.config.class} - ${spanner.testenv.instance} - ${spanner.gce.config.project_id} - ${spanner.testenv.kms_key.name} - - 3000 - - org.graalvm.buildtools native-maven-plugin @@ -142,7 +139,7 @@ org.apache.maven.plugins maven-dependency-plugin - io.grpc:grpc-protobuf-lite,org.hamcrest:hamcrest,org.hamcrest:hamcrest-core,com.google.errorprone:error_prone_annotations,org.openjdk.jmh:jmh-generator-annprocess,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,javax.annotation:javax.annotation-api,io.opencensus:opencensus-impl + io.grpc:grpc-protobuf-lite,org.hamcrest:hamcrest,org.hamcrest:hamcrest-core,com.google.errorprone:error_prone_annotations,org.openjdk.jmh:jmh-generator-annprocess,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,javax.annotation:javax.annotation-api,io.opencensus:opencensus-impl,org.graalvm.sdk:graal-sdk @@ -296,6 +293,13 @@ grpc-alts + + org.graalvm.sdk + graal-sdk + ${graalvm.version} + provided + + junit @@ -352,6 +356,7 @@ 1.35 test + diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/nativeimage/SpannerFeature.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/nativeimage/SpannerFeature.java new file mode 100644 index 00000000000..38007ddc073 --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/nativeimage/SpannerFeature.java @@ -0,0 +1,116 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.nativeimage; + +import com.google.api.gax.nativeimage.NativeImageUtils; +import org.graalvm.nativeimage.hosted.Feature; + +/** Registers Spanner library classes for reflection. */ +final class SpannerFeature implements Feature { + + private static final String SPANNER_CLASS = "com.google.spanner.v1.SpannerGrpc"; + private static final String SPANNER_TEST_CLASS = "com.google.cloud.spanner.GceTestEnvConfig"; + private static final String MOCK_CLASS = "com.google.cloud.spanner.MockDatabaseAdminServiceImpl"; + private static final String CLIENT_SIDE_IMPL_CLASS = + "com.google.cloud.spanner.connection.ClientSideStatementImpl"; + private static final String CLIENT_SIDE_VALUE_CONVERTER = + "com.google.cloud.spanner.connection.ClientSideStatementValueConverters"; + private static final String CONNECTION_IMPL = + "com.google.cloud.spanner.connection.ConnectionImpl"; + private static final String CLIENT_SIDE_STATEMENTS = + "com.google.cloud.spanner.connection.ClientSideStatements"; + private static final String CONNECTION_STATEMENT_EXECUTOR = + "com.google.cloud.spanner.connection.ConnectionStatementExecutor"; + private static final String CLIENT_SIDE_STATEMENT_NO_PARAM_EXECUTOR = + "com.google.cloud.spanner.connection.ClientSideStatementNoParamExecutor"; + private static final String CLIENT_SIDE_STATEMENT_SET_EXECUTOR = + "com.google.cloud.spanner.connection.ClientSideStatementSetExecutor"; + private static final String CLIENT_SIDE_STATEMENT_PG_EXECUTOR = + "com.google.cloud.spanner.connection.ClientSideStatementPgBeginExecutor"; + private static final String ABSTRACT_STATEMENT_PARSER = + "com.google.cloud.spanner.connection.AbstractStatementParser"; + private static final String STATEMENT_PARSER = + "com.google.cloud.spanner.connection.SpannerStatementParser"; + private static final String POSTGRESQL_STATEMENT_PARSER = + "com.google.cloud.spanner.connection.PostgreSQLStatementParser"; + private static final String STATEMENT_RESULT = + "com.google.cloud.spanner.connection.StatementResult$ResultType"; + + @Override + public void beforeAnalysis(BeforeAnalysisAccess access) { + registerSpannerTestClasses(access); + if (access.findClassByName(CLIENT_SIDE_IMPL_CLASS) != null) { + NativeImageUtils.registerClassHierarchyForReflection(access, CLIENT_SIDE_IMPL_CLASS); + } + if (access.findClassByName(CLIENT_SIDE_STATEMENT_NO_PARAM_EXECUTOR) != null) { + NativeImageUtils.registerClassForReflection(access, CLIENT_SIDE_STATEMENT_NO_PARAM_EXECUTOR); + } + if (access.findClassByName(CLIENT_SIDE_STATEMENT_PG_EXECUTOR) != null) { + NativeImageUtils.registerClassForReflection(access, CLIENT_SIDE_STATEMENT_PG_EXECUTOR); + } + if (access.findClassByName(CLIENT_SIDE_STATEMENT_SET_EXECUTOR) != null) { + NativeImageUtils.registerClassForReflection(access, CLIENT_SIDE_STATEMENT_SET_EXECUTOR); + } + if (access.findClassByName(CLIENT_SIDE_VALUE_CONVERTER) != null) { + NativeImageUtils.registerClassHierarchyForReflection(access, CLIENT_SIDE_VALUE_CONVERTER); + } + if (access.findClassByName(CLIENT_SIDE_STATEMENTS) != null) { + NativeImageUtils.registerClassForReflection(access, CLIENT_SIDE_STATEMENTS); + } + if (access.findClassByName(CONNECTION_STATEMENT_EXECUTOR) != null) { + NativeImageUtils.registerClassForReflection(access, CONNECTION_STATEMENT_EXECUTOR); + } + if (access.findClassByName(CONNECTION_IMPL) != null) { + NativeImageUtils.registerClassForReflection(access, CONNECTION_IMPL); + } + if (access.findClassByName(ABSTRACT_STATEMENT_PARSER) != null) { + NativeImageUtils.registerClassHierarchyForReflection(access, ABSTRACT_STATEMENT_PARSER); + NativeImageUtils.registerClassForReflection(access, "com.google.cloud.spanner.Dialect"); + } + if (access.findClassByName(STATEMENT_PARSER) != null) { + NativeImageUtils.registerConstructorsForReflection(access, STATEMENT_PARSER); + } + if (access.findClassByName(POSTGRESQL_STATEMENT_PARSER) != null) { + NativeImageUtils.registerConstructorsForReflection(access, POSTGRESQL_STATEMENT_PARSER); + } + if (access.findClassByName(STATEMENT_RESULT) != null) { + NativeImageUtils.registerClassForReflection(access, STATEMENT_RESULT); + } + + Class spannerClass = access.findClassByName(SPANNER_CLASS); + if (spannerClass != null) { + NativeImageUtils.registerClassHierarchyForReflection( + access, "com.google.spanner.admin.database.v1.Database"); + NativeImageUtils.registerClassHierarchyForReflection( + access, "com.google.spanner.admin.instance.v1.Instance"); + NativeImageUtils.registerClassForReflection( + access, "com.google.spanner.admin.database.v1.RestoreInfo"); + } + } + + private void registerSpannerTestClasses(BeforeAnalysisAccess access) { + Class spannerTestClass = access.findClassByName(SPANNER_TEST_CLASS); + if (spannerTestClass != null) { + NativeImageUtils.registerConstructorsForReflection(access, SPANNER_TEST_CLASS); + } + Class mockClass = access.findClassByName(MOCK_CLASS); + if (mockClass != null) { + NativeImageUtils.registerClassForReflection( + access, "com.google.cloud.spanner.MockDatabaseAdminServiceImpl$MockBackup"); + } + } +} diff --git a/google-cloud-spanner/src/main/resources/META-INF/native-image/native-image.properties b/google-cloud-spanner/src/main/resources/META-INF/native-image/native-image.properties new file mode 100644 index 00000000000..0bcf872e79b --- /dev/null +++ b/google-cloud-spanner/src/main/resources/META-INF/native-image/native-image.properties @@ -0,0 +1,4 @@ +Args = --initialize-at-build-time=com.google.cloud.spanner.IntegrationTestEnv,\ + org.junit.experimental.categories.CategoryValidator,\ + org.junit.validator.AnnotationValidator \ + --features=com.google.cloud.spanner.nativeimage.SpannerFeature diff --git a/google-cloud-spanner/src/main/resources/META-INF/native-image/resource-config.json b/google-cloud-spanner/src/main/resources/META-INF/native-image/resource-config.json new file mode 100644 index 00000000000..91d913b239b --- /dev/null +++ b/google-cloud-spanner/src/main/resources/META-INF/native-image/resource-config.json @@ -0,0 +1,8 @@ +{ + "resources": [ + {"pattern": "\\Qcom/google/cloud/spanner/connection/ClientSideStatements.json\\E"}, + {"pattern": "\\Qcom/google/cloud/spanner/connection/PG_ClientSideStatements.json\\E"}, + {"pattern": "\\Qcom/google/cloud/spanner/spi/v1/grpc-gcp-apiconfig.json\\E"}, + {"pattern": "\\Qcom/google/cloud/spanner/connection/ITSqlScriptTest_TestQueryOptions.sql\\E"} + ] +} \ No newline at end of file diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionClientTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionClientTests.java similarity index 99% rename from google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionClientTest.java rename to google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionClientTests.java index 7c2710156c7..6d1f88b26fb 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionClientTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionClientTests.java @@ -54,7 +54,7 @@ import org.mockito.MockitoAnnotations; @RunWith(Parameterized.class) -public class SessionClientTest { +public class SessionClientTests { private final class TestExecutorFactory implements ExecutorFactory { @Override public ScheduledExecutorService get() { diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/DdlClientTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/DdlClientTests.java similarity index 99% rename from google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/DdlClientTest.java rename to google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/DdlClientTests.java index a490949a31b..d46e4dca592 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/DdlClientTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/DdlClientTests.java @@ -37,7 +37,7 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public class DdlClientTest { +public class DdlClientTests { private final String instanceId = "test-instance"; private final String databaseId = "test-database"; diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SqlScriptVerifier.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SqlScriptVerifier.java index d9de876ceda..15b031705e3 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SqlScriptVerifier.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SqlScriptVerifier.java @@ -16,10 +16,8 @@ package com.google.cloud.spanner.connection; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.startsWith; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import com.google.cloud.spanner.Dialect; import com.google.cloud.spanner.ErrorCode; @@ -175,17 +173,16 @@ public SqlScriptVerifier(GenericConnectionProvider provider) { @Override protected void verifyExpectedException( String statement, Exception e, String code, String messagePrefix) { - assertThat(e instanceof SpannerException, is(true)); + assertTrue(e instanceof SpannerException); SpannerException spannerException = (SpannerException) e; - assertThat( - statement + " resulted in " + spannerException.toString(), - spannerException.getErrorCode(), - is(equalTo(ErrorCode.valueOf(code)))); + assertEquals( + statement + " resulted in " + spannerException, + ErrorCode.valueOf(code), + spannerException.getErrorCode()); if (messagePrefix != null) { - assertThat( + assertTrue( statement, - e.getMessage(), - startsWith(messagePrefix.substring(1, messagePrefix.length() - 1))); + e.getMessage().startsWith(messagePrefix.substring(1, messagePrefix.length() - 1))); } } } diff --git a/owlbot.py b/owlbot.py index cb907a9878c..22fabca5c94 100644 --- a/owlbot.py +++ b/owlbot.py @@ -42,6 +42,8 @@ ".github/blunderbuss.yml", ".github/workflows/samples.yaml", ".github/workflows/ci.yaml", + ".kokoro/common.sh", ".kokoro/build.sh", + ".kokoro/dependencies.sh", ] )