Skip to content

Commit

Permalink
fix: add native image configurations for Spanner classes (#1858)
Browse files Browse the repository at this point in the history
* fix: add native image configurations for Spanner classes
  • Loading branch information
mpeddada1 authored May 25, 2022
1 parent 67863d8 commit 92d0292
Show file tree
Hide file tree
Showing 14 changed files with 242 additions and 43 deletions.
14 changes: 13 additions & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,22 @@ jobs:
java: [8, 11, 17]
steps:
- uses: actions/checkout@v3
# For Java 8 tests, use JDK 11 to compile
- if: ${{matrix.java}} == '8'
uses: actions/setup-java@v3
with:
java-version: 11
distribution: zulu
- if: ${{matrix.java}} == '8'
run: echo "JAVA11_HOME=${JAVA_HOME}" >> $GITHUB_ENV
shell: bash
- uses: actions/setup-java@v3
with:
distribution: zulu
java-version: ${{matrix.java}}
- if: ${{matrix.java}} == '8'
run: echo "JAVA8_HOME=${JAVA_HOME}" >> $GITHUB_ENV
shell: bash
- run: java -version
- run: .kokoro/dependencies.sh
lint:
Expand All @@ -108,7 +120,7 @@ jobs:
- uses: actions/setup-java@v3
with:
distribution: zulu
java-version: 8
java-version: 11
- run: java -version
- run: .kokoro/build.sh
env:
Expand Down
21 changes: 18 additions & 3 deletions .github/workflows/integration-tests-against-emulator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,27 @@ jobs:
- uses: stCarolas/setup-maven@v4
with:
maven-version: 3.8.1
- uses: actions/setup-java@v1
# Build with JDK 11 and run tests with JDK 8
- uses: actions/setup-java@v3
with:
java-version: 11
distribution: zulu
- run: echo "JAVA11_HOME=${JAVA_HOME}" >> $GITHUB_ENV
shell: bash
- uses: actions/setup-java@v3
with:
java-version: 8
distribution: zulu
- run: echo "JAVA8_HOME=${JAVA_HOME}" >> $GITHUB_ENV
shell: bash
- run: java -version
- run: .kokoro/build.sh
- run: mvn -B -Dspanner.testenv.instance="" -Penable-integration-tests -DtrimStackTrace=false -Dclirr.skip=true -Denforcer.skip=true -fae verify
- name: Compiling main library
run: .kokoro/build.sh
- name: Running tests
run: |
mvn -B -Dspanner.testenv.instance="" -Penable-integration-tests \
-DtrimStackTrace=false -Dclirr.skip=true -Denforcer.skip=true \
-Dmaven.main.skip=true -fae verify
env:
JOB_TYPE: test
SPANNER_EMULATOR_HOST: localhost:9010
Expand Down
27 changes: 24 additions & 3 deletions .github/workflows/samples.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,39 @@ jobs:
- name: Run checkstyle
run: mvn -P lint --quiet --batch-mode checkstyle:check
working-directory: samples/snippets
compile-java8:
name: "compile (8)"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-java@v3
with:
# Java 11 to generate class file targeting Java 8
java-version: 11
distribution: zulu
- name: Compile Spanner
run: mvn clean install
- uses: actions/setup-java@v3
with:
java-version: 8
distribution: zulu
- name: Compile samples
run: mvn compile
working-directory: samples

compile:
runs-on: ubuntu-latest
strategy:
matrix:
java: [8, 11]
java: [11, 17]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-java@v1
- uses: actions/setup-java@v3
with:
java-version: ${{matrix.java}}
distribution: zulu
- name: Compile Spanner
run: mvn clean install
- name: Compile samples
run: mvn compile
working-directory: samples
working-directory: samples
18 changes: 12 additions & 6 deletions .kokoro/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,6 @@ cd ${scriptDir}/..
# include common functions
source ${scriptDir}/common.sh

function setJava() {
export JAVA_HOME=$1
export PATH=${JAVA_HOME}/bin:$PATH
}

# units-java8 uses both JDK 11 and JDK 8. GraalVM dependencies require JDK 11 to
# compile the classes touching GraalVM classes.
if [ ! -z "${JAVA11_HOME}" ]; then
Expand Down Expand Up @@ -57,7 +52,6 @@ fi
# are compatible with Java 8 when running tests.
if [ ! -z "${JAVA8_HOME}" ]; then
setJava "${JAVA8_HOME}"
mvn -version
fi

RETURN_CODE=0
Expand Down Expand Up @@ -87,10 +81,21 @@ integration)
-DtrimStackTrace=false \
-Dclirr.skip=true \
-Denforcer.skip=true \
-Dmaven.main.skip=true \
-fae \
verify
RETURN_CODE=$?
;;
graalvm)
# Run Unit and Integration Tests with Native Image
mvn test -Pnative -Penable-integration-tests
RETURN_CODE=$?
;;
graalvm17)
# Run Unit and Integration Tests with Native Image
mvn test -Pnative -Penable-integration-tests
RETURN_CODE=$?
;;
slowtests)
mvn -B ${INTEGRATION_TEST_ARGS} \
-ntp \
Expand All @@ -100,6 +105,7 @@ slowtests)
-DtrimStackTrace=false \
-Dclirr.skip=true \
-Denforcer.skip=true \
-Dmaven.main.skip=true \
-fae \
verify
RETURN_CODE=$?
Expand Down
7 changes: 6 additions & 1 deletion .kokoro/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,9 @@ function retry_with_backoff {
## Helper functionss
function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; }
function msg() { println "$*" >&2; }
function println() { printf '%s\n' "$(now) $*"; }
function println() { printf '%s\n' "$(now) $*"; }

function setJava() {
export JAVA_HOME=$1
export PATH=${JAVA_HOME}/bin:$PATH
}
8 changes: 8 additions & 0 deletions .kokoro/dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,21 @@ function determineMavenOpts() {

export MAVEN_OPTS=$(determineMavenOpts)

if [ ! -z "${JAVA11_HOME}" ]; then
setJava "${JAVA11_HOME}"
fi

# this should run maven enforcer
retry_with_backoff 3 10 \
mvn install -B -V -ntp \
-DskipTests=true \
-Dmaven.javadoc.skip=true \
-Dclirr.skip=true

if [ ! -z "${JAVA8_HOME}" ]; then
setJava "${JAVA8_HOME}"
fi

mvn -B dependency:analyze -DfailOnWarning=true

echo "****************** DEPENDENCY LIST COMPLETENESS CHECK *******************"
Expand Down
35 changes: 20 additions & 15 deletions google-cloud-spanner/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
<properties>
<site.installationModule>google-cloud-spanner</site.installationModule>
<opencensus.version>0.31.1</opencensus.version>
<graalvm.version>22.1.0</graalvm.version>
<spanner.testenv.config.class>com.google.cloud.spanner.GceTestEnvConfig</spanner.testenv.config.class>
<spanner.testenv.instance>projects/gcloud-devel/instances/spanner-testing-east1</spanner.testenv.instance>
<spanner.gce.config.project_id>gcloud-devel</spanner.gce.config.project_id>
Expand Down Expand Up @@ -56,6 +57,16 @@
<id>default-test</id>
<configuration>
<excludedGroups>com.google.cloud.spanner.TracerTest,com.google.cloud.spanner.IntegrationTest</excludedGroups>

<!-- These configurations are only needed for native tests. The native profile
in shared-config uses maven-surefire to run both unit tests and integration tests -->
<systemPropertyVariables>
<spanner.testenv.config.class>${spanner.testenv.config.class}</spanner.testenv.config.class>
<spanner.testenv.instance>${spanner.testenv.instance}</spanner.testenv.instance>
<spanner.gce.config.project_id>${spanner.gce.config.project_id}</spanner.gce.config.project_id>
<spanner.testenv.kms_key.name>${spanner.testenv.kms_key.name}</spanner.testenv.kms_key.name>
</systemPropertyVariables>

</configuration>
</execution>
<execution>
Expand Down Expand Up @@ -105,20 +116,6 @@
</executions>
</plugin>

<!--Configurations for native image tests-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<spanner.testenv.config.class>${spanner.testenv.config.class}</spanner.testenv.config.class>
<spanner.testenv.instance>${spanner.testenv.instance}</spanner.testenv.instance>
<spanner.gce.config.project_id>${spanner.gce.config.project_id}</spanner.gce.config.project_id>
<spanner.testenv.kms_key.name>${spanner.testenv.kms_key.name}</spanner.testenv.kms_key.name>
</systemPropertyVariables>
<forkedProcessTimeoutInSeconds>3000</forkedProcessTimeoutInSeconds>
</configuration>
</plugin>
<plugin>
<groupId>org.graalvm.buildtools</groupId>
<artifactId>native-maven-plugin</artifactId>
Expand All @@ -142,7 +139,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<ignoredDependencies>io.grpc:grpc-protobuf-lite,org.hamcrest:hamcrest,org.hamcrest:hamcrest-core,com.google.errorprone:error_prone_annotations,org.openjdk.jmh:jmh-generator-annprocess,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,javax.annotation:javax.annotation-api,io.opencensus:opencensus-impl</ignoredDependencies>
<ignoredDependencies>io.grpc:grpc-protobuf-lite,org.hamcrest:hamcrest,org.hamcrest:hamcrest-core,com.google.errorprone:error_prone_annotations,org.openjdk.jmh:jmh-generator-annprocess,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,javax.annotation:javax.annotation-api,io.opencensus:opencensus-impl,org.graalvm.sdk:graal-sdk</ignoredDependencies>
</configuration>
</plugin>
</plugins>
Expand Down Expand Up @@ -296,6 +293,13 @@
<artifactId>grpc-alts</artifactId>
</dependency>

<dependency>
<groupId>org.graalvm.sdk</groupId>
<artifactId>graal-sdk</artifactId>
<version>${graalvm.version}</version>
<scope>provided</scope>
</dependency>

<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
Expand Down Expand Up @@ -352,6 +356,7 @@
<version>1.35</version>
<scope>test</scope>
</dependency>

</dependencies>

<profiles>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.spanner.nativeimage;

import com.google.api.gax.nativeimage.NativeImageUtils;
import org.graalvm.nativeimage.hosted.Feature;

/** Registers Spanner library classes for reflection. */
final class SpannerFeature implements Feature {

private static final String SPANNER_CLASS = "com.google.spanner.v1.SpannerGrpc";
private static final String SPANNER_TEST_CLASS = "com.google.cloud.spanner.GceTestEnvConfig";
private static final String MOCK_CLASS = "com.google.cloud.spanner.MockDatabaseAdminServiceImpl";
private static final String CLIENT_SIDE_IMPL_CLASS =
"com.google.cloud.spanner.connection.ClientSideStatementImpl";
private static final String CLIENT_SIDE_VALUE_CONVERTER =
"com.google.cloud.spanner.connection.ClientSideStatementValueConverters";
private static final String CONNECTION_IMPL =
"com.google.cloud.spanner.connection.ConnectionImpl";
private static final String CLIENT_SIDE_STATEMENTS =
"com.google.cloud.spanner.connection.ClientSideStatements";
private static final String CONNECTION_STATEMENT_EXECUTOR =
"com.google.cloud.spanner.connection.ConnectionStatementExecutor";
private static final String CLIENT_SIDE_STATEMENT_NO_PARAM_EXECUTOR =
"com.google.cloud.spanner.connection.ClientSideStatementNoParamExecutor";
private static final String CLIENT_SIDE_STATEMENT_SET_EXECUTOR =
"com.google.cloud.spanner.connection.ClientSideStatementSetExecutor";
private static final String CLIENT_SIDE_STATEMENT_PG_EXECUTOR =
"com.google.cloud.spanner.connection.ClientSideStatementPgBeginExecutor";
private static final String ABSTRACT_STATEMENT_PARSER =
"com.google.cloud.spanner.connection.AbstractStatementParser";
private static final String STATEMENT_PARSER =
"com.google.cloud.spanner.connection.SpannerStatementParser";
private static final String POSTGRESQL_STATEMENT_PARSER =
"com.google.cloud.spanner.connection.PostgreSQLStatementParser";
private static final String STATEMENT_RESULT =
"com.google.cloud.spanner.connection.StatementResult$ResultType";

@Override
public void beforeAnalysis(BeforeAnalysisAccess access) {
registerSpannerTestClasses(access);
if (access.findClassByName(CLIENT_SIDE_IMPL_CLASS) != null) {
NativeImageUtils.registerClassHierarchyForReflection(access, CLIENT_SIDE_IMPL_CLASS);
}
if (access.findClassByName(CLIENT_SIDE_STATEMENT_NO_PARAM_EXECUTOR) != null) {
NativeImageUtils.registerClassForReflection(access, CLIENT_SIDE_STATEMENT_NO_PARAM_EXECUTOR);
}
if (access.findClassByName(CLIENT_SIDE_STATEMENT_PG_EXECUTOR) != null) {
NativeImageUtils.registerClassForReflection(access, CLIENT_SIDE_STATEMENT_PG_EXECUTOR);
}
if (access.findClassByName(CLIENT_SIDE_STATEMENT_SET_EXECUTOR) != null) {
NativeImageUtils.registerClassForReflection(access, CLIENT_SIDE_STATEMENT_SET_EXECUTOR);
}
if (access.findClassByName(CLIENT_SIDE_VALUE_CONVERTER) != null) {
NativeImageUtils.registerClassHierarchyForReflection(access, CLIENT_SIDE_VALUE_CONVERTER);
}
if (access.findClassByName(CLIENT_SIDE_STATEMENTS) != null) {
NativeImageUtils.registerClassForReflection(access, CLIENT_SIDE_STATEMENTS);
}
if (access.findClassByName(CONNECTION_STATEMENT_EXECUTOR) != null) {
NativeImageUtils.registerClassForReflection(access, CONNECTION_STATEMENT_EXECUTOR);
}
if (access.findClassByName(CONNECTION_IMPL) != null) {
NativeImageUtils.registerClassForReflection(access, CONNECTION_IMPL);
}
if (access.findClassByName(ABSTRACT_STATEMENT_PARSER) != null) {
NativeImageUtils.registerClassHierarchyForReflection(access, ABSTRACT_STATEMENT_PARSER);
NativeImageUtils.registerClassForReflection(access, "com.google.cloud.spanner.Dialect");
}
if (access.findClassByName(STATEMENT_PARSER) != null) {
NativeImageUtils.registerConstructorsForReflection(access, STATEMENT_PARSER);
}
if (access.findClassByName(POSTGRESQL_STATEMENT_PARSER) != null) {
NativeImageUtils.registerConstructorsForReflection(access, POSTGRESQL_STATEMENT_PARSER);
}
if (access.findClassByName(STATEMENT_RESULT) != null) {
NativeImageUtils.registerClassForReflection(access, STATEMENT_RESULT);
}

Class<?> spannerClass = access.findClassByName(SPANNER_CLASS);
if (spannerClass != null) {
NativeImageUtils.registerClassHierarchyForReflection(
access, "com.google.spanner.admin.database.v1.Database");
NativeImageUtils.registerClassHierarchyForReflection(
access, "com.google.spanner.admin.instance.v1.Instance");
NativeImageUtils.registerClassForReflection(
access, "com.google.spanner.admin.database.v1.RestoreInfo");
}
}

private void registerSpannerTestClasses(BeforeAnalysisAccess access) {
Class<?> spannerTestClass = access.findClassByName(SPANNER_TEST_CLASS);
if (spannerTestClass != null) {
NativeImageUtils.registerConstructorsForReflection(access, SPANNER_TEST_CLASS);
}
Class<?> mockClass = access.findClassByName(MOCK_CLASS);
if (mockClass != null) {
NativeImageUtils.registerClassForReflection(
access, "com.google.cloud.spanner.MockDatabaseAdminServiceImpl$MockBackup");
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Args = --initialize-at-build-time=com.google.cloud.spanner.IntegrationTestEnv,\
org.junit.experimental.categories.CategoryValidator,\
org.junit.validator.AnnotationValidator \
--features=com.google.cloud.spanner.nativeimage.SpannerFeature
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"resources": [
{"pattern": "\\Qcom/google/cloud/spanner/connection/ClientSideStatements.json\\E"},
{"pattern": "\\Qcom/google/cloud/spanner/connection/PG_ClientSideStatements.json\\E"},
{"pattern": "\\Qcom/google/cloud/spanner/spi/v1/grpc-gcp-apiconfig.json\\E"},
{"pattern": "\\Qcom/google/cloud/spanner/connection/ITSqlScriptTest_TestQueryOptions.sql\\E"}
]
}
Loading

0 comments on commit 92d0292

Please sign in to comment.