Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/secure-hermetic-build-docker-ima…
Browse files Browse the repository at this point in the history
…ge' into secure-hermetic-build-docker-image
  • Loading branch information
diegomarquezp committed Sep 17, 2024
2 parents db2e8e7 + 3f3deec commit fb98222
Show file tree
Hide file tree
Showing 23 changed files with 546 additions and 91 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ ENV DOCKER_GAPIC_GENERATOR_VERSION="2.45.1-SNAPSHOT"
# {x-version-update-end}

# use Docker Buildkit caching for faster local builds
RUN --mount=type=cache,target=/root/.m2 mvn install -T 1.5C \
RUN --mount=type=cache,target=/root/.m2 mvn install -B -ntp -T 1.5C \
-DskipTests -Dclirr.skip -Dcheckstyle.skip -Djacoco.skip -Dmaven.test.skip \
-Dmaven.site.skikip -Dmaven.javadoc.skip -pl gapic-generator-java -am

Expand Down
60 changes: 60 additions & 0 deletions .github/scripts/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# GitHub action job to test core java library features on
# downstream client libraries before they are released.

# This composite action should be used in google-cloud-java and handwritten
# libraries to generate changed libraries.
# This composite action serves as a source of truth of scripts that run
# library generation and create pull requests.
name: Hermetic library generation
description: Runs hermetic library generation to produce changed libraries
inputs:
base_ref:
description: base branch
required: true
head_ref:
description: head branch
required: true
image_tag:
description: the tag of hermetic build image
required: false
token:
description: Personal Access Token
required: true

runs:
using: "composite"
steps:
- name: Copy shell script
shell: bash
run: |
cd ${{ github.action_path }}
cp hermetic_library_generation.sh $GITHUB_WORKSPACE
- name: Generate changed libraries
shell: bash
run: |
set -x
[ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com"
[ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot"
cd "${GITHUB_WORKSPACE}"
bash hermetic_library_generation.sh \
--target_branch "${BASE_REF}" \
--current_branch "${HEAD_REF}" \
--image_tag "${IMAGE_TAG}"
env:
BASE_REF: ${{ inputs.base_ref }}
HEAD_REF: ${{ inputs.head_ref }}
IMAGE_TAG: ${{ inputs.image_tag }}
GH_TOKEN: ${{ inputs.token }}
42 changes: 21 additions & 21 deletions .github/scripts/hermetic_library_generation.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ set -e
# The parameters of this script is:
# 1. target_branch, the branch into which the pull request is merged.
# 2. current_branch, the branch with which the pull request is associated.
# 3. [optional] generation_config, the path to the generation configuration,
# 3. [optional] image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation.
# 4. [optional] generation_config, the path to the generation configuration,
# the default value is generation_config.yaml in the repository root.
while [[ $# -gt 0 ]]; do
key="$1"
Expand All @@ -35,6 +36,10 @@ case "${key}" in
current_branch="$2"
shift
;;
--image_tag)
image_tag="$2"
shift
;;
--generation_config)
generation_config="$2"
shift
Expand Down Expand Up @@ -62,7 +67,10 @@ if [ -z "${generation_config}" ]; then
echo "Use default generation config: ${generation_config}"
fi

image_tag=local
if [ -z "${image_tag}" ]; then
image_tag=$(grep "gapic_generator_version" "${generation_config}" | cut -d ':' -f 2 | xargs)
fi

workspace_name="/workspace"
baseline_generation_config="baseline_generation_config.yaml"
message="chore: generate libraries at $(date)"
Expand All @@ -73,40 +81,32 @@ git checkout "${current_branch}"
# copy generation configuration from target branch to current branch.
git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}"

generator_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout -pl gapic-generator-java)
echo "Local generator version: ${generator_version}"

# install generator locally since we're using a SNAPSHOT version.
mvn -V -B -ntp clean install -DskipTests
# get .m2 folder so it's mapped into the docker container
m2_folder=$(dirname "$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)")

# build image locally since we want to include latest change.
docker build \
-f .cloudbuild/library_generation/library_generation.Dockerfile \
-t gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \
.
# run hermetic code generation docker image.
docker run \
--rm \
-u "$(id -u):$(id -g)" \
-v "$(pwd):${workspace_name}" \
-v "$HOME"/.m2:/home/.m2 \
-e GENERATOR_VERSION="${generator_version}" \
-v "${m2_folder}":/home/.m2 \
-e GENERATOR_VERSION="${image_tag}" \
gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \
--baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \
--current-generation-config-path="${workspace_name}/${generation_config}"

# commit the change to the pull request.
rm -rdf output googleapis "${baseline_generation_config}"
git add --all -- ':!pr_description.txt'
git add --all -- ':!pr_description.txt' ':!hermetic_library_generation.sh'
changed_files=$(git diff --cached --name-only)
if [[ "${changed_files}" == "" ]]; then
echo "There is no generated code change."
echo "Skip committing to the pull request."
exit 0
if [[ "${changed_files}" != "" ]]; then
echo "Commit changes..."
git commit -m "${message}"
git push
else
echo "There is no generated code change, skip commit."
fi

git commit -m "${message}"
git push
# set pr body if pr_description.txt is generated.
if [[ -f "pr_description.txt" ]]; then
pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number")
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ jobs:
run: |
mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip
- name: Validate gapic-generator-java-bom
uses: googleapis/java-cloud-bom/tests/validate-bom@6626f91d2ea122ba19d15e33c8a6bc21aebe5177
uses: googleapis/java-cloud-bom/tests/validate-bom@06f632d634598a149339826143ee9533b7dbfac5
with:
bom-path: gapic-generator-java-bom/pom.xml

Expand Down
41 changes: 29 additions & 12 deletions .github/workflows/hermetic_library_generation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,41 @@ jobs:
library_generation:
runs-on: ubuntu-latest
steps:
- name: Determine whether the pull request comes from a fork
run: |
if [[ "${GITHUB_REPOSITORY}" != "${REPO_FULL_NAME}" ]]; then
echo "This PR comes from a fork. Skip library generation."
echo "SHOULD_RUN=false" >> $GITHUB_ENV
else
echo "SHOULD_RUN=true" >> $GITHUB_ENV
fi
- uses: actions/checkout@v4
if: env.SHOULD_RUN == 'true'
with:
fetch-depth: 0
token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
- name: Generate changed libraries
- name: Install Maven modules
if: env.SHOULD_RUN == 'true'
shell: bash
run: |
set -x
if [[ "${GITHUB_REPOSITORY}" != "${REPO_FULL_NAME}" ]]; then
echo "This PR comes from a fork. Skip library generation."
exit 0
fi
[ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com"
[ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot"
bash .github/scripts/hermetic_library_generation.sh \
--target_branch "${base_ref}" \
--current_branch "${head_ref}"
git checkout "${HEAD_REF}"
mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip
env:
HEAD_REF: ${{ github.head_ref }}
- name: Build image
if: env.SHOULD_RUN == 'true'
shell: bash
run: |
GENERATOR_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout -pl gapic-generator-java)
echo "GENERATOR_VERSION=${GENERATOR_VERSION}" >> "$GITHUB_ENV"
docker build \
-f .cloudbuild/library_generation/library_generation.Dockerfile \
-t gcr.io/cloud-devrel-public-resources/java-library-generation:"${GENERATOR_VERSION}" \
.
- uses: ./.github/scripts
if: env.SHOULD_RUN == 'true'
with:
base_ref: ${{ github.base_ref }}
head_ref: ${{ github.head_ref }}
GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
image_tag: ${{ env.GENERATOR_VERSION }}
token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
4 changes: 2 additions & 2 deletions gapic-generator-java-pom-parent/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@
<gson.version>2.11.0</gson.version>
<guava.version>33.3.0-jre</guava.version>
<protobuf.version>3.25.4</protobuf.version>
<opentelemetry.version>1.42.0</opentelemetry.version>
<opentelemetry.version>1.42.1</opentelemetry.version>
<maven.compiler.release>8</maven.compiler.release>
<errorprone.version>2.31.0</errorprone.version>
<errorprone.version>2.32.0</errorprone.version>
<j2objc-annotations.version>3.0.0</j2objc-annotations.version>
<threetenbp.version>1.6.9</threetenbp.version>
<junit.version>5.11.0</junit.version>
Expand Down
14 changes: 7 additions & 7 deletions gax-java/dependencies.properties
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@ version.io_grpc=1.66.0
# It should be constructed the following way:
# 1) Take full artifact id (including the group and classifier (if any) portions) and remove version portion.
# 2) Replace all characters which are neither alphabetic nor digits with the underscore ('_') character
maven.com_google_api_grpc_proto_google_common_protos=com.google.api.grpc:proto-google-common-protos:2.43.0
maven.com_google_api_grpc_grpc_google_common_protos=com.google.api.grpc:grpc-google-common-protos:2.43.0
maven.com_google_api_grpc_proto_google_common_protos=com.google.api.grpc:proto-google-common-protos:2.44.0
maven.com_google_api_grpc_grpc_google_common_protos=com.google.api.grpc:grpc-google-common-protos:2.44.0
maven.com_google_auth_google_auth_library_oauth2_http=com.google.auth:google-auth-library-oauth2-http:1.25.0
maven.com_google_auth_google_auth_library_credentials=com.google.auth:google-auth-library-credentials:1.25.0
maven.io_opentelemetry_opentelemetry_api=io.opentelemetry:opentelemetry-api:1.42.0
maven.io_opentelemetry_opentelemetry_api=io.opentelemetry:opentelemetry-api:1.42.1
maven.io_opencensus_opencensus_api=io.opencensus:opencensus-api:0.31.1
maven.io_opencensus_opencensus_contrib_grpc_metrics=io.opencensus:opencensus-contrib-grpc-metrics:0.31.1
maven.io_opencensus_opencensus_contrib_http_util=io.opencensus:opencensus-contrib-http-util:0.31.1
Expand All @@ -63,14 +63,14 @@ maven.com_google_guava_failureaccess=com.google.guava:failureaccess:1.0.2
maven.org_apache_commons_commons_lang3=org.apache.commons:commons-lang3:3.17.0
maven.com_google_android_annotations=com.google.android:annotations:4.1.1.4
maven.com_google_code_findbugs_jsr305=com.google.code.findbugs:jsr305:3.0.2
maven.com_google_errorprone_error_prone_annotations=com.google.errorprone:error_prone_annotations:2.31.0
maven.com_google_errorprone_error_prone_annotations=com.google.errorprone:error_prone_annotations:2.32.0
maven.com_google_j2objc_j2objc_annotations=com.google.j2objc:j2objc-annotations:2.8
maven.com_google_auto_value_auto_value=com.google.auto.value:auto-value:1.11.0
maven.com_google_auto_value_auto_value_annotations=com.google.auto.value:auto-value-annotations:1.11.0
maven.com_google_api_api_common=com.google.api:api-common:2.35.0
maven.com_google_api_api_common=com.google.api:api-common:2.36.0
maven.org_threeten_threetenbp=org.threeten:threetenbp:1.6.9
maven.com_google_api_grpc_grpc_google_iam_v1=com.google.api.grpc:grpc-google-iam-v1:1.38.0
maven.com_google_api_grpc_proto_google_iam_v1=com.google.api.grpc:proto-google-iam-v1:1.38.0
maven.com_google_api_grpc_grpc_google_iam_v1=com.google.api.grpc:grpc-google-iam-v1:1.39.0
maven.com_google_api_grpc_proto_google_iam_v1=com.google.api.grpc:proto-google-iam-v1:1.39.0
maven.com_google_http_client_google_http_client=com.google.http-client:google-http-client:1.45.0
maven.com_google_http_client_google_http_client_gson=com.google.http-client:google-http-client-gson:1.45.0
maven.org_codehaus_mojo_animal_sniffer_annotations=org.codehaus.mojo:animal-sniffer-annotations:1.24
Expand Down
10 changes: 10 additions & 0 deletions gax-java/gax-grpc/clirr-ignored-differences.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- see http://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
<differences>
<!-- Add AutoValue abstract method isDirectPath -->
<difference>
<differenceType>7013</differenceType>
<className>com/google/api/gax/grpc/GrpcTransportChannel</className>
<method>boolean isDirectPath()</method>
</difference>
</differences>
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ public GrpcCallContext getEmptyCallContext() {
/** The channel in use. */
abstract ManagedChannel getManagedChannel();

public abstract boolean isDirectPath();

public Channel getChannel() {
return getManagedChannel();
}
Expand Down Expand Up @@ -100,7 +102,7 @@ public void close() {
}

public static Builder newBuilder() {
return new AutoValue_GrpcTransportChannel.Builder();
return new AutoValue_GrpcTransportChannel.Builder().setDirectPath(false);
}

public static GrpcTransportChannel create(ManagedChannel channel) {
Expand All @@ -111,6 +113,8 @@ public static GrpcTransportChannel create(ManagedChannel channel) {
public abstract static class Builder {
public abstract Builder setManagedChannel(ManagedChannel value);

abstract Builder setDirectPath(boolean value);

public abstract GrpcTransportChannel build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -258,9 +258,12 @@ public TransportChannel getTransportChannel() throws IOException {
}

private TransportChannel createChannel() throws IOException {
return GrpcTransportChannel.create(
ChannelPool.create(
channelPoolSettings, InstantiatingGrpcChannelProvider.this::createSingleChannel));
return GrpcTransportChannel.newBuilder()
.setManagedChannel(
ChannelPool.create(
channelPoolSettings, InstantiatingGrpcChannelProvider.this::createSingleChannel))
.setDirectPath(this.canUseDirectPath())
.build();
}

private boolean isDirectPathEnabled() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Copyright 2024 Google LLC
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google LLC nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package com.google.api.gax.grpc;

import static org.junit.jupiter.api.Assertions.*;

import io.grpc.ManagedChannel;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;

class GrpcTransportChannelTest {

@Test
void testBuilderDefaults() {
ManagedChannel channel = Mockito.mock(ManagedChannel.class);
GrpcTransportChannel grpcTransportChannel =
GrpcTransportChannel.newBuilder().setManagedChannel(channel).build();
assertFalse(grpcTransportChannel.isDirectPath());
}
}
Loading

0 comments on commit fb98222

Please sign in to comment.