Skip to content
Continuous Integration

Support Snowflake (#5500) #2079

Sign in to view logs

Support Snowflake (#5500)

Support Snowflake (#5500) #2079

Workflow file for this run

.github/workflows/ci.yml at d179ff2
# This file was automatically generated by sbt-github-actions using the
# githubWorkflowGenerate task. You should add and commit this file to
# your git repository. It goes without saying that you shouldn't edit
# this file by hand! Instead, if you wish to make changes, you should
# change your sbt build configuration to revise the workflow description
# to meet your needs, then regenerate this file.
name: Continuous Integration
on:
pull_request:
branches: [main]
push:
branches: [main]
tags: [v*]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
concurrency:
group: ${{ github.workflow }} @ ${{ github.ref }}
cancel-in-progress: true
jobs:
build:
name: Build and Test
strategy:
matrix:
os: [ubuntu-latest]
scala: [2.13, 2.12]
java: [corretto@11, corretto@17, corretto@21]
exclude:
- scala: 2.12
java: corretto@17
- scala: 2.12
java: corretto@21
runs-on: ${{ matrix.os }}
timeout-minutes: 60
steps:
- name: Install sbt
if: contains(runner.os, 'macos')
run: brew install sbt
- name: Checkout current branch (full)
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Java (corretto@11)
id: setup-java-corretto-11
if: matrix.java == 'corretto@11'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 11
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false'
run: sbt +update
- name: Setup Java (corretto@17)
id: setup-java-corretto-17
if: matrix.java == 'corretto@17'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 17
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@17' && steps.setup-java-corretto-17.outputs.cache-hit == 'false'
run: sbt +update
- name: Setup Java (corretto@21)
id: setup-java-corretto-21
if: matrix.java == 'corretto@21'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 21
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@21' && steps.setup-java-corretto-21.outputs.cache-hit == 'false'
run: sbt +update
- name: gcloud auth
if: (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork)
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GCP_CREDENTIALS }}
export_environment_variables: true
create_credentials_file: true
- name: Setup GitHub Action
run: scripts/gha_setup.sh
- name: Check that workflows are up to date
run: sbt githubWorkflowCheck
- name: Check headers and formatting
if: matrix.java == 'corretto@11' && matrix.os == 'ubuntu-latest'
run: sbt '++ ${{ matrix.scala }}' headerCheckAll javafmtCheckAll scalafmtCheckAll 'project /' scalafmtSbtCheck
- name: Test
run: sbt '++ ${{ matrix.scala }}' test
- name: Check binary compatibility
if: matrix.java == 'corretto@11' && matrix.os == 'ubuntu-latest'
run: sbt '++ ${{ matrix.scala }}' mimaReportBinaryIssues
- name: Generate API documentation
if: matrix.java == 'corretto@11' && matrix.os == 'ubuntu-latest'
run: sbt '++ ${{ matrix.scala }}' doc
- name: Check dependencies
run: sbt '++ ${{ matrix.scala }}' undeclaredCompileDependenciesTest unusedCompileDependenciesTest
- name: Make target directories
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
run: mkdir -p scio-bom/target scio-tensorflow/target site/target scio-cassandra/cassandra3/target scio-elasticsearch/es8/target scio-jdbc/target scio-macros/target scio-grpc/target scio-elasticsearch/common/target scio-test/target scio-avro/target scio-elasticsearch/es7/target scio-snowflake/target scio-redis/target scio-extra/target scio-test/parquet/target scio-test/core/target scio-google-cloud-platform/target scio-smb/target scio-test/google-cloud-platform/target scio-neo4j/target scio-parquet/target scio-core/target scio-repl/target project/target
- name: Compress target directories
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
run: tar cf targets.tar scio-bom/target scio-tensorflow/target site/target scio-cassandra/cassandra3/target scio-elasticsearch/es8/target scio-jdbc/target scio-macros/target scio-grpc/target scio-elasticsearch/common/target scio-test/target scio-avro/target scio-elasticsearch/es7/target scio-snowflake/target scio-redis/target scio-extra/target scio-test/parquet/target scio-test/core/target scio-google-cloud-platform/target scio-smb/target scio-test/google-cloud-platform/target scio-neo4j/target scio-parquet/target scio-core/target scio-repl/target project/target
- name: Upload target directories
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
uses: actions/upload-artifact@v4
with:
name: target-${{ matrix.os }}-${{ matrix.java }}-${{ matrix.scala }}
path: targets.tar
publish:
name: Publish Artifacts
needs: [build]
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
strategy:
matrix:
os: [ubuntu-latest]
java: [corretto@11]
runs-on: ${{ matrix.os }}
steps:
- name: Install sbt
if: contains(runner.os, 'macos')
run: brew install sbt
- name: Checkout current branch (full)
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Java (corretto@11)
id: setup-java-corretto-11
if: matrix.java == 'corretto@11'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 11
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false'
run: sbt +update
- name: Setup Java (corretto@17)
id: setup-java-corretto-17
if: matrix.java == 'corretto@17'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 17
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@17' && steps.setup-java-corretto-17.outputs.cache-hit == 'false'
run: sbt +update
- name: Setup Java (corretto@21)
id: setup-java-corretto-21
if: matrix.java == 'corretto@21'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 21
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@21' && steps.setup-java-corretto-21.outputs.cache-hit == 'false'
run: sbt +update
- name: Download target directories (2.13)
uses: actions/download-artifact@v4
with:
name: target-${{ matrix.os }}-${{ matrix.java }}-2.13
- name: Inflate target directories (2.13)
run: |
tar xf targets.tar
rm targets.tar
- name: Download target directories (2.12)
uses: actions/download-artifact@v4
with:
name: target-${{ matrix.os }}-${{ matrix.java }}-2.12
- name: Inflate target directories (2.12)
run: |
tar xf targets.tar
rm targets.tar
- name: Import signing key
if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE == ''
env:
PGP_SECRET: ${{ secrets.PGP_SECRET }}
PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
run: echo $PGP_SECRET | base64 -d -i - | gpg --import
- name: Import signing key and strip passphrase
if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE != ''
env:
PGP_SECRET: ${{ secrets.PGP_SECRET }}
PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
run: |
echo "$PGP_SECRET" | base64 -d -i - > /tmp/signing-key.gpg
echo "$PGP_PASSPHRASE" | gpg --pinentry-mode loopback --passphrase-fd 0 --import /tmp/signing-key.gpg
(echo "$PGP_PASSPHRASE"; echo; echo) | gpg --command-fd 0 --pinentry-mode loopback --change-passphrase $(gpg --list-secret-keys --with-colons 2> /dev/null | grep '^sec:' | cut --delimiter ':' --fields 5 | tail -n 1)
- name: Package repl
run: sbt scio-repl/assembly
- name: Publish
env:
SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
SONATYPE_CREDENTIAL_HOST: ${{ secrets.SONATYPE_CREDENTIAL_HOST }}
run: sbt tlCiRelease
- name: Upload Repl
uses: softprops/action-gh-release@v1
with:
files: scio-repl/target/scala-2.13/scio-repl.jar
draft: true
dependency-submission:
name: Submit Dependencies
if: github.event.repository.fork == false && github.event_name != 'pull_request'
strategy:
matrix:
os: [ubuntu-latest]
java: [corretto@11]
runs-on: ${{ matrix.os }}
steps:
- name: Install sbt
if: contains(runner.os, 'macos')
run: brew install sbt
- name: Checkout current branch (full)
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Java (corretto@11)
id: setup-java-corretto-11
if: matrix.java == 'corretto@11'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 11
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false'
run: sbt +update
- name: Setup Java (corretto@17)
id: setup-java-corretto-17
if: matrix.java == 'corretto@17'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 17
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@17' && steps.setup-java-corretto-17.outputs.cache-hit == 'false'
run: sbt +update
- name: Setup Java (corretto@21)
id: setup-java-corretto-21
if: matrix.java == 'corretto@21'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 21
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@21' && steps.setup-java-corretto-21.outputs.cache-hit == 'false'
run: sbt +update
- name: Submit Dependencies
uses: scalacenter/sbt-dependency-submission@v2
with:
modules-ignore: scio_2.13 scio_2.12 scio-examples_2.13 scio-examples_2.12 integration_2.13 integration_2.12 scio-jmh_2.13 scio-jmh_2.12
configs-ignore: test scala-tool scala-doc-tool test-internal
validate-steward:
name: Validate Steward Config
strategy:
matrix:
os: [ubuntu-latest]
java: [temurin@11]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout current branch (fast)
uses: actions/checkout@v4
- name: Setup Java (temurin@11)
id: setup-java-temurin-11
if: matrix.java == 'temurin@11'
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: 11
- uses: coursier/setup-action@v1
with:
apps: scala-steward
- run: scala-steward validate-repo-config .scala-steward.conf
coverage:
name: Test Coverage
strategy:
matrix:
os: [ubuntu-latest]
scala: [2.13]
java: [corretto@11]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout current branch (full)
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Java (corretto@11)
id: setup-java-corretto-11
if: matrix.java == 'corretto@11'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 11
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false'
run: sbt +update
- name: Check that workflows are up to date
run: sbt '++ ${{ matrix.scala }}' githubWorkflowCheck
- name: Test coverage
run: sbt '++ ${{ matrix.scala }}' coverage test coverageAggregate
- name: Upload coverage report
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
avro-legacy:
name: Test Legacy Avro
if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
strategy:
matrix:
os: [ubuntu-latest]
scala: [2.13]
java: [corretto@11]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout current branch (full)
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Java (corretto@11)
id: setup-java-corretto-11
if: matrix.java == 'corretto@11'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 11
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false'
run: sbt +update
- name: Test
env:
JAVA_OPTS: '-Davro.version=1.8.2'
run: sbt '++ ${{ matrix.scala }}' scio-avro/test
it-test:
name: Integration Test
if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
strategy:
matrix:
os: [ubuntu-latest]
scala: [2.13]
java: [corretto@11]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout current branch (full)
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Java (corretto@11)
id: setup-java-corretto-11
if: matrix.java == 'corretto@11'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 11
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false'
run: sbt +update
- name: Check that workflows are up to date
run: sbt '++ ${{ matrix.scala }}' githubWorkflowCheck
- name: gcloud auth
if: (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork)
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GCP_CREDENTIALS }}
export_environment_variables: true
create_credentials_file: true
- name: Setup GitHub Action
env:
BQ_READ_TIMEOUT: 30000
CLOUDSQL_SQLSERVER_PASSWORD: ${{ secrets.CLOUDSQL_SQLSERVER_PASSWORD }}
run: scripts/gha_setup.sh
- name: Test
run: 'sbt ''++ ${{ matrix.scala }}'' ''set integration/test/skip := false'' integration/test'
site:
name: Generate Site
if: (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork)
strategy:
matrix:
os: [ubuntu-latest]
scala: [2.13]
java: [corretto@11]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout current branch (full)
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Java (corretto@11)
id: setup-java-corretto-11
if: matrix.java == 'corretto@11'
uses: actions/setup-java@v4
with:
distribution: corretto
java-version: 11
cache: sbt
- name: sbt update
if: matrix.java == 'corretto@11' && steps.setup-java-corretto-11.outputs.cache-hit == 'false'
run: sbt +update
- name: Check that workflows are up to date
run: sbt '++ ${{ matrix.scala }}' githubWorkflowCheck
- name: gcloud auth
if: (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork)
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GCP_CREDENTIALS }}
export_environment_variables: true
create_credentials_file: true
- name: Setup GitHub Action
run: scripts/gha_setup.sh
- name: Generate site
env:
SOCCO: true
run: sbt '++ ${{ matrix.scala }}' scio-examples/compile site/makeSite
- name: Publish site
if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/v')
uses: peaceiris/actions-gh-pages@v3.9.3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: site/target/site
keep_files: true