Feature / Python build optimizations #1632
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration | |
| on: | |
| # Integration tests always run on pull requests | |
| pull_request: | |
| # Re-run integration tests in main, to make sure there are no issues from the merge | |
| push: | |
| branches: | |
| - main | |
| # Allow manual triggering of integration tests | |
| workflow_dispatch: | |
| # Use latest supported language versions for integration testing | |
| env: | |
| JAVA_VERSION: "21" | |
| JAVA_DISTRIBUTION: "zulu" | |
| PYTHON_VERSION: "3.12" | |
| # Node 20 breaks relative imports used in examples for web API tests | |
| NODE_VERSION: "18" | |
| jobs: | |
| web-api: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup - Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| java-version: ${{ env.JAVA_VERSION }} | |
| # Turn on Gradle dependency caching | |
| cache: gradle | |
| - name: Setup -Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Platform - Build distribution files | |
| run: ./gradlew installDist | |
| - name: Platform - Install the sandbox | |
| run: | | |
| VERSION=`dev/version.sh` | |
| mkdir -p build/dist/tracdap-sandbox-${VERSION} | |
| for MODULE in build/modules/*/install/*; do | |
| cp -R $MODULE/* build/dist/tracdap-sandbox-${VERSION} | |
| done | |
| cd build/dist | |
| ln -s tracdap-sandbox-${VERSION}/ current | |
| # Currently not starting svc-orch, we'd need to build the runtime and deploy an execution venv | |
| # OUr JavaScript tests only do data and metadata so far, no spawning / checking jobs | |
| # We should aim to get there though! | |
| - name: Platform - Run setup tools | |
| run: | | |
| cd build/dist/current | |
| deploy_dir=`pwd` | |
| mkdir metadata | |
| mkdir -p data/storage1 | |
| echo SECRET_KEY=xHjFeFlo$k= >> etc/env.sh | |
| sed -i "s#/path/to#${deploy_dir}#g" etc/trac-platform.yaml | |
| sed -i "s#jwtIssuer: trac_platform#disableAuth: true#" etc/trac-platform.yaml | |
| bin/secret-tool run --task init_secrets | |
| bin/secret-tool run --task create_root_auth_key EC 256 | |
| bin/deploy-metadb run --task deploy_schema | |
| bin/deploy-metadb run --task add_tenant ACME_CORP "ACME Rockets always get the Coyote!" | |
| - name: Platform - Start | |
| run: | | |
| cd build/dist/current | |
| bin/tracdap-svc-meta start | |
| bin/tracdap-svc-data start | |
| bin/tracdap-gateway start | |
| - name: Web API - Install dependencies | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm install | |
| - name: Web API - Set TRAC version | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm run tracVersion:posix | |
| - name: Web API - Build package | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm run buildApi | |
| - name: Examples - Install dependencies | |
| run: | | |
| cd examples/apps/javascript | |
| npm install | |
| - name: Examples - Run all | |
| run: | | |
| cd examples/apps/javascript | |
| npm run examples | |
| - name: Platform - Shut down | |
| if: always() | |
| run: | | |
| cd build/dist/current | |
| bin/tracdap-gateway stop | |
| bin/tracdap-svc-data stop | |
| bin/tracdap-svc-meta stop | |
| slow-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| java-version: ${{ env.JAVA_VERSION }} | |
| cache: gradle | |
| - name: Build | |
| run: ./gradlew testClasses | |
| - name: Slow tests | |
| run: ./gradlew integration -DintegrationTags="slow" | |
| # If the tests fail, make the output available for download | |
| - name: Store failed test results | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: junit-test-results | |
| path: build/modules/*/reports/** | |
| retention-days: 7 | |
| end-to-end: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Java - Setup | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| java-version: ${{ env.JAVA_VERSION }} | |
| # Turn on Gradle dependency caching | |
| cache: gradle | |
| - name: Java - Build platform | |
| run: ./gradlew testClasses | |
| - name: Python - setup | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Python - PIP | |
| run: python -m pip install --upgrade pip | |
| - name: Python - Install build dependencies | |
| run: | | |
| pip install -r tracdap-runtime/python/requirements.txt | |
| - name: Python - Build runtime package | |
| env: | |
| TRAC_PYTHON_BUILD_ISOLATION: false | |
| run: python tracdap-runtime/python/build_runtime.py --target dist | |
| - name: Run end to end tests | |
| run: ./gradlew integration -DintegrationTags="int-e2e" | |
| # If the tests fail, make the output available for download | |
| - name: Store failed test results | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: junit-test-results | |
| path: build/modules/*/reports/** | |
| retention-days: 7 | |
| int-sql: | |
| strategy: | |
| # Try to finish all jobs - it can be helpful to see if some succeed and others fail | |
| fail-fast: false | |
| matrix: | |
| database: | |
| - { DIALECT: mysql, | |
| DB_IMAGE: 'mysql:8.4', | |
| DB_PORT: 3306, | |
| DB_OPTIONS: '--health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3', | |
| MYSQL_ALLOW_EMPTY_PASSWORD: yes, | |
| MYSQL_DATABASE: trac, | |
| MYSQL_USER: trac_admin, | |
| MYSQL_PASSWORD: trac_admin, | |
| DB_SECRET: trac_admin } | |
| - { DIALECT: mariadb, | |
| DB_IMAGE: 'mariadb:11.4', | |
| DB_PORT: 3306, | |
| DB_OPTIONS: '--health-cmd="healthcheck.sh --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3', | |
| MYSQL_ALLOW_EMPTY_PASSWORD: yes, | |
| MYSQL_DATABASE: trac, | |
| MYSQL_USER: trac_admin, | |
| MYSQL_PASSWORD: trac_admin, | |
| DB_SECRET: trac_admin } | |
| - { DIALECT: postgresql, | |
| DB_IMAGE: 'postgres:16-alpine', | |
| DB_PORT: 5432, | |
| DB_OPTIONS: '--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5', | |
| POSTGRES_DB: trac, | |
| POSTGRES_USER: trac_admin, | |
| POSTGRES_PASSWORD: trac_admin, | |
| DB_SECRET: trac_admin,} | |
| - { DIALECT: sqlserver, | |
| DB_IMAGE: 'mcr.microsoft.com/mssql/server:2022-latest', | |
| DB_PORT: 1433, | |
| DB_OPTIONS: '-e "NO_DB_OPTIONS=not_used"', # docker run -e flag sets an env variable, passing '' causes errors | |
| MSSQL_PID: Developer, | |
| ACCEPT_EULA: Y, | |
| SA_PASSWORD: "tR4c_aDm!n", | |
| DB_SECRET: "tR4c_aDm!n" } | |
| uses: ./.github/workflows/integration-sql.yaml | |
| with: | |
| matrix: ${{ toJson( matrix.database ) }} | |
| dialect: ${{ matrix.database.DIALECT }} | |
| db_image: ${{ matrix.database.DB_IMAGE }} | |
| db_port: ${{ matrix.database.DB_PORT }} | |
| db_options: ${{ matrix.database.DB_OPTIONS }} | |
| int-cloud-storage: | |
| strategy: | |
| # Try to finish all jobs - it can be helpful to see if some succeed and others fail | |
| fail-fast: false | |
| matrix: | |
| storage: | |
| - { SERVICE: storage, | |
| TARGET: aws, | |
| PROTOCOL: S3, | |
| ENABLE_IF: TRAC_AWS_BUCKET, | |
| BUILD_aws_storage: true, | |
| INSTALL_SCRIPT: ".github/scripts/int-storage-s3-install.sh", | |
| SETUP_SCRIPT: ".github/scripts/int-storage-s3-setup.sh", | |
| TRAC_CONFIG_FILE: '.github/config/int-storage-s3.yaml', | |
| TRAC_SECRET_KEY: short-lived-secret, | |
| PYTHON_TESTS: int_storage_aws*.py, | |
| JAVA_TESTS: int-storage } | |
| # GCP does not support Python 3.12 yet, so run integration tests on 3.11 | |
| # See https://github.com/googleapis/python-crc32c/issues/178 (there may be more dependencies affected) | |
| - { SERVICE: storage, | |
| TARGET: gcp, | |
| PROTOCOL: GCS, | |
| ENABLE_IF: TRAC_GCP_BUCKET, | |
| BUILD_gcp_storage: true, | |
| INSTALL_SCRIPT: ".github/scripts/int-storage-gcs-install.sh", | |
| SETUP_SCRIPT: ".github/scripts/int-storage-gcs-setup.sh", | |
| TRAC_CONFIG_FILE: '.github/config/int-storage-gcs.yaml', | |
| TRAC_SECRET_KEY: short-lived-secret, | |
| PYTHON_TESTS: int_storage_gcp*.py, | |
| PYTHON_VERSION: "3.11", | |
| JAVA_TESTS: int-storage } | |
| - { SERVICE: storage, | |
| TARGET: azure, | |
| PROTOCOL: BLOB, | |
| ENABLE_IF: TRAC_AZURE_CONTAINER, | |
| BUILD_azure_storage: true, | |
| INSTALL_SCRIPT: ".github/scripts/int-storage-azure-install.sh", | |
| SETUP_SCRIPT: ".github/scripts/int-storage-azure-setup.sh", | |
| TRAC_CONFIG_FILE: '.github/config/int-storage-azure.yaml', | |
| TRAC_SECRET_KEY: short-lived-secret, | |
| PYTHON_TESTS: int_storage_azure*.py, | |
| JAVA_TESTS: int-storage } | |
| uses: ./.github/workflows/integration-cloud.yaml | |
| with: | |
| service: ${{ matrix.storage.SERVICE }} | |
| target: ${{ matrix.storage.TARGET }} | |
| protocol: ${{ matrix.storage.PROTOCOL }} | |
| matrix: ${{ toJson( matrix.storage ) }} | |
| enabled: ${{ vars[ matrix.storage.ENABLE_IF ] != '' }} | |
| has_java: ${{ matrix.storage.JAVA_TESTS != '' }} | |
| has_python: ${{ matrix.storage.PYTHON_TESTS != '' }} | |
| secrets: inherit | |
| # Permissions required to use IdP to connect to cloud platforms | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| contents: read # This is required for actions/checkout |