build-deploy-pudl #595
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-deploy-pudl | |
| on: | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - "v20*" | |
| schedule: | |
| - cron: "0 6 * * 1-5" # Weekdays at midnight on MST | |
| env: | |
| GCP_BILLING_PROJECT: ${{ secrets.GCP_BILLING_PROJECT }} | |
| BUILD_REF: ${{ github.ref_name }} # This is changed to dev if running on a schedule | |
| GCE_INSTANCE: pudl-deployment-tag # This is changed to pudl-deployment-dev if running on a schedule | |
| GCE_INSTANCE_ZONE: ${{ secrets.GCE_INSTANCE_ZONE }} | |
| GCS_OUTPUT_BUCKET: gs://nightly-build-outputs.catalyst.coop | |
| jobs: | |
| build_and_deploy_pudl: | |
| name: Build Docker image, push to Docker Hub and deploy to a GCE VM | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| id-token: write | |
| steps: | |
| - name: Use pudl-deployment-dev vm and dev branch if running on a schedule | |
| if: ${{ (github.event_name == 'schedule') }} | |
| run: | | |
| echo "This action was triggered by a schedule." | |
| echo "GCE_INSTANCE=pudl-deployment-dev" >> $GITHUB_ENV | |
| echo "BUILD_REF=dev" >> $GITHUB_ENV | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.BUILD_REF }} | |
| - name: Set action environment variables | |
| run: | | |
| echo "NIGHTLY_TAG=nightly-$(date +%Y-%m-%d)" >> $GITHUB_ENV | |
| echo "BUILD_ID=$(date +%Y-%m-%d-%H%M)-$(git rev-parse --short HEAD)-${BUILD_REF}" >> $GITHUB_ENV | |
| - name: Show freshly set envvars | |
| run: | | |
| echo "GCE_INSTANCE: $GCE_INSTANCE" | |
| echo "BUILD_REF: $BUILD_REF" | |
| echo "NIGHTLY_TAG: $NIGHTLY_TAG" | |
| echo "BUILD_ID: $BUILD_ID" | |
| - name: Tag nightly build | |
| # TESTING -- SHOULD ONLY RUN ON SCHEDULE | |
| if: ${{ (github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch') }} | |
| run: | | |
| git config user.email "pudl@catalyst.coop" | |
| git config user.name "pudlbot" | |
| git tag -a -m "$NIGHTLY_TAG" $NIGHTLY_TAG $BUILD_REF | |
| git push origin $NIGHTLY_TAG | |
| - name: Docker Metadata | |
| id: docker_metadata | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: catalystcoop/pudl-etl | |
| flavor: | | |
| latest=auto | |
| tags: | | |
| type=raw,value=${{ env.BUILD_REF}} | |
| type=ref,event=tag | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to DockerHub | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build image and push to Docker Hub | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: docker/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.docker_metadata.outputs.tags }} | |
| labels: ${{ steps.docker_metadata.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - id: "auth" | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider" | |
| service_account: "deploy-pudl-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com" | |
| # Setup gcloud CLI | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| # Deploy PUDL image to GCE | |
| - name: Deploy | |
| env: | |
| DAGSTER_PG_PASSWORD: ${{ secrets.DAGSTER_PG_PASSWORD }} | |
| PUDL_OUTPUT_PATH: ${{ env.GCS_OUTPUT_BUCKET }}/${{ env.BUILD_ID }} | |
| run: |- | |
| gcloud compute instances add-metadata "$GCE_INSTANCE" \ | |
| --zone "$GCE_INSTANCE_ZONE" \ | |
| --metadata-from-file startup-script=./docker/vm_startup_script.sh | |
| gcloud compute instances update-container "$GCE_INSTANCE" \ | |
| --zone "$GCE_INSTANCE_ZONE" \ | |
| --container-image "docker.io/catalystcoop/pudl-etl:${{ env.BUILD_REF}}" \ | |
| --container-command "micromamba" \ | |
| --container-arg="run" \ | |
| --container-arg="--prefix" \ | |
| --container-arg="/home/mambauser/env" \ | |
| --container-arg="--attach" \ | |
| --container-arg='' \ | |
| --container-arg="bash" \ | |
| --container-arg="./docker/gcp_pudl_etl.sh" \ | |
| --container-env-file="./docker/.env" \ | |
| --container-env BUILD_REF=${{ env.BUILD_REF}} \ | |
| --container-env BUILD_ID=${{ env.BUILD_ID }} \ | |
| --container-env NIGHTLY_TAG=${{ env.NIGHTLY_TAG }} \ | |
| --container-env GITHUB_ACTION_TRIGGER=${{ github.event_name }} \ | |
| --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \ | |
| --container-env GCE_INSTANCE=${{ env.GCE_INSTANCE }} \ | |
| --container-env GCE_INSTANCE_ZONE=${{ env.GCE_INSTANCE_ZONE }} \ | |
| --container-env GCP_BILLING_PROJECT=${{ secrets.GCP_BILLING_PROJECT }} \ | |
| --container-env AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| --container-env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| --container-env AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION }} \ | |
| --container-env DAGSTER_PG_USERNAME="postgres" \ | |
| --container-env DAGSTER_PG_PASSWORD="$DAGSTER_PG_PASSWORD" \ | |
| --container-env DAGSTER_PG_HOST="104.154.182.24" \ | |
| --container-env DAGSTER_PG_DB="dagster-storage" \ | |
| --container-env FLY_ACCESS_TOKEN=${{ secrets.FLY_ACCESS_TOKEN }} \ | |
| --container-env PUDL_BOT_PAT=${{ secrets.PUDL_BOT_PAT }} \ | |
| --container-env ZENODO_SANDBOX_TOKEN_PUBLISH=${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} \ | |
| --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_full.yml" \ | |
| --container-env PUDL_GCS_OUTPUT=${{ env.PUDL_OUTPUT_PATH }} | |
| # Start the VM | |
| - name: Start the deploy-pudl-vm | |
| run: gcloud compute instances start "$GCE_INSTANCE" --zone="$GCE_INSTANCE_ZONE" | |
| - name: Post to a pudl-deployments channel | |
| id: slack | |
| uses: slackapi/slack-github-action@v1 | |
| with: | |
| channel-id: "C03FHB9N0PQ" | |
| slack-message: "build-deploy-pudl status: ${{ job.status }}\n${{ env.GCS_OUTPUT_BUCKET }}/${{ env.BUILD_ID }}" | |
| env: | |
| channel-id: "C03FHB9N0PQ" | |
| SLACK_BOT_TOKEN: ${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} |