From 2a06871b400f58f50c36efba09570fa8fc607486 Mon Sep 17 00:00:00 2001 From: Zane Selvans Date: Tue, 14 Nov 2023 12:45:26 -0600 Subject: [PATCH] Bring renamed/scheduled workflow files into main from dev. --- .../workflows/{tox-pytest.yml => pytest.yml} | 59 +++++-------- .github/workflows/update-conda-lockfile.yml | 84 +++++++++++++++++++ .github/workflows/update-lockfile.yml | 57 ------------- 3 files changed, 107 insertions(+), 93 deletions(-) rename .github/workflows/{tox-pytest.yml => pytest.yml} (81%) create mode 100644 .github/workflows/update-conda-lockfile.yml delete mode 100644 .github/workflows/update-lockfile.yml diff --git a/.github/workflows/tox-pytest.yml b/.github/workflows/pytest.yml similarity index 81% rename from .github/workflows/tox-pytest.yml rename to .github/workflows/pytest.yml index 27e8737459..b808b84c42 100644 --- a/.github/workflows/tox-pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,5 +1,5 @@ --- -name: tox-pytest +name: pytest on: pull_request: @@ -10,19 +10,15 @@ on: - ready_for_review env: - PUDL_OUTPUT: /home/runner/pudl-work/output - PUDL_INPUT: /home/runner/pudl-work/data/ + PUDL_OUTPUT: /home/runner/pudl-work/output/ + PUDL_INPUT: /home/runner/pudl-work/input/ DAGSTER_HOME: /home/runner/pudl-work/dagster_home/ jobs: - ci-static: + ci-docs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - tox-env: - - linters - - docs defaults: run: shell: bash -l {0} @@ -32,16 +28,12 @@ jobs: with: fetch-depth: 2 - - name: Install Conda environment using mamba + - name: Install conda-lock environment with micromamba uses: mamba-org/setup-micromamba@v1 with: - environment-file: test/test-environment.yml + environment-file: environments/conda-lock.yml + environment-name: pudl-dev cache-environment: true - condarc: | - channels: - - conda-forge - - defaults - channel_priority: strict - name: Log environment details run: | @@ -51,13 +43,13 @@ jobs: conda config --show printenv | sort - - name: Build ${{ matrix.tox-env}} with Tox + - name: Lint and build PUDL documentation with Sphinx run: | - tox -e ${{ matrix.tox-env }} + pip install --no-deps --editable . + make docs-build - name: Upload coverage uses: actions/upload-artifact@v3 - if: ${{ matrix.tox-env == 'docs' }} with: name: coverage-docs path: coverage.xml @@ -75,16 +67,12 @@ jobs: with: fetch-depth: 2 - - name: Install Conda environment using mamba + - name: Install conda-lock environment with micromamba uses: mamba-org/setup-micromamba@v1 with: - environment-file: test/test-environment.yml + environment-file: environments/conda-lock.yml + environment-name: pudl-dev cache-environment: true - condarc: | - channels: - - conda-forge - - defaults - channel_priority: strict - name: Log environment details run: | @@ -99,9 +87,10 @@ jobs: which sqlite3 sqlite3 --version - - name: Run unit tests with Tox + - name: Run PUDL unit tests and collect test coverage run: | - tox -e unit -- --durations 0 + pip install --no-deps --editable . + make pytest-unit - name: Upload coverage uses: actions/upload-artifact@v3 @@ -131,13 +120,9 @@ jobs: - name: Install Conda environment using mamba uses: mamba-org/setup-micromamba@v1 with: - environment-file: test/test-environment.yml + environment-file: environments/conda-lock.yml + environment-name: pudl-dev cache-environment: true - condarc: | - channels: - - conda-forge - - defaults - channel_priority: strict - name: Log environment details run: | @@ -180,7 +165,8 @@ jobs: - name: Run integration tests, trying to use GCS cache if possible run: | - tox -e integration -- --gcs-cache-path=gs://zenodo-cache.catalyst.coop --durations 0 + pip install --no-deps --editable . + make pytest-integration - name: Upload coverage uses: actions/upload-artifact@v3 @@ -194,13 +180,13 @@ jobs: ci-coverage: runs-on: ubuntu-latest needs: + - ci-docs - ci-unit - ci-integration - - ci-static steps: - uses: actions/checkout@v4 - name: Download coverage - id: download-unit + id: download-coverage uses: actions/download-artifact@v3 with: path: coverage @@ -216,6 +202,7 @@ jobs: runs-on: ubuntu-latest if: ${{ always() }} needs: + - ci-docs - ci-unit - ci-integration steps: diff --git a/.github/workflows/update-conda-lockfile.yml b/.github/workflows/update-conda-lockfile.yml new file mode 100644 index 0000000000..21a839f727 --- /dev/null +++ b/.github/workflows/update-conda-lockfile.yml @@ -0,0 +1,84 @@ +--- +name: update-conda-lockfile + +on: + workflow_dispatch: + schedule: + - cron: "0 9 * * 1-5" # Weekdays at 9AM UTC + push: + paths: + - "pyproject.toml" + - "environments/*" + - ".github/workflows/update-conda-lockfile.yml" + +# What branch does this action run on? +# - workflow_dispatch: Whatever branch it was run against. +# - schedule: Always the same branch (will be dev or main) +# - push: Base branch of the PR. + +jobs: + update-conda-lockfile: + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'push' && github.actor != 'pudlbot') || (github.event_name == 'schedule' && github.repository == 'catalyst-cooperative/pudl') || (github.event_name == 'workflow_dispatch') }} + defaults: + run: + shell: bash -l {0} + steps: + - name: Set GITHUB_REF for use with workflow_dispatch + if: ${{ (github.event_name == 'workflow_dispatch') }} + run: | + echo "GITHUB_REF="${{ github.ref_name }} >> $GITHUB_ENV + - name: Set GITHUB_REF for use with schedule + if: ${{ (github.event_name == 'schedule') }} + run: | + echo "GITHUB_REF=dev" >> $GITHUB_ENV + - name: Set GITHUB_REF for use with push + if: ${{ (github.event_name == 'push') }} + run: | + echo "GITHUB_REF="${{ github.ref_name }} >> $GITHUB_ENV + - name: Log final value of GITHUB_REF + run: | + echo "Final GITHUB_REF:" ${{ env.GITHUB_REF }} + - uses: actions/checkout@v4 + with: + token: ${{ secrets.PUDL_BOT_PAT }} + ref: ${{ env.GITHUB_REF }} + - name: Install Micromamba + uses: mamba-org/setup-micromamba@v1 + with: + environment-name: conda-lock + create-args: >- + python=3.11 + conda-lock + prettier + - name: Run conda-lock to recreate lockfile from scratch + run: | + make conda-clean + make conda-lock.yml + - name: Commit updated conda lockfiles to branch + # If running on push due to dependency changes, commit directly to the base + # branch of the existing PR. Don't trigger the workflow again if we're already + # running it as pudlbot (to avoid infinite recursion). + if: ${{ (github.event_name == 'push' && github.actor != 'pudlbot') }} + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: "environments/*" + commit_message: "Update conda-lock.yml and rendered conda environment files." + - name: Make a PR to merge updated conda lockfiles + # If we are relocking dependencies on a schedule or workflow_dispatch, we need + # to make our own PR to check whether the updated environment actually solves + # and the tests pass. + if: ${{ (github.event_name == 'schedule' && github.repository == 'catalyst-cooperative/pudl') || (github.event_name == 'workflow_dispatch') }} + uses: peter-evans/create-pull-request@v5 + with: + commit-message: "Update conda-lock.yml and rendered conda environment files." + title: Update Lockfile + body: > + This pull request relocks the dependencies with conda-lock. + It is triggered by [update-conda-lockfile](https://github.com/catalyst-cooperative/pudl/blob/main/.github/workflows/update-conda-lockfile.yml). + labels: dependencies, conda-lock + reviewers: zaneselvans + branch: update-conda-lockfile + base: ${{ env.GITHUB_REF }} + draft: true + delete-branch: true diff --git a/.github/workflows/update-lockfile.yml b/.github/workflows/update-lockfile.yml deleted file mode 100644 index 42d9ec0393..0000000000 --- a/.github/workflows/update-lockfile.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -name: update-lockfile - -on: - workflow_dispatch: - # schedule: - # At 5:28am UTC Monday and Thursday - # - cron: 28 5 * * MON,THU - -jobs: - conda-lock: - # Don't run scheduled job on forks. - if: (github.event_name == 'schedule' && github.repository == 'catalyst-cooperative/pudl') || (github.event_name != 'schedule') - defaults: - run: - # Ensure the environment is activated - # - shell: bash -l {0} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install Micromamba - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: environments/conda-lock.yml - environment-name: pudl-dev - - - name: Install pudl from branch - run: pip install --editable "./[dev,docs,test,datasette]" - - - name: Run conda-lock to recreate lockfile from scratch - run: | - rm environments/conda-lock.yml - conda-lock \ - --file=environments/dev-environment.yml \ - --file=pyproject.toml \ - --lockfile=environments/conda-lock.yml - - name: Open a pull request - uses: peter-evans/create-pull-request@v5 - with: - # # The default GITHUB_TOKEN doesn't allow other workflows to trigger. - # # Thus if there are tests to be run, they won't be run. For more info, - # # see the note under - # # . - # # One possible workaround is to specify a Personal Access Token (PAT). - # # This PAT should have read-write permissions for "Pull Requests" - # # and read-write permissions for "Contents". - # token: ${{ secrets.GH_PAT_FOR_PR }} - commit-message: Update lockfile - title: Update Lockfile - body: > - This pull request relocks the dependencies with conda-lock. - It is triggered by [update-lockfile](https://github.com/catalyst-cooperative/pudl/blob/main/.github/workflows/update-lockfile.yml). - branch: update-lockfile - labels: dependencies, conda-lock - reviewers: zaneselvans - delete-branch: true