scan-for-updates.yml

#
# This workflow is run regularly and scans for package updates, by
# downloading the file specified by the `PackageInfoURL` in each package
# and comparing it against what was there already.
#
# Then for each package where a diff is found, a new `CI` job is created,
# see `create-pull-request` below, which creates a pull request updating
# the package. Note that we perform no validation of the update here;
# that is a task for the GItHub workflow in `pull-request.yml`.
#
name: "Scan for updates"

on:
  workflow_dispatch:  # for debugging
    inputs:
      pkginfo_urls:
        description: 'A space separated list of PackageInfo.g URLs'
        required: false
        type: string
        default: ''
  schedule:
    - cron: '3 * * * *' # Every hours at 3 past

jobs:
  scan-for-updates:
    name: "Scan for package updates"
    runs-on: ubuntu-20.04
    outputs:
      matrix: ${{ steps.get-names.outputs.matrix }}
    steps:
      - uses: actions/checkout@v4

      # Cache the package archives we download across runs of this job
      # to speed up things considerably
      - name: "Cache archives"
        uses: actions/cache@v4
        with:
          path: _archives
          key: archives-${{ hashFiles('packages/*/meta.json') }}
          restore-keys: |
            archives-

      # the following step is useful for debugging
      - name: "Determine system architecture for -march=native"
        run: |
          echo "cc -march=native: $(tools/compiler_arch.sh cc)"
          echo "c++ -march=native: $(tools/compiler_arch.sh c++)"

      # Setup ccache, to speed up repeated compilation of the same binaries
      # (i.e., GAP and the packages)
      - name: "Setup ccache"
        uses: Chocobo1/setup-ccache-action@v1
        with:
          update_packager_index: false

      - name: "Install GAP"
        uses: gap-actions/setup-gap@v2
        with:
          GAP_BOOTSTRAP: 'minimal'
          GAP_PKGS_TO_CLONE: 'crypting json'
          GAP_PKGS_TO_BUILD: 'crypting json'

      - name: "Set up Python"
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: "Install prerequisites for package distribution tools"
        run: python -m pip install -r tools/requirements.txt

      - name: "Scan for updates"
        run: tools/scan_for_updates.py

      - name: "Import user specified packages"
        run: |
          tools/import_packages.py ${{ github.event.inputs.pkginfo_urls }}

      # remove archives we don't need anymore; this reduces the size of the caches
      - name: "Cleanup archives"
        run: tools/cleanup_archives.py

      # The following builds a job matrix; we launch a copy of the "create-pull-request"
      # for each package whose meta.json was modified or added
      - name: "Create jobs for modified packages"
        id: get-names
        run: |
            # detect all added or modified files in the `packages` directory
            modified=$(git status --porcelain -uall -- packages/*/meta.json | cut -c4-)

            # create an archive of the added or modified files, to be uploaded as an
            # artifact in the next step, and for use by subsequent jobs
            # HACK: add 'LICENSE' to the files in the archive, to avoid
            # creating an empty archive when there are no modified files
            tar cvf modified.tar.gz $modified LICENSE

            # create the actual build matrix
            jobnames=$(tools/group_packages.py $modified)
            MATRIX="{\"package_or_group\":["
            for PKG in ${jobnames}; do
              echo "${PKG}"
              MATRIX="${MATRIX}\"${PKG}\","
            done
            MATRIX="${MATRIX}]}"
            echo "matrix=$MATRIX" >> $GITHUB_OUTPUT

      - name: "Upload metadata as artifact for pull request jobs"
        uses: actions/upload-artifact@v4
        with:
          name: metadata
          path: modified.tar.gz

  create-pull-request:
    name: "Create pull request for ${{ matrix.package_or_group }}"
    if: ${{ needs.scan-for-updates.outputs.matrix != '{"package_or_group":[]}' }}
    needs: scan-for-updates
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.scan-for-updates.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v4

      - name: "Download metadata from previous job"
        uses: actions/download-artifact@v4
        with:
          name: metadata

      - name: "Prepare pull request"
        run: |
          tar xvf modified.tar.gz
          # detect all added or modified files in the `packages` directory
          modified=$(git status --porcelain -uall -- packages/*/meta.json | cut -c4-)
          echo "modified:"
          echo "$modified"
          tools/prepare_pr.py ${{ matrix.package_or_group }} $modified >> $GITHUB_ENV

          cat $GITHUB_ENV

      # We set up a GitHub App in order to ensure that workflows are run on the PRs
      # created by us, following the instructions here:
      # <https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#authenticating-with-github-app-generated-tokens>
      # The bot can also be set up for personal forks of this repository, for testing
      # and debugging, see <https://github.com/apps/gap-package-distribution-bot>.
      - uses: tibdex/github-app-token@v1
        id: generate-token
        with:
          app_id: ${{ secrets.APP_ID }}
          private_key: ${{ secrets.APP_PRIVATE_KEY }}

      - uses: peter-evans/create-pull-request@v6
        with:
          token: ${{ steps.generate-token.outputs.token }}
          # https://api.github.com/users/gap-package-distribution-bot%5Bbot%5D
          author: "gap-package-distribution-bot <100730870+gap-package-distribution-bot[bot]@users.noreply.github.com>"
          add-paths: ${{ env.PR_FILES }}
          commit-message: "${{ env.PR_TITLE }}"
          body: ${{ env.PR_BODY }}
          branch: automatic/${{ matrix.package_or_group }}
          delete-branch: true
          title: "${{ env.PR_TITLE }}"
          labels: |
            automated pr
            ${{ env.PR_LABEL }}