cookies, hashing, retries, redis improvements

.github/actions/install-python/action.yml

@@ -0,0 +1,54 @@
+name: Setup Python
+description: "Installs python, setting up caching."
+inputs:
+  pdm:
+    description: "Whether pdm/python packages need installing as well as base python or not."
+    required: false
+    default: "false"
+runs:
+  using: composite
+  steps:
+    - name: Hack setup-python cache
+      # Create an empty requirements.txt if this file (or pyproject.toml) doesn't exist.
+      # This work around and issue with setup-python for non-Python projects, which ends up with ends up
+      # with this error:
+      #
+      #   Run actions/setup-python@v5.0.0
+      #     with:
+      #       python-version: 3.12
+      #       cache: pip
+      #   Installed versions
+      #     Successfully set up CPython (3.12.1)
+      #   Error: No file in /home/runner/work/awesome-iam/awesome-iam matched to
+      #   [**/requirements.txt or **/pyproject.toml], make sure you have checked out the target repository
+      #
+      # This has been reported at: https://github.com/actions/setup-python/issues/807
+      # In the future this might be addressed by: https://github.com/actions/setup-python/pull/762
+      # or https://github.com/actions/setup-python/issues/751
+      if: hashFiles('**/requirements.txt', '**/pyproject.toml') == ''
+      shell: bash
+      run: |
+        touch ./requirements.txt
+    - name: Get python binary
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.12"
+        cache: pip
+    - name: Always install zetch
+      shell: bash
+      run: |
+        pip install zetch
+    - uses: actions/checkout@v4
+      if: ${{ inputs.pdm == 'true' }}
+    - name: Set up PDM
+      if: ${{ inputs.pdm == 'true' }}
+      uses: pdm-project/setup-pdm@v3
+      with:
+        python-version: "3.12"
+        cache: true
+        cache-dependency-path: 'py/pdm.lock'
+    - name: Install dependencies
+      if: ${{ inputs.pdm == 'true' }}
+      shell: bash
+      run: |
+        pdm sync -p ./py -G:all

.github/actions/install-redis/action.yml

@@ -0,0 +1,10 @@
+name: install redis
+description: "install redis without starting a server (ONLY IF NOT WINDOWS)"
+runs:
+  using: composite
+  steps:
+    - uses: shogo82148/actions-setup-redis@v1
+      if: runner.os != 'Windows'
+      with:
+        redis-version: "latest"
+        auto-start: false

.github/actions/install-rust/action.yml

@@ -1,6 +1,8 @@
 name: Setup Rust
 description: "Installs latest stable rust, and sets up sscache for caching."
 inputs:
+  secret_config_toml:
+    description: "The CUSTOM_RUST_CONFIG_TOML secret if it exists."
   qa:
     description: "Whether things like cargo-hack need installing."
     required: false
@@ -16,22 +18,33 @@ runs:
       uses: dtolnay/rust-toolchain@stable
       with:
         components: rustfmt, clippy
+    # This will store compiled crates, allowing them to be reused between runs.
+    # The action caller will have a post job setup that will print a summary of sccache usage during the run.
     - name: Run sccache-cache
       uses: mozilla-actions/sccache-action@v0.0.4
     - name: Set Rust caching env vars
       shell: bash
       run: |
         echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
         echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
-    # Really this would need to be run as a post task to be of any use (to actually see hits),
-    # but post in composite action not currently supported:
-    # https://github.com/actions/runner/issues/1478
-    # - name: Run sccache stat for check
-    #   shell: bash
-    #   run: ${SCCACHE_PATH} --show-stats
     - name: "Install cargo-hack, used for feature checking in pre-commit."
       if: ${{ inputs.qa == 'true' }}
       uses: taiki-e/install-action@cargo-hack
     - name: Install nextest
       if: ${{ inputs.test == 'true' }}
       uses: taiki-e/install-action@nextest
+    - name: Setup default global config.toml
+      shell: bash
+      if: ${{ inputs.secret_config_toml == '' }}
+      run: |
+        mkdir -p ~/.cargo
+        echo '[build]' > ~/.cargo/config.toml
+        echo '[registries]' >> ~/.cargo/config.toml
+    # If CUSTOM_RUST_CONFIG_TOML is set, it will be copied to the global config.toml
+    - name: Setup custom global config.toml
+      shell: bash
+      if: ${{ inputs.secret_config_toml != '' }}
+      run: |
+        mkdir -p ~/.cargo
+        echo '${{ inputs.secret_config_toml }}' > ~/.cargo/config.toml
+

.github/workflows/py-rust-build.yml

@@ -18,7 +18,10 @@ jobs:
     strategy:
       fail-fast: true  # Want release to cancel if any fail, so may as well fail fast to save some minutes
       matrix:
-        os: [linux, macos, windows]
+        os:
+          - linux
+          - macos
+          - windows
         target: [x86_64, aarch64]
         manylinux: [auto]
         include:

.github/workflows/release.yml

@@ -53,29 +53,34 @@ jobs:
         uses: actions/checkout@v4
         with:
           token: ${{ secrets.VERSION_BOT_PAT }}
-      - uses: actions/setup-python@v4
 
-      - name: Install zetch
-        run: pip install zetch
+        # Always installing python and bun, quick and so often randomly needed:
+      - uses: ./.github/actions/install-python
+        with:
+          pdm: true
+      - name: Install Bun, no npm should be needed
+        uses: oven-sh/setup-bun@v1
+        with:
+          bun-version: latest
 
-      - name: Set up PDM
-        if: ${{ inputs.py_release }}
-        uses: pdm-project/setup-pdm@v3
+      - uses: ./.github/actions/install-rust
+        if: ${{ inputs.py_rust_release }} || ${{ inputs.rust_release }}
         with:
-          python-version: '3.12'
-          cache: true
-          cache-dependency-path: py/pdm.lock
+          secret_config_toml: ${{ secrets.CUSTOM_RUST_CONFIG_TOML }}
+          qa: true
+
+        # Need to checkout with PAT again, in-case above custom actions checked out with unpermissioned token
+      - name: Checks out repo with a token that allows pushing to main without PR
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.VERSION_BOT_PAT }}
+
       - name: Update Python version
         if: ${{ inputs.py_release }}
         run: |
           zetch put zetch.config.toml context.static.PY_VERSION ${{ inputs.py_version }}
 
-          # Js project
-      - name: Install Bun, no npm should be needed
-        if: ${{ inputs.js_release }}
-        uses: oven-sh/setup-bun@v1
-        with:
-          bun-version: latest
+        # Js project
       - name: Install js dependencies
         if: ${{ inputs.js_release }}
         run: |
@@ -86,11 +91,6 @@ jobs:
         run: |
           zetch put zetch.config.toml context.static.JS_VERSION ${{ inputs.js_version }}
 
-      - uses: ./.github/actions/install-rust
-        if: ${{ inputs.py_rust_release }} || ${{ inputs.rust_release }}
-        with:
-          qa: true
-
       - name: Update Rust-backed Python version
         if: ${{ inputs.py_rust_release }}
         run: |
@@ -139,6 +139,11 @@ jobs:
         with:
           ref: ${{ needs.commit_versions.outputs.new-sha }}
 
+        # Basic python always needed:
+      - uses: ./.github/actions/install-python
+        with:
+          pdm: false
+
       - name: Set up PDM
         uses: pdm-project/setup-pdm@v3
         with:
@@ -156,6 +161,8 @@ jobs:
           node-version: '20'
 
       - uses: ./.github/actions/install-rust
+        with:
+          secret_config_toml: ${{ secrets.CUSTOM_RUST_CONFIG_TOML }}
 
       - name: Build docs
         run: |
@@ -284,9 +291,9 @@ jobs:
           name: py_rust_build_files
           path: py_rust/dist
 
-      - uses: actions/setup-python@v4
+      - uses: ./.github/actions/install-python
         with:
-          python-version: '3.12'
+          pdm: false
       - run: pip install twine
 
       - name: Make sure release contents seem valid
@@ -322,8 +329,22 @@ jobs:
         with:
           ref: ${{ needs.commit_versions.outputs.new-sha }}
 
+        # Basic python always needed:
+      - uses: ./.github/actions/install-python
+        with:
+          pdm: false
+      - name: Install zetch
+        run: pip install zetch
 
       - uses: ./.github/actions/install-rust
+        with:
+          secret_config_toml: ${{ secrets.CUSTOM_RUST_CONFIG_TOML }}
+
+        # Doing again as default checkout might have happened in one of the actions:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.commit_versions.outputs.new-sha }}
+
       - name: upload to Crates.io
         run: |
           cd rust

.github/workflows/tests.yml

@@ -34,23 +34,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: ./.github/actions/install-python
         with:
-          python-version: "3.12"
-          cache: pip
-
-      # Python project
-      - name: Set up PDM
-        uses: pdm-project/setup-pdm@v3
-        with:
-          python-version: "3.12"
-          cache: true
-          cache-dependency-path: 'py/pdm.lock'
-
-      - name: Install dependencies
-        run: |
-          pdm sync -p ./py -G:all
-
+          pdm: true
       # Js project
       - name: "Install Bun, no npm should be needed"
         uses: oven-sh/setup-bun@v1
@@ -63,6 +49,7 @@ jobs:
 
       - uses: ./.github/actions/install-rust
         with:
+          secret_config_toml: ${{ secrets.CUSTOM_RUST_CONFIG_TOML }}
           qa: true
 
       - uses: ./.github/actions/install-pre-commit
@@ -82,6 +69,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      # Basic python always needed:
+      - uses: ./.github/actions/install-python
+        with:
+          pdm: false
+
       - name: Set up PDM
         uses: pdm-project/setup-pdm@v3
         with:
@@ -96,6 +88,7 @@ jobs:
 
       - uses: ./.github/actions/install-rust
         with:
+          secret_config_toml: ${{ secrets.CUSTOM_RUST_CONFIG_TOML }}
           qa: true
 
       - name: Install dependencies
@@ -126,6 +119,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - uses: ./.github/actions/install-redis
+
       - name: Set up PDM
         uses: pdm-project/setup-pdm@v3
         with:
@@ -158,6 +153,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - uses: ./.github/actions/install-redis
+
       # Js project
       - name: "Install Bun, no npm should be needed"
         uses: oven-sh/setup-bun@v1
@@ -192,14 +189,17 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: ./.github/actions/install-rust
-        with:
-          test: true
+      - uses: ./.github/actions/install-redis
 
       - uses: actions/setup-python@v4
         with:
           python-version: "${{ matrix.python }}"
 
+      - uses: ./.github/actions/install-rust
+        with:
+          secret_config_toml: ${{ secrets.CUSTOM_RUST_CONFIG_TOML }}
+          test: true
+
       - name: Setup venv (problems with the automatic creating in scripts as it uses pipx and for some
           reason ends with wrong py version)
         run: |
@@ -227,12 +227,22 @@ jobs:
       matrix:
         # Run on both linux and windows (dev testing on mac and much more expensive in terms of build minutes),
         # windows most likely to fail unexpectedly.
-        os: [ubuntu-latest, windows-latest]
+        os:
+          - ubuntu-latest
+          - windows-latest
     steps:
       - uses: actions/checkout@v4
 
+      - uses: ./.github/actions/install-redis
+
+      # Basic python always needed:
+      - uses: ./.github/actions/install-python
+        with:
+          pdm: false
+
       - uses: ./.github/actions/install-rust
         with:
+          secret_config_toml: ${{ secrets.CUSTOM_RUST_CONFIG_TOML }}
           test: true
 
       - name: Run tests (linux)

dev_scripts/initial_setup.sh

@@ -332,6 +332,17 @@ initial_setup () {
     ./dev_scripts/py_rust.sh ensure_venv
 
 
+    echo "Make sure redis installed..."
+    if command -v redis-server > /dev/null 2>&1; then
+        echo "redis-server already installed"
+    else
+        echo "redis-server could not be found, installing..."
+        if [ "$(uname)" == "Darwin" ]; then
+            brew install redis
+        elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
+            sudo apt-get install redis-server
+        fi
+    fi
 
 }