Merge branch 'master' into ao-fix-parainclusion-weight-overestimation

* master: (51 commits)
  Remove unused feature gated code from the minimal template (#5237)
  make polkadot-parachain startup errors pretty (#5214)
  Coretime auto-renew (#4424)
  network/strategy: Backoff and ban overloaded peers to avoid submitting the same request multiple times (#5029)
  Fix frame crate usage doc (#5222)
  beefy: Tolerate pruned state on runtime API call (#5197)
  rpc: Enable ChainSpec for polkadot-parachain (#5205)
  Add an adapter for configuring AssetExchanger (#5130)
  Replace env_logger with sp_tracing (#5065)
  Adjust sync templates flow to use new release branch (#5182)
  litep2p/discovery: Publish authority records with external addresses only (#5176)
  Run UI tests in CI for some other crates (#5167)
  Remove `pallet::getter` usage from the pallet-balances (#4967)
  pallet-timestamp: `UnixTime::now` implementation logs error only if called at genesis (#5055)
  [CI] Cache try-runtime check (#5179)
  [Backport] version bumps and the prdocs reordering from stable2407 (#5178)
  [subsystem-benchmark] Update availability-distribution-regression-bench baseline after recent subsystem changes (#5180)
  Remove pallet::getter usage from proxy (#4963)
  Remove pallet::getter macro usage from pallet-election-provider-multi-phase (#4487)
  Review-bot@2.6.0 (#5177)
  ...

.config/zepter.yaml

@@ -27,7 +27,7 @@ workflows:
       ]
   # The umbrella crate uses more features, so we to check those too:
   check_umbrella:
-    - [ $check.0, '--features=serde,experimental,with-tracing,tuples-96,with-tracing', '-p=polkadot-sdk' ]
+    - [ $check.0, '--features=serde,experimental,runtime,with-tracing,tuples-96,with-tracing', '-p=polkadot-sdk' ]
   # Same as `check_*`, but with the `--fix` flag.
   default:
     - [ $check.0, '--fix' ]

.github/dependabot.yml

@@ -5,13 +5,17 @@ updates:
     directory: '/'
     labels: ["A1-insubstantial", "R0-silent"]
     schedule:
-      interval: daily
+      interval: weekly
+    groups:
+      ci_dependencies:
+        patterns:
+        - "*"
   # Update Rust dependencies:
   - package-ecosystem: "cargo"
     directory: "/"
     labels: ["A1-insubstantial", "R0-silent"]
     schedule:
-      interval: "daily"
+      interval: "weekly"
     groups:
       # We assume these crates to be semver abiding and can therefore group them together.
       known_good_semver:

.github/workflows/check-licenses.yml

@@ -16,7 +16,7 @@ jobs:
       NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Checkout sources
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - uses: actions/setup-node@v4.0.3
         with:
           node-version: "18.x"

.github/workflows/check-links.yml

@@ -18,14 +18,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Restore lychee cache
-        uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.2 (7. Sep 2023)
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v3.3.2 (7. Sep 2023)
         with:
           path: .lycheecache
           key: cache-lychee-${{ github.sha }}
           # This should restore from the most recent one:
           restore-keys: cache-lychee-
 
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.0 (22. Sep 2023)
 
       - name: Lychee link checker
         uses: lycheeverse/lychee-action@2b973e86fc7b1f6b36a93795fe2c9c6ae1118621 # for v1.9.1 (10. Jan 2024)

.github/workflows/check-prdoc.yml

@@ -20,7 +20,7 @@ jobs:
     if: github.event.pull_request.number != ''
     steps:
       - name: Checkout repo
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
       # we cannot show the version in this step (ie before checking out the repo)
       # due to https://github.com/paritytech/prdoc/issues/15
       - name: Check if PRdoc is required

.github/workflows/check-runtime-migration.yml

@@ -7,6 +7,8 @@ on:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
   merge_group:
+  workflow_dispatch:
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -27,7 +29,9 @@ jobs:
   # rococo and westend are disabled for now (no access to parity-chains.parity.io)
   check-runtime-migration:
     runs-on: arc-runners-polkadot-sdk-beefy
-    timeout-minutes: 40
+    # We need to set this to rather long to allow the snapshot to be created, but the average time
+    # should be much lower.
+    timeout-minutes: 60
     needs: [set-image]
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
@@ -94,22 +98,45 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: script
-        run: |
-          echo "Running ${{ matrix.network }} runtime migration check"
-          export RUST_LOG=remote-ext=debug,runtime=debug
 
-          echo "---------- Downloading try-runtime CLI ----------"
-          curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.5.4/try-runtime-x86_64-unknown-linux-musl -o try-runtime
+      - name: Download CLI
+        run: |
+          curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.7.0/try-runtime-x86_64-unknown-linux-musl -o try-runtime
           chmod +x ./try-runtime
           echo "Using try-runtime-cli version:"
           ./try-runtime --version
 
+      - name: Get Date
+        id: get-date
+        run: |
+          echo "today=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Download Snapshot
+        uses: actions/cache@v4
+        with:
+          path: snapshot.raw
+          key: try-runtime-snapshot-${{ matrix.network }}-${{ steps.get-date.outputs.today }}
+          save-always: true
+
+      - name: Create Snapshot If Stale
+        if: ${{ hashFiles('snapshot.raw') == '' }}
+        run: |
+          echo "Creating new snapshot for today (${{ steps.get-date.outputs.today }})"
+          ./try-runtime create-snapshot --uri ${{ matrix.uri }} snapshot.raw
+
+      - name: Build Runtime
+        run: |
           echo "---------- Building ${{ matrix.package }} runtime ----------"
-          time forklift cargo build --release --locked -p ${{ matrix.package }} --features try-runtime
+          time forklift cargo build --release --locked -p ${{ matrix.package }} --features try-runtime -q
+
+      - name: Run Check
+        run: |
+          echo "Running ${{ matrix.network }} runtime migration check"
+          export RUST_LOG=remote-ext=debug,runtime=debug
 
           echo "---------- Executing on-runtime-upgrade for ${{ matrix.network }} ----------"
           time ./try-runtime ${{ matrix.command_extra_args }} \
               --runtime ./target/release/wbuild/${{ matrix.package }}/${{ matrix.wasm }} \
-              on-runtime-upgrade --disable-spec-version-check --checks=all ${{ matrix.subcommand_extra_args }} live --uri ${{ matrix.uri }}
+              on-runtime-upgrade --disable-spec-version-check --checks=all ${{ matrix.subcommand_extra_args }} snap -p snapshot.raw
           sleep 5

.github/workflows/check-semver.yml

@@ -5,43 +5,85 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
     paths:
       - prdoc/*.prdoc
+  workflow_dispatch:
+
+concurrency:
+  group: check-semver-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   TOOLCHAIN: nightly-2024-06-01
 
+
 jobs:
   check-semver:
     runs-on: ubuntu-latest
     container:
       image: docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v20240408
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: extra git setup
+        run: |
+          git config --global --add safe.directory '*'
+          git fetch --no-tags --no-recurse-submodules --depth=1 origin master
+          git branch old origin/master
+
+      - name: Comment If Backport
+        if: ${{ startsWith(github.event.pull_request.base.ref, 'stable') }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR: ${{ github.event.pull_request.number }}
+        run: |
+          echo "This is a backport into stable."
+
+          wget -q https://github.com/cli/cli/releases/download/v2.51.0/gh_2.51.0_linux_amd64.tar.gz -O gh.tar.gz && \
+          tar -xzf gh.tar.gz && mv gh_2.51.0_linux_amd64/bin/gh /usr/local/bin/gh && rm gh.tar.gz
+          chmod +x /usr/local/bin/gh
+
+            cat > msg.txt <<EOF
+            This pull request is amending an existing release. Please proceed with extreme caution,
+            as to not impact downstream teams that rely on the stability of it. Some things to consider:
+            - Backports are only for 'patch' or 'minor' changes. No 'major' or other breaking change.
+            - Should be a legit *fix* for some  bug, not adding tons of new features.
+            - Must either be already audited or trivial (not sure audit).
+
+            <details><summary><i>Emergency Bypass</i></summary>
+            <p>
+
+            If you really need to bypass this check: add <code>validate: false</code> to each crate
+            in the Prdoc where a breaking change is introduced. This will release a new major
+            version of that crate and all its reverse dependencies and basically break the release.
+
+            </p>
+            </details> 
+          EOF
+          gh issue comment $PR --edit-last -F msg.txt || gh issue comment $PR -F msg.txt
+
+          echo "PRDOC_EXTRA_ARGS=--max-bump minor" >> $GITHUB_ENV
 
       - name: Rust Cache
         uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
         with:
           cache-on-failure: true
 
-      - name: install parity-publish
-        run: cargo install parity-publish@0.7.0
-
       - name: Rust compilation prerequisites
         run: |
           rustup default $TOOLCHAIN
-          rustup target add wasm32-unknown-unknown --toolchain $TOOLCHAIN
           rustup component add rust-src --toolchain $TOOLCHAIN
 
-      - name: extra git setup
-        run: |
-          git config --global --add safe.directory '*'
-          git fetch --no-tags --no-recurse-submodules --depth=1 origin master
-          git branch old origin/master
+      - name: install parity-publish
+        # Set the target dir to cache the build.
+        run: CARGO_TARGET_DIR=./target/ cargo install parity-publish -q
 
       - name: check semver
         run: |
           export CARGO_TARGET_DIR=target
           export RUSTFLAGS='-A warnings -A missing_docs'
           export SKIP_WASM_BUILD=1
-          if ! parity-publish --color always prdoc --since old --validate prdoc/pr_$PR.prdoc -v --toolchain $TOOLCHAIN; then
+
+          if ! parity-publish --color always prdoc --since old --validate prdoc/pr_$PR.prdoc $PRDOC_EXTRA_ARGS -v --toolchain $TOOLCHAIN; then
+
             cat <<EOF
           👋 Hello developer! The SemVer information that you declared in the prdoc file did not match what the CI detected.
 
@@ -56,3 +98,4 @@ jobs:
           fi
         env:
           PR: ${{ github.event.pull_request.number }}
+          PRDOC_EXTRA_ARGS: ${{ env.PRDOC_EXTRA_ARGS }}

.github/workflows/checks-quick.yml

@@ -35,14 +35,14 @@ jobs:
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Cargo fmt
         run: cargo +nightly fmt --all -- --check
   check-dependency-rules:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: check dependency rules
         run: |
           cd substrate/
@@ -54,7 +54,7 @@ jobs:
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: fetch deps
         run: |
           # Pull all dependencies eagerly:
@@ -70,7 +70,7 @@ jobs:
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: run rust features
         run: bash .gitlab/rust-features.sh .
   check-toml-format:
@@ -80,7 +80,7 @@ jobs:
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: check toml format
         run: |
           taplo format --check --config .config/taplo.toml
@@ -89,7 +89,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.0 (22. Sep 2023)
       - name: install python deps
         run: |
           sudo apt-get update && sudo apt-get install -y python3-pip python3
@@ -107,7 +107,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Checkout sources
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Setup Node.js
         uses: actions/setup-node@v4.0.3
         with:
@@ -132,7 +132,7 @@ jobs:
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.0 (22. Sep 2023)
       - name: install python deps
         run: pip3 install "cargo-workspace>=1.2.4" toml
       - name: check umbrella correctness

.github/workflows/misc-sync-templates.yml

@@ -18,11 +18,10 @@ on:
   # A manual dispatch for now - automatic on releases later.
   workflow_dispatch:
     inputs:
-      crate_release_version:
-        description: 'A release version to use, e.g. 1.9.0'
+      stable_release_branch:
+        description: 'Stable release branch, e.g. stable2407'
         required: true
 
-
 jobs:
   sync-templates:
     runs-on: ubuntu-latest
@@ -41,10 +40,10 @@ jobs:
         run: |
           git config --global user.name "Template Bot"
           git config --global user.email "163342540+paritytech-polkadotsdk-templatebot[bot]@users.noreply.github.com"
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           path: polkadot-sdk
-          ref: "release-crates-io-v${{ github.event.inputs.crate_release_version }}"
+          ref: "${{ github.event.inputs.stable_release_branch }}"
       - name: Generate a token for the template repository
         id: app_token
         uses: actions/create-github-app-token@v1.9.3
@@ -53,7 +52,7 @@ jobs:
           repositories: "polkadot-sdk-${{ matrix.template }}-template"
           app-id: ${{ secrets.TEMPLATE_APP_ID }}
           private-key: ${{ secrets.TEMPLATE_APP_KEY }}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: "paritytech/polkadot-sdk-${{ matrix.template }}-template"
           path: "${{ env.template-path }}"
@@ -69,19 +68,19 @@ jobs:
             protobuf-compiler
           rustup target add wasm32-unknown-unknown
           rustup component add rustfmt clippy rust-src
-      
+
       # 2. Yanking the template out of the monorepo workspace.
 
-      - name: Use psvm to replace git references with released creates.
-        run: find . -type f -name 'Cargo.toml' -exec psvm -o -v ${{ github.event.inputs.crate_release_version }} -p {} \;
+      - name: Replace dev-dependencies path references with workspace references
+        run: find . -type f -name 'Cargo.toml' -exec sed -i'' -E "s/path = \"\.\.\/.*\"/workspace = true/g" {} \; 
         working-directory: polkadot-sdk/templates/${{ matrix.template }}/
       - name: Create a new workspace Cargo.toml
         run: |
           cat << EOF > Cargo.toml
           [workspace.package]
           license = "MIT-0"
           authors = ["Parity Technologies <admin@parity.io>"]
-          homepage = "https://substrate.io"
+          homepage = "https://paritytech.github.io/polkadot-sdk/"
 
           [workspace]
           members = [
@@ -123,7 +122,7 @@ jobs:
           cp -r polkadot-sdk/templates/${{ matrix.template }}/* "${{ env.template-path }}/"
 
       - name: Run psvm on monorepo workspace dependencies
-        run: psvm -o -v ${{ github.event.inputs.crate_release_version }} -p ./Cargo.toml
+        run: psvm -o -v ${{ github.event.inputs.stable_release_branch }} -p ./Cargo.toml
         working-directory: polkadot-sdk/
       - name: Copy over required workspace dependencies
         run: |
@@ -164,12 +163,12 @@ jobs:
           token: ${{ steps.app_token.outputs.token }}
           add-paths: |
             ./*
-          title: "[Don't merge] Update the ${{ matrix.template }} template to ${{ github.event.inputs.crate_release_version }}"
+          title: "[Don't merge] Update the ${{ matrix.template }} template to ${{ github.event.inputs.stable_release_branch }}"
           body: "The template has NOT been successfully built and needs to be inspected."
-          branch: "update-template/${{ github.event.inputs.crate_release_version }}"
+          branch: "update-template/${{ github.event.inputs.stable_release_branch }}"
       - name: Push changes
         run: |
           git add -A .
-          git commit --allow-empty -m "Update to ${{ github.event.inputs.crate_release_version }} triggered by ${{ github.event_name }}"
+          git commit --allow-empty -m "Update to ${{ github.event.inputs.stable_release_branch }} triggered by ${{ github.event_name }}"
           git push
         working-directory: "${{ env.template-path }}"