Merge branch 'master' into ao-fix-parainclusion-weight-overestimation

* master: (35 commits)
  Fix OurViewChange small race (#5356)
  Make ticket non-optional and add ensure_successful method to Consideration trait (#5359)
  [tests] dedup test code, add more tests, improve naming and docs (#5338)
  Stop running the wishlist workflow on forks (#5297)
  Migrate foreign assets v3::Location to v4::Location (#4129)
  Minor clean up (#5284)
  [Pools] Ensure members can always exit the pool gracefully (#4998)
  StorageWeightReclaim: set to node pov size if higher (#5281)
  [Bot] Add prdoc generation (#5331)
  Small nits found accidentally along the way (#5341)
  Create subsystem-benchmarks.yml (#5325)
  Bump libp2p-identity from 0.2.8 to 0.2.9 (#5232)
  Bump authoring duration for async backing to 2s. (#5195)
  Fix spelling issues (#5206)
  Bump the known_good_semver group across 1 directory with 3 updates (#5315)
  `polkadot-node-core-pvf-common`: Fix test compilation error (#5310)
  ci: Paused `cmd-action` commenter (#5287)
  Remove unnecessary mut (#5318)
  chain-spec: minor clarification on the genesis config patch (#5324)
  fix av-distribution Jaeger spans mem leak (#5321)
  ...

.github/commands-readme.md

@@ -10,6 +10,7 @@ The current available command actions are:
 
 - [Command FMT](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-fmt.yml)
 - [Command Update UI](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-update-ui.yml)
+- [Command Prdoc](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-prdoc.yml)
 - [Command Sync](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-sync.yml)
 - [Command Bench](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-bench.yml)
 - [Command Bench All](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-bench-all.yml)
@@ -235,6 +236,16 @@ You can use the following [`gh cli`](https://cli.github.com/) inside the repo:
 gh workflow run command-bench-overheard.yml -f pr=1000 -f benchmark=substrate -f runtime=rococo -f target_dir=substrate
 ```
 
+### PrDoc
+
+Generate a PrDoc with the crates populated by all modified crates.
+
+Options:
+- `pr`: The PR number to generate the PrDoc for.
+- `audience`: The audience of whom the changes may concern.
+- `bump`: A default bump level for all crates. The PrDoc will likely need to be edited to reflect the actual changes after generation.
+- `overwrite`: Whether to overwrite any existing PrDoc.
+
 ### Sync
 
 Run sync and commit back results to PR.

.github/scripts/common/lib.sh

@@ -315,6 +315,7 @@ function import_gpg_keys() {
     ) &
   done
   wait
+  gpg -k $SEC
 }
 
 # Check the GPG signature for a given binary
@@ -457,3 +458,15 @@ function get_polkadot_node_version_from_code() {
   # Remove the semicolon
   sed 's/;//g'
 }
+
+validate_stable_tag() {
+    tag="$1"
+    pattern='^stable[0-9]+(-[0-9]+)?$'
+
+    if [[ $tag =~ $pattern ]]; then
+        echo $tag
+    else
+        echo "The input '$tag' does not match the pattern."
+        exit 1
+    fi
+}

.github/scripts/generate-prdoc.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+"""
+Generate the PrDoc for a Pull Request with a specific number, audience and bump level.
+
+It downloads and parses the patch from the GitHub API to opulate the prdoc with all modified crates.
+This will delete any prdoc that already exists for the PR if `--force` is passed.
+
+Usage:
+	python generate-prdoc.py --pr 1234 --audience "TODO" --bump "TODO"
+"""
+
+import argparse
+import os
+import re
+import sys
+import subprocess
+import toml
+import yaml
+import requests
+
+from github import Github
+import whatthepatch
+from cargo_workspace import Workspace
+
+# Download the patch and pass the info into `create_prdoc`.
+def from_pr_number(n, audience, bump, force):
+	print(f"Fetching PR '{n}' from GitHub")
+	g = Github()
+
+	repo = g.get_repo("paritytech/polkadot-sdk")
+	pr = repo.get_pull(n)
+
+	patch_url = pr.patch_url
+	patch = requests.get(patch_url).text
+
+	create_prdoc(n, audience, pr.title, pr.body, patch, bump, force)
+
+def create_prdoc(pr, audience, title, description, patch, bump, force):
+	path = f"prdoc/pr_{pr}.prdoc"
+
+	if os.path.exists(path):
+		if force == True:
+			print(f"Overwriting existing PrDoc for PR {pr}")
+		else:
+			print(f"PrDoc already exists for PR {pr}. Use --force to overwrite.")
+			sys.exit(1)
+	else:
+		print(f"No preexisting PrDoc for PR {pr}")
+
+	prdoc = { "doc": [{}], "crates": [] }
+
+	prdoc["title"] = title
+	prdoc["doc"][0]["audience"] = audience
+	prdoc["doc"][0]["description"] = description
+
+	workspace = Workspace.from_path(".")
+
+	modified_paths = []
+	for diff in whatthepatch.parse_patch(patch):
+		modified_paths.append(diff.header.new_path)
+
+	modified_crates = {}
+	for p in modified_paths:
+		# Go up until we find a Cargo.toml
+		p = os.path.join(workspace.path, p)
+		while not os.path.exists(os.path.join(p, "Cargo.toml")):
+			p = os.path.dirname(p)
+
+		with open(os.path.join(p, "Cargo.toml")) as f:
+			manifest = toml.load(f)
+
+		if not "package" in manifest:
+			print(f"File was not in any crate: {p}")
+			continue
+
+		crate_name = manifest["package"]["name"]
+		if workspace.crate_by_name(crate_name).publish:
+			modified_crates[crate_name] = True
+		else:
+			print(f"Skipping unpublished crate: {crate_name}")
+
+	print(f"Modified crates: {modified_crates.keys()}")
+
+	for crate_name in modified_crates.keys():
+		entry = { "name": crate_name }
+
+		if bump == 'silent' or bump == 'ignore' or bump == 'no change':
+			entry["validate"] = False
+		else:
+			entry["bump"] = bump
+
+		print(f"Adding crate {entry}")
+		prdoc["crates"].append(entry)
+
+	# write the parsed PR documentation back to the file
+	with open(path, "w") as f:
+		yaml.dump(prdoc, f)
+
+def parse_args():
+	parser = argparse.ArgumentParser()
+	parser.add_argument("--pr", type=int, required=True)
+	parser.add_argument("--audience", type=str, default="TODO")
+	parser.add_argument("--bump", type=str, default="TODO")
+	parser.add_argument("--force", type=str)
+	return parser.parse_args()
+
+if __name__ == "__main__":
+	args = parse_args()
+	force = True if args.force.lower() == "true" else False
+	print(f"Args: {args}, force: {force}")
+	from_pr_number(args.pr, args.audience, args.bump, force)

.github/workflows/check-frame-omni-bencher.yml

@@ -0,0 +1,85 @@
+name: Short benchmarks (frame-omni-bencher)
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [ opened, synchronize, reopened, ready_for_review, labeled ]
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  ARTIFACTS_NAME: frame-omni-bencher-artifacts
+
+jobs:
+  changes:
+    # TODO: remove once migration is complete or this workflow is fully stable
+    if: contains(github.event.label.name, 'GHA-migration')
+    permissions:
+      pull-requests: read
+    uses: ./.github/workflows/reusable-check-changed-files.yml
+
+  set-image:
+    # GitHub Actions allows using 'env' in a container context.
+    # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+    # This workaround sets the container image for each job using 'set-image' job output.
+    runs-on: ubuntu-latest
+    needs: changes
+    if: ${{ needs.changes.outputs.rust }}
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+
+  run-frame-omni-bencher:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    needs: [ set-image, changes ] # , build-frame-omni-bencher ]
+    if: ${{ needs.changes.outputs.rust }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false # keep running other workflows even if one fails, to see the logs of all possible failures
+      matrix:
+        runtime:
+          [
+            westend-runtime,
+            rococo-runtime,
+            asset-hub-rococo-runtime,
+            asset-hub-westend-runtime,
+            bridge-hub-rococo-runtime,
+            bridge-hub-westend-runtime,
+            collectives-westend-runtime,
+            coretime-rococo-runtime,
+            coretime-westend-runtime,
+            people-rococo-runtime,
+            people-westend-runtime,
+            glutton-westend-runtime,
+          ]
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    env:
+      PACKAGE_NAME: ${{ matrix.runtime }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: script
+        run: |
+          RUNTIME_BLOB_NAME=$(echo $PACKAGE_NAME | sed 's/-/_/g').compact.compressed.wasm
+          RUNTIME_BLOB_PATH=./target/release/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME
+          forklift cargo build --release --locked -p $PACKAGE_NAME -p frame-omni-bencher --features runtime-benchmarks
+          echo "Running short benchmarking for PACKAGE_NAME=$PACKAGE_NAME and RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH"
+          ls -lrt $RUNTIME_BLOB_PATH
+          ./target/release/frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1
+  confirm-frame-omni-benchers-passed:
+    runs-on: ubuntu-latest
+    name: All benchmarks passed
+    needs: run-frame-omni-bencher
+    steps:
+      - run: echo '### Good job! All the benchmarks passed 🚀' >> $GITHUB_STEP_SUMMARY

.github/workflows/checks.yml

@@ -0,0 +1,90 @@
+name: checks
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review, labeled]
+  merge_group:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  changes:
+    # TODO: remove once migration is complete or this workflow is fully stable
+    if: contains(github.event.label.name, 'GHA-migration')
+    permissions:
+      pull-requests: read
+    uses: ./.github/workflows/reusable-check-changed-files.yml
+  set-image:
+    # GitHub Actions allows using 'env' in a container context.
+    # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+    # This workaround sets the container image for each job using 'set-image' job output.
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+  cargo-clippy:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    needs: [set-image, changes] # , build-frame-omni-bencher ]
+    if: ${{ needs.changes.outputs.rust }}
+    timeout-minutes: 40
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    env:
+      RUSTFLAGS: "-D warnings"
+      SKIP_WASM_BUILD: 1
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: script
+        run: |
+          forklift cargo clippy --all-targets --locked --workspace
+          forklift cargo clippy --all-targets --all-features --locked --workspace
+  check-try-runtime:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    needs: [set-image, changes] # , build-frame-omni-bencher ]
+    if: ${{ needs.changes.outputs.rust }}
+    timeout-minutes: 40
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: script
+        run: |
+          forklift cargo check --locked --all --features try-runtime
+          # this is taken from cumulus
+          # Check that parachain-template will compile with `try-runtime` feature flag.
+          forklift cargo check --locked -p parachain-template-node --features try-runtime
+          # add after https://github.com/paritytech/substrate/pull/14502 is merged
+          # experimental code may rely on try-runtime and vice-versa
+          forklift cargo check --locked --all --features try-runtime,experimental
+  # check-core-crypto-features works fast without forklift
+  check-core-crypto-features:
+    runs-on: arc-runners-polkadot-sdk-beefy
+    needs: [set-image, changes] # , build-frame-omni-bencher ]
+    if: ${{ needs.changes.outputs.rust }}
+    timeout-minutes: 30
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: script
+        run: |
+          cd substrate/primitives/core
+          ./check-features-variants.sh
+          cd -
+          cd substrate/primitives/application-crypto
+          ./check-features-variants.sh
+          cd -
+          cd substrate/primitives/keyring
+          ./check-features-variants.sh
+          cd -