Merge branch 'master' into AndreiEres/pvf-precompilation-kusama

.config/lychee.toml

@@ -18,7 +18,10 @@ accept = [
 	"429",
 ]
 
-exclude_path = ["./target"]
+exclude_path = [
+	"./prdoc",
+	"./target",
+]
 
 exclude = [
 	# Place holders (no need to fix these):

.github/actions/build-push-image/action.yml

@@ -0,0 +1,47 @@
+name: 'build and push image'
+inputs:
+  dockerfile:
+    description: "dockerfile to build"
+    required: true
+  image-name:
+    description: ""
+    required: true
+outputs:
+  branch:
+    description: 'Branch name for the PR'
+    value: ${{ steps.branch.outputs.branch }}
+
+
+runs:
+  using: "composite"
+  steps:
+
+    # gcloud
+    # https://github.com/paritytech/ci_cd/wiki/GitHub:-Push-Docker-image-to-GCP-Registry
+    - name: "Set up Cloud SDK"
+      uses: "google-github-actions/setup-gcloud@v2"
+    - name: "gcloud info"
+      shell: bash
+      run: "gcloud info"
+    - name: "Auth in gcloud registry"
+      shell: bash
+      run: "gcloud auth configure-docker europe-docker.pkg.dev --quiet"  
+
+    - name: build
+      shell: bash
+      env:
+        ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.105"
+      run: |
+        export BRANCH_NAME=${{ github.head_ref || github.ref_name }}
+        export DOCKER_IMAGES_VERSION=${BRANCH_NAME/\//-}
+        if [[ ${{ github.event_name }} == "merge_group" ]]; then export DOCKER_IMAGES_VERSION="${GITHUB_SHA::8}"; fi
+        docker build \
+          --build-arg VCS_REF="${GITHUB_SHA}" \
+          --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
+          --build-arg IMAGE_NAME="${{ inputs.image-name }}" \
+          --build-arg ZOMBIENET_IMAGE="${ZOMBIENET_IMAGE}" \
+          -t "${{ inputs.image-name }}:$DOCKER_IMAGES_VERSION" \
+          -f ${{ inputs.dockerfile }} \
+          .
+        docker push "${{ inputs.image-name }}:$DOCKER_IMAGES_VERSION"
+

.github/codecov.yml

@@ -6,4 +6,10 @@ coverage:
     project:
       default:
         target: 1.0
-        threshold: 2.0
+        threshold: 2.0
+
+comment:
+  behavior: new
+
+fixes:
+  - "/__w/polkadot-sdk/polkadot-sdk/::"

.gitlab/check-each-crate.py → .github/scripts/check-each-crate.py

@@ -9,6 +9,7 @@
 #
 # - `target_group`: Integer starting from 1, the group this script should execute.
 # - `groups_total`: Integer starting from 1, total number of groups.
+# - `disable_forklift`: Boolean, whether to disable forklift or not.
 
 import subprocess, sys
 
@@ -31,6 +32,9 @@
 
 target_group = int(sys.argv[1]) - 1
 groups_total = int(sys.argv[2])
+disable_forklift = bool(sys.argv[3] if len(sys.argv) > 3 else False)
+
+print(f"Target group: {target_group}, Total groups: {groups_total}, Disable forklift: {disable_forklift}", file=sys.stderr)
 
 if len(crates) == 0:
 	print("No crates detected!", file=sys.stderr)
@@ -55,7 +59,11 @@
 
 	print(f"Checking {crates[crate][0]}", file=sys.stderr)
 
-	res = subprocess.run(["forklift", "cargo", "check", "--locked"], cwd = crates[crate][1])
+	cmd = ["cargo", "check", "--locked"]
+
+	cmd.insert(0, 'forklift') if not disable_forklift else None
+
+	res = subprocess.run(cmd, cwd = crates[crate][1])
 
 	if res.returncode != 0:
 		sys.exit(1)

.github/scripts/cmd/cmd.py

@@ -11,6 +11,8 @@
 f = open('.github/workflows/runtimes-matrix.json', 'r')
 runtimesMatrix = json.load(f)
 
+print(f'runtimesMatrix: {runtimesMatrix}\n')
+
 runtimeNames = list(map(lambda x: x['name'], runtimesMatrix))
 
 common_args = {
@@ -67,130 +69,153 @@
 for arg, config in common_args.items():
     parser_ui.add_argument(arg, **config)
 
+def main():
+    global args, unknown, runtimesMatrix
+    args, unknown = parser.parse_known_args()
+
+    print(f'args: {args}')
+
+    if args.command == 'bench':
+        runtime_pallets_map = {}
+        failed_benchmarks = {}
+        successful_benchmarks = {}
+
+        profile = "release"
+
+        print(f'Provided runtimes: {args.runtime}')
+        # convert to mapped dict
+        runtimesMatrix = list(filter(lambda x: x['name'] in args.runtime, runtimesMatrix))
+        runtimesMatrix = {x['name']: x for x in runtimesMatrix}
+        print(f'Filtered out runtimes: {runtimesMatrix}')
+
+        # loop over remaining runtimes to collect available pallets
+        for runtime in runtimesMatrix.values():
+            os.system(f"forklift cargo build -p {runtime['package']} --profile {profile} --features runtime-benchmarks")
+            print(f'-- listing pallets for benchmark for {runtime["name"]}')
+            wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm"
+            output = os.popen(
+                f"frame-omni-bencher v1 benchmark pallet --no-csv-header --no-storage-info --no-min-squares --no-median-slopes --all --list --runtime={wasm_file}").read()
+            raw_pallets = output.strip().split('\n')
+
+            all_pallets = set()
+            for pallet in raw_pallets:
+                if pallet:
+                    all_pallets.add(pallet.split(',')[0].strip())
+
+            pallets = list(all_pallets)
+            print(f'Pallets in {runtime["name"]}: {pallets}')
+            runtime_pallets_map[runtime['name']] = pallets
+
+        print(f'\n')
+
+        # filter out only the specified pallets from collected runtimes/pallets
+        if args.pallet:
+            print(f'Pallets: {args.pallet}')
+            new_pallets_map = {}
+            # keep only specified pallets if they exist in the runtime
+            for runtime in runtime_pallets_map:
+                if set(args.pallet).issubset(set(runtime_pallets_map[runtime])):
+                    new_pallets_map[runtime] = args.pallet
+
+            runtime_pallets_map = new_pallets_map
+
+        print(f'Filtered out runtimes & pallets: {runtime_pallets_map}\n')
+
+        if not runtime_pallets_map:
+            if args.pallet and not args.runtime:
+                print(f"No pallets {args.pallet} found in any runtime")
+            elif args.runtime and not args.pallet:
+                print(f"{args.runtime} runtime does not have any pallets")
+            elif args.runtime and args.pallet:
+                print(f"No pallets {args.pallet} found in {args.runtime}")
+            else:
+                print('No runtimes found')
+            sys.exit(1)
 
-args, unknown = parser.parse_known_args()
-
-print(f'args: {args}')
-
-if args.command == 'bench':
-    runtime_pallets_map = {}
-    failed_benchmarks = {}
-    successful_benchmarks = {}
-
-    profile = "release"
-
-    print(f'Provided runtimes: {args.runtime}')
-    # convert to mapped dict
-    runtimesMatrix = list(filter(lambda x: x['name'] in args.runtime, runtimesMatrix))
-    runtimesMatrix = {x['name']: x for x in runtimesMatrix}
-    print(f'Filtered out runtimes: {runtimesMatrix}')
-
-    # loop over remaining runtimes to collect available pallets
-    for runtime in runtimesMatrix.values():
-        os.system(f"forklift cargo build -p {runtime['package']} --profile {profile} --features runtime-benchmarks")
-        print(f'-- listing pallets for benchmark for {runtime["name"]}')
-        wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm"
-        output = os.popen(
-            f"frame-omni-bencher v1 benchmark pallet --no-csv-header --no-storage-info --no-min-squares --no-median-slopes --all --list --runtime={wasm_file}").read()
-        raw_pallets = output.strip().split('\n')
-
-        all_pallets = set()
-        for pallet in raw_pallets:
-            if pallet:
-                all_pallets.add(pallet.split(',')[0].strip())
-
-        pallets = list(all_pallets)
-        print(f'Pallets in {runtime}: {pallets}')
-        runtime_pallets_map[runtime['name']] = pallets
-
-    # filter out only the specified pallets from collected runtimes/pallets
-    if args.pallet:
-        print(f'Pallet: {args.pallet}')
-        new_pallets_map = {}
-        # keep only specified pallets if they exist in the runtime
         for runtime in runtime_pallets_map:
-            if set(args.pallet).issubset(set(runtime_pallets_map[runtime])):
-                new_pallets_map[runtime] = args.pallet
-
-        runtime_pallets_map = new_pallets_map
-
-    print(f'Filtered out runtimes & pallets: {runtime_pallets_map}')
-
-    if not runtime_pallets_map:
-        if args.pallet and not args.runtime:
-            print(f"No pallets {args.pallet} found in any runtime")
-        elif args.runtime and not args.pallet:
-            print(f"{args.runtime} runtime does not have any pallets")
-        elif args.runtime and args.pallet:
-            print(f"No pallets {args.pallet} found in {args.runtime}")
-        else:
-            print('No runtimes found')
-        sys.exit(1)
-
-    header_path = os.path.abspath('./substrate/HEADER-APACHE2')
-
-    for runtime in runtime_pallets_map:
-        for pallet in runtime_pallets_map[runtime]:
-            config = runtimesMatrix[runtime]
-            print(f'-- config: {config}')
-            if runtime == 'dev':
-                # to support sub-modules (https://github.com/paritytech/command-bot/issues/275)
-                search_manifest_path = f"cargo metadata --locked --format-version 1 --no-deps | jq -r '.packages[] | select(.name == \"{pallet.replace('_', '-')}\") | .manifest_path'"
-                print(f'-- running: {search_manifest_path}')
-                manifest_path = os.popen(search_manifest_path).read()
-                if not manifest_path:
-                    print(f'-- pallet {pallet} not found in dev runtime')
-                    exit(1)
-                package_dir = os.path.dirname(manifest_path)
-                print(f'-- package_dir: {package_dir}')
-                print(f'-- manifest_path: {manifest_path}')
-                output_path = os.path.join(package_dir, "src", "weights.rs")
-            else:
-                default_path = f"./{config['path']}/src/weights"
-                xcm_path = f"./{config['path']}/src/weights/xcm"
-                output_path = default_path if not pallet.startswith("pallet_xcm_benchmarks") else xcm_path
-            print(f'-- benchmarking {pallet} in {runtime} into {output_path}')
-            cmd = f"frame-omni-bencher v1 benchmark pallet --extrinsic=* --runtime=target/{profile}/wbuild/{config['package']}/{config['package'].replace('-', '_')}.wasm --pallet={pallet} --header={header_path} --output={output_path} --wasm-execution=compiled --steps=50 --repeat=20 --heap-pages=4096 --no-storage-info --no-min-squares --no-median-slopes"
-            print(f'-- Running: {cmd}')
-            status = os.system(cmd)
-            if status != 0 and not args.continue_on_fail:
-                print(f'Failed to benchmark {pallet} in {runtime}')
-                sys.exit(1)
-
-            # Otherwise collect failed benchmarks and print them at the end
-            # push failed pallets to failed_benchmarks
-            if status != 0:
-                failed_benchmarks[f'{runtime}'] = failed_benchmarks.get(f'{runtime}', []) + [pallet]
-            else:
-                successful_benchmarks[f'{runtime}'] = successful_benchmarks.get(f'{runtime}', []) + [pallet]
-
-    if failed_benchmarks:
-        print('❌ Failed benchmarks of runtimes/pallets:')
-        for runtime, pallets in failed_benchmarks.items():
-            print(f'-- {runtime}: {pallets}')
-
-    if successful_benchmarks:
-        print('✅ Successful benchmarks of runtimes/pallets:')
-        for runtime, pallets in successful_benchmarks.items():
-            print(f'-- {runtime}: {pallets}')
-
-elif args.command == 'fmt':
-    command = f"cargo +nightly fmt"
-    print(f'Formatting with `{command}`')
-    nightly_status = os.system(f'{command}')
-    taplo_status = os.system('taplo format --config .config/taplo.toml')
-
-    if (nightly_status != 0 or taplo_status != 0) and not args.continue_on_fail:
-        print('❌ Failed to format code')
-        sys.exit(1)
-
-elif args.command == 'update-ui':
-    command = 'sh ./scripts/update-ui-tests.sh'
-    print(f'Updating ui with `{command}`')
-    status = os.system(f'{command}')
-
-    if status != 0 and not args.continue_on_fail:
-        print('❌ Failed to format code')
-        sys.exit(1)
-
-print('🚀 Done')
+            for pallet in runtime_pallets_map[runtime]:
+                config = runtimesMatrix[runtime]
+                header_path = os.path.abspath(config['header'])
+                template = None
+
+                print(f'-- config: {config}')
+                if runtime == 'dev':
+                    # to support sub-modules (https://github.com/paritytech/command-bot/issues/275)
+                    search_manifest_path = f"cargo metadata --locked --format-version 1 --no-deps | jq -r '.packages[] | select(.name == \"{pallet.replace('_', '-')}\") | .manifest_path'"
+                    print(f'-- running: {search_manifest_path}')
+                    manifest_path = os.popen(search_manifest_path).read()
+                    if not manifest_path:
+                        print(f'-- pallet {pallet} not found in dev runtime')
+                        exit(1)
+                    package_dir = os.path.dirname(manifest_path)
+                    print(f'-- package_dir: {package_dir}')
+                    print(f'-- manifest_path: {manifest_path}')
+                    output_path = os.path.join(package_dir, "src", "weights.rs")
+                    template = config['template']
+                else:
+                    default_path = f"./{config['path']}/src/weights"
+                    xcm_path = f"./{config['path']}/src/weights/xcm"
+                    output_path = default_path
+                    if pallet.startswith("pallet_xcm_benchmarks"):
+                        template = config['template']
+                        output_path = xcm_path
+
+                print(f'-- benchmarking {pallet} in {runtime} into {output_path}')
+                cmd = f"frame-omni-bencher v1 benchmark pallet " \
+                    f"--extrinsic=* " \
+                    f"--runtime=target/{profile}/wbuild/{config['package']}/{config['package'].replace('-', '_')}.wasm " \
+                    f"--pallet={pallet} " \
+                    f"--header={header_path} " \
+                    f"--output={output_path} " \
+                    f"--wasm-execution=compiled " \
+                    f"--steps=50 " \
+                    f"--repeat=20 " \
+                    f"--heap-pages=4096 " \
+                    f"{f'--template={template} ' if template else ''}" \
+                    f"--no-storage-info --no-min-squares --no-median-slopes"
+                print(f'-- Running: {cmd} \n')
+                status = os.system(cmd)
+                if status != 0 and not args.continue_on_fail:
+                    print(f'Failed to benchmark {pallet} in {runtime}')
+                    sys.exit(1)
+
+                # Otherwise collect failed benchmarks and print them at the end
+                # push failed pallets to failed_benchmarks
+                if status != 0:
+                    failed_benchmarks[f'{runtime}'] = failed_benchmarks.get(f'{runtime}', []) + [pallet]
+                else:
+                    successful_benchmarks[f'{runtime}'] = successful_benchmarks.get(f'{runtime}', []) + [pallet]
+
+        if failed_benchmarks:
+            print('❌ Failed benchmarks of runtimes/pallets:')
+            for runtime, pallets in failed_benchmarks.items():
+                print(f'-- {runtime}: {pallets}')
+
+        if successful_benchmarks:
+            print('✅ Successful benchmarks of runtimes/pallets:')
+            for runtime, pallets in successful_benchmarks.items():
+                print(f'-- {runtime}: {pallets}')
+
+    elif args.command == 'fmt':
+        command = f"cargo +nightly fmt"
+        print(f'Formatting with `{command}`')
+        nightly_status = os.system(f'{command}')
+        taplo_status = os.system('taplo format --config .config/taplo.toml')
+
+        if (nightly_status != 0 or taplo_status != 0) and not args.continue_on_fail:
+            print('❌ Failed to format code')
+            sys.exit(1)
+
+    elif args.command == 'update-ui':
+        command = 'sh ./scripts/update-ui-tests.sh'
+        print(f'Updating ui with `{command}`')
+        status = os.system(f'{command}')
+
+        if status != 0 and not args.continue_on_fail:
+            print('❌ Failed to format code')
+            sys.exit(1)
+
+    print('🚀 Done')
+
+if __name__ == '__main__':
+    main()