diff --git a/.circleci/config.yml b/.circleci/config.yml index ac54d75ec06..c77f64bc8a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -407,29 +407,29 @@ jobs: command: create_ecr_manifest yarn-project x86_64,arm64 aztec_manifest_key: yarn-project - prover-client-test: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + aztec-package: + machine: + image: default + resource_class: large steps: - *checkout - *setup_env - run: name: "Build and test" - command: cond_spot_run_build prover-client-test 128 - aztec_manifest_key: prover-client-test + command: build aztec + aztec_manifest_key: aztec - aztec-package: + aztec-builder: machine: image: default - resource_class: large + resource_class: small steps: - *checkout - *setup_env - run: - name: "Build and test" - command: build aztec - aztec_manifest_key: aztec + name: "Build image" + command: build aztec-builder + aztec_manifest_key: aztec-builder boxes: docker: @@ -482,55 +482,6 @@ jobs: # For old e2e tests see yarn-project/end-to-end/Earthfile # Semantics are similar to Dockerfile - # NOTE: Unlike other e2e, these will be re-enabled here as currently the logs functionality is not in the new earthfile setup - bench-publish-rollup: - steps: - - *checkout - - *setup_env - - run: - name: "Benchmark" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees - aztec_manifest_key: end-to-end - <<: *defaults_e2e_test - - bench-process-history: - steps: - - *checkout - - *setup_env - - run: - name: "Benchmark" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees - aztec_manifest_key: end-to-end - <<: *defaults_e2e_test - - bench-tx-size: - steps: - - *checkout - - *setup_env - - run: - name: "Benchmark" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_tx_size_fees.test.ts ENABLE_GAS=1 DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees - aztec_manifest_key: end-to-end - <<: *defaults_e2e_test - - build-docs: - machine: - image: default - resource_class: large - steps: - - *checkout - - *setup_env - - run: - name: "Build docs" - command: | - echo "Building docs" - build docs - - run: - name: "Deploy docs" - command: | - echo "Deploying docs" - docs/deploy_netlify.sh $BRANCH $PULL_REQUEST - e2e-join: docker: - image: cimg/base:2023.09 @@ -549,16 +500,6 @@ jobs: name: "Noop" command: echo Noop - bench-summary: - machine: - image: default - steps: - - *checkout - - *setup_env - - run: - name: "Assemble benchmark summary from uploaded logs" - command: ./scripts/ci/assemble_e2e_benchmark.sh - # Deploy & release jobs. deploy-and-release: machine: @@ -573,6 +514,7 @@ jobs: should_release || exit 0 deploy_dockerhub noir deploy_dockerhub aztec + deploy_dockerhub aztec-builder - run: name: "Release canary to NPM: bb.js" command: | @@ -741,8 +683,6 @@ workflows: - noir-projects <<: *defaults - end-to-end: *defaults_yarn_project - - build-docs: *defaults_yarn_project_pre_join - - prover-client-test: *defaults_yarn_project - yarn-project-x86_64: *defaults_yarn_project_pre_join - yarn-project-arm64: *defaults_yarn_project_pre_join - yarn-project-ecr-manifest: @@ -753,6 +693,7 @@ workflows: # Artifacts - aztec-package: *defaults_yarn_project + - aztec-builder: *defaults_yarn_project # Boxes. - boxes: @@ -787,23 +728,11 @@ workflows: - barretenberg-acir-tests-bb - barretenberg-acir-tests-bb-sol - barretenberg-docs - - build-docs - boxes-vanilla - boxes-react - noir-packages-tests - - prover-client-test - e2e-join - <<: *defaults - - # Benchmark jobs. - - bench-publish-rollup: *e2e_test - - bench-process-history: *e2e_test - - bench-tx-size: *e2e_test - - bench-summary: - requires: - - bench-publish-rollup - - bench-process-history - - bench-tx-size + - aztec-builder <<: *defaults # Production releases. diff --git a/.devcontainer/dev/devcontainer.json b/.devcontainer/dev/devcontainer.json index 50354b2b570..792ffdbc010 100644 --- a/.devcontainer/dev/devcontainer.json +++ b/.devcontainer/dev/devcontainer.json @@ -1,17 +1,22 @@ { - "image": "node:lts-bookworm-slim", - "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": {} + "name": "Development", + "build": { + "dockerfile": "../../build-images/Dockerfile", + "context": "../../build-images", + "target": "devbox" }, - "postCreateCommand": "curl -s install.aztec.network | VERSION=master NON_INTERACTIVE=1 BIN_PATH=/usr/local/bin bash -s", - "customizations": { - "vscode": { - "settings": {}, - "extensions": [ - "noir-lang.vscode-noir" - ] - } - }, - "workspaceMount": "source=${localWorkspaceFolder},target=/root/workspace,type=bind", - "workspaceFolder": "/root/workspace" + "containerUser": "aztec-dev", + // ubuntu:noble is currently not supported. + // Can possibly workaround cherry-picking from here: + // https://github.com/devcontainers/features/blob/main/src/docker-in-docker/install.sh + // + // "image": "aztecprotocol/codespace", + // "features": { + // "docker-in-docker": { + // "version": "latest", + // "moby": true, + // "dockerDashComposeVersion": "v1" + // } + // }, + "mounts": ["source=devbox-home,target=/home/aztec-dev,type=volume"] } diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index 4b1d7da6dbb..f5d8029f656 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -1,14 +1,14 @@ # Reusable setup workflow for CI tasks name: Setup Workflow -description: 'Reusable setup steps' +description: "Reusable setup steps" inputs: dockerhub_password: required: true - description: 'DockerHub Password' + description: "DockerHub Password" concurrency_key: required: false - description: 'Concurrency key for locking jobs' + description: "Concurrency key for locking jobs" runs: # define an action, runs in OS of caller using: composite @@ -43,11 +43,14 @@ runs: with: # permission issue with spot runners, simpler to leave out use-cache: false - version: 'v0.8.5' + version: "v0.8.5" - name: Setup Env shell: bash run: ./scripts/setup_env.sh ${{ inputs.dockerhub_password }} + env: + PULL_REQUEST: "${{ github.event.pull_request.number }}" + BRANCH: "${{ github.ref_name }}" # As detailed in https://github.com/ben-z/gh-action-mutex # things do not become 'pending' in github actions, and instead just cancel one another diff --git a/.github/spot-runner-action/action.yaml b/.github/spot-runner-action/action.yaml index a0290b63b8a..4b33c7d344f 100644 --- a/.github/spot-runner-action/action.yaml +++ b/.github/spot-runner-action/action.yaml @@ -40,10 +40,6 @@ inputs: ec2_ami_id: description: 'Ec2 ami ID' required: true - ec2_instance_iam_role: - description: 'IAM role for to associate with ec2 instance' - required: false - default: '' ec2_instance_tags: description: 'List of extra aws resource tags for ec2 instance' required: false diff --git a/.github/spot-runner-action/dist/index.js b/.github/spot-runner-action/dist/index.js index 54dc696e1c3..11ef8b6dde7 100644 --- a/.github/spot-runner-action/dist/index.js +++ b/.github/spot-runner-action/dist/index.js @@ -53,7 +53,6 @@ class ActionConfig { // Ec2 params this.ec2InstanceType = core.getInput("ec2_instance_type").split(" "); this.ec2AmiId = core.getInput("ec2_ami_id"); - this.ec2InstanceIamRole = core.getInput("ec2_instance_iam_role"); this.ec2InstanceTags = core.getInput("ec2_instance_tags"); this.ec2InstanceTtl = core.getInput("ec2_instance_ttl"); this.ec2SubnetId = core.getInput("ec2_subnet_id"); @@ -244,14 +243,14 @@ class Ec2Instance { getLaunchTemplate() { return __awaiter(this, void 0, void 0, function* () { const client = yield this.getEc2Client(); - // NOTE: This should be deterministic or we will create a launch template each time const userData = yield new userdata_1.UserData(this.config).getUserData(); - const ec2InstanceTypeHash = this.getHashOfStringArray(this.config.ec2InstanceType.concat([userData]).concat([JSON.stringify(this.tags)])); + const ec2InstanceTypeHash = this.getHashOfStringArray(this.config.ec2InstanceType.concat([userData, JSON.stringify(this.tags), this.config.ec2KeyName])); const launchTemplateName = "aztec-packages-spot-" + this.config.ec2AmiId + "-" + ec2InstanceTypeHash; const launchTemplateParams = { LaunchTemplateName: launchTemplateName, LaunchTemplateData: { ImageId: this.config.ec2AmiId, + InstanceInitiatedShutdownBehavior: "terminate", InstanceRequirements: { // We do not know what the instance types correspond to // just let the user send a list of allowed instance types @@ -259,6 +258,8 @@ class Ec2Instance { MemoryMiB: { Min: 0 }, AllowedInstanceTypes: this.config.ec2InstanceType, }, + SecurityGroupIds: [this.config.ec2SecurityGroupId], + KeyName: this.config.ec2KeyName, UserData: userData, TagSpecifications: [ { @@ -276,33 +277,20 @@ class Ec2Instance { ], }, }; - let arr = []; - try { - arr = (yield client - .describeLaunchTemplates({ - LaunchTemplateNames: [launchTemplateName], - }) - .promise()).LaunchTemplates || []; - } - catch (err) { - core.info("Launch templates describe error, note this will be likely resolved by creating the template in the next step: " + err); - } - core.info("Launch templates found: " + JSON.stringify(arr, null, 2)); - if (arr.length <= 0) { - core.info("Creating launch template: " + launchTemplateName); - yield client.createLaunchTemplate(launchTemplateParams).promise(); - } + core.info("Creating launch template: " + launchTemplateName); + yield client.createLaunchTemplate(launchTemplateParams).promise(); return launchTemplateName; }); } requestMachine(useOnDemand) { return __awaiter(this, void 0, void 0, function* () { // Note advice re max bid: "If you specify a maximum price, your instances will be interrupted more frequently than if you do not specify this parameter." + const launchTemplateName = yield this.getLaunchTemplate(); const availabilityZone = yield this.getSubnetAz(); const fleetLaunchConfig = { LaunchTemplateSpecification: { Version: "$Latest", - LaunchTemplateName: yield this.getLaunchTemplate(), + LaunchTemplateName: launchTemplateName, }, Overrides: this.config.ec2InstanceType.map((instanceType) => ({ InstanceType: instanceType, @@ -323,6 +311,10 @@ class Ec2Instance { const client = yield this.getEc2Client(); const fleet = yield client.createFleet(createFleetRequest).promise(); const instances = ((fleet === null || fleet === void 0 ? void 0 : fleet.Instances) || [])[0] || {}; + // cleanup + yield client.deleteLaunchTemplate({ + LaunchTemplateName: launchTemplateName, + }); return (instances.InstanceIds || [])[0]; }); } @@ -871,23 +863,14 @@ class UserData { `mkdir -p shutdown-refcount`, // Shutdown rules: // - github actions job starts and ends always bump +ec2InstanceTtl minutes - // - when the amount of started jobs (start_run_* files) equal the amount of finished jobs (end_run_* files), we shutdown in 5 minutes - `echo "${bumpShutdown}; touch /run/shutdown-refcount/start_run_\\$(date +%s)_\\$RANDOM" > /run/delay_shutdown.sh`, - // `echo "[ \\$(find /run/shutdown-refcount/ -name 'start_run_*' | wc -l) -eq \\$(find /run/shutdown-refcount/ -name 'end_run_*' | wc -l) ] && shutdown -P 5 ; true" > /run/if_refcount0_shutdown.sh`, - `echo "echo refcounting disabled for now" > /run/if_refcount0_shutdown.sh`, - `echo "${bumpShutdown}; touch /run/shutdown-refcount/end_run_\\$(date +%s)_\\$RANDOM ; /run/if_refcount0_shutdown.sh " > /run/refcount_and_delay_shutdown.sh`, - `echo "flock /run/refcount-lock /run/delay_shutdown.sh" > /run/safe_delay_shutdown.sh`, - `echo "flock /run/refcount-lock /run/refcount_and_delay_shutdown.sh" > /run/safe_refcount_and_delay_shutdown.sh`, + // - when the amount of started jobs (start_run_* files) equal the amount of finished jobs (end_run_* files), we shutdown in 5 minutes (with a reaper script installed later) + `echo "${bumpShutdown}" > /run/delay_shutdown.sh`, "chmod +x /run/delay_shutdown.sh", - "chmod +x /run/refcount_and_delay_shutdown.sh", - "chmod +x /run/if_refcount0_shutdown.sh", - "chmod +x /run/safe_refcount_and_delay_shutdown.sh", - "chmod +x /run/safe_if_refcount0_shutdown.sh", - "export ACTIONS_RUNNER_HOOK_JOB_STARTED=/run/safe_delay_shutdown.sh", - "export ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/run/safe_refcount_and_delay_shutdown.sh", + "export ACTIONS_RUNNER_HOOK_JOB_STARTED=/run/delay_shutdown.sh", + "export ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/run/delay_shutdown.sh", "mkdir -p actions-runner && cd actions-runner", - 'echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/run/safe_delay_shutdown.sh" > .env', - 'echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/run/safe_refcount_and_delay_shutdown.sh" > .env', + 'echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/run/delay_shutdown.sh" > .env', + 'echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/run/delay_shutdown.sh" > .env', `GH_RUNNER_VERSION=${githubActionRunnerVersion}`, 'case $(uname -m) in aarch64) ARCH="arm64" ;; amd64|x86_64) ARCH="x64" ;; esac && export RUNNER_ARCH=${ARCH}', "curl -O -L https://github.com/actions/runner/releases/download/v${GH_RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${GH_RUNNER_VERSION}.tar.gz", diff --git a/.github/spot-runner-action/src/config.ts b/.github/spot-runner-action/src/config.ts index da7ec1c9d8b..1c15bd00e65 100644 --- a/.github/spot-runner-action/src/config.ts +++ b/.github/spot-runner-action/src/config.ts @@ -19,7 +19,6 @@ export interface ConfigInterface { ec2InstanceType: string[]; ec2AmiId: string; - ec2InstanceIamRole: string; ec2InstanceTags: string; ec2InstanceTtl: string; ec2SecurityGroupId: string; @@ -46,7 +45,6 @@ export class ActionConfig implements ConfigInterface { ec2InstanceType: string[]; ec2AmiId: string; - ec2InstanceIamRole: string; ec2InstanceTags: string; ec2InstanceTtl: string; ec2SecurityGroupId: string; @@ -77,7 +75,6 @@ export class ActionConfig implements ConfigInterface { // Ec2 params this.ec2InstanceType = core.getInput("ec2_instance_type").split(" "); this.ec2AmiId = core.getInput("ec2_ami_id"); - this.ec2InstanceIamRole = core.getInput("ec2_instance_iam_role"); this.ec2InstanceTags = core.getInput("ec2_instance_tags"); this.ec2InstanceTtl = core.getInput("ec2_instance_ttl"); this.ec2SubnetId = core.getInput("ec2_subnet_id"); diff --git a/.github/spot-runner-action/src/ec2.ts b/.github/spot-runner-action/src/ec2.ts index c9a736cd1e3..a2719d4f50f 100644 --- a/.github/spot-runner-action/src/ec2.ts +++ b/.github/spot-runner-action/src/ec2.ts @@ -4,9 +4,7 @@ import { CreateFleetInstance, CreateFleetRequest, CreateLaunchTemplateRequest, - FleetLaunchTemplateConfig, FleetLaunchTemplateConfigRequest, - RunInstancesRequest, } from "aws-sdk/clients/ec2"; import * as crypto from "crypto"; import * as core from "@actions/core"; @@ -160,10 +158,9 @@ export class Ec2Instance { async getLaunchTemplate(): Promise { const client = await this.getEc2Client(); - // NOTE: This should be deterministic or we will create a launch template each time const userData = await new UserData(this.config).getUserData(); const ec2InstanceTypeHash = this.getHashOfStringArray( - this.config.ec2InstanceType.concat([userData]).concat([JSON.stringify(this.tags)]) + this.config.ec2InstanceType.concat([userData, JSON.stringify(this.tags), this.config.ec2KeyName]) ); const launchTemplateName = "aztec-packages-spot-" + this.config.ec2AmiId + "-" + ec2InstanceTypeHash; @@ -172,6 +169,7 @@ export class Ec2Instance { LaunchTemplateName: launchTemplateName, LaunchTemplateData: { ImageId: this.config.ec2AmiId, + InstanceInitiatedShutdownBehavior: "terminate", InstanceRequirements: { // We do not know what the instance types correspond to // just let the user send a list of allowed instance types @@ -179,6 +177,8 @@ export class Ec2Instance { MemoryMiB: { Min: 0 }, AllowedInstanceTypes: this.config.ec2InstanceType, }, + SecurityGroupIds: [this.config.ec2SecurityGroupId], + KeyName: this.config.ec2KeyName, UserData: userData, TagSpecifications: [ { @@ -196,34 +196,19 @@ export class Ec2Instance { ], }, }; - let arr: any[] = []; - - try { - arr = ( - await client - .describeLaunchTemplates({ - LaunchTemplateNames: [launchTemplateName], - }) - .promise() - ).LaunchTemplates || []; - } catch (err) { - core.info("Launch templates describe error, note this will be likely resolved by creating the template in the next step: " + err); - } - core.info("Launch templates found: " + JSON.stringify(arr, null, 2)); - if (arr.length <= 0) { - core.info("Creating launch template: " + launchTemplateName); - await client.createLaunchTemplate(launchTemplateParams).promise(); - } + core.info("Creating launch template: " + launchTemplateName); + await client.createLaunchTemplate(launchTemplateParams).promise(); return launchTemplateName; } async requestMachine(useOnDemand: boolean): Promise { // Note advice re max bid: "If you specify a maximum price, your instances will be interrupted more frequently than if you do not specify this parameter." + const launchTemplateName = await this.getLaunchTemplate(); const availabilityZone = await this.getSubnetAz(); const fleetLaunchConfig: FleetLaunchTemplateConfigRequest = { LaunchTemplateSpecification: { Version: "$Latest", - LaunchTemplateName: await this.getLaunchTemplate(), + LaunchTemplateName: launchTemplateName, }, Overrides: this.config.ec2InstanceType.map((instanceType) => ({ InstanceType: instanceType, @@ -244,6 +229,10 @@ export class Ec2Instance { const client = await this.getEc2Client(); const fleet = await client.createFleet(createFleetRequest).promise(); const instances: CreateFleetInstance = (fleet?.Instances || [])[0] || {}; + // cleanup + await client.deleteLaunchTemplate({ + LaunchTemplateName: launchTemplateName, + }); return (instances.InstanceIds || [])[0]; } diff --git a/.github/spot-runner-action/src/userdata.ts b/.github/spot-runner-action/src/userdata.ts index 9e996393273..4455735b8d2 100644 --- a/.github/spot-runner-action/src/userdata.ts +++ b/.github/spot-runner-action/src/userdata.ts @@ -35,23 +35,14 @@ export class UserData { `mkdir -p shutdown-refcount`, // Shutdown rules: // - github actions job starts and ends always bump +ec2InstanceTtl minutes - // - when the amount of started jobs (start_run_* files) equal the amount of finished jobs (end_run_* files), we shutdown in 5 minutes - `echo "${bumpShutdown}; touch /run/shutdown-refcount/start_run_\\$(date +%s)_\\$RANDOM" > /run/delay_shutdown.sh`, - // `echo "[ \\$(find /run/shutdown-refcount/ -name 'start_run_*' | wc -l) -eq \\$(find /run/shutdown-refcount/ -name 'end_run_*' | wc -l) ] && shutdown -P 5 ; true" > /run/if_refcount0_shutdown.sh`, - `echo "echo refcounting disabled for now" > /run/if_refcount0_shutdown.sh`, - `echo "${bumpShutdown}; touch /run/shutdown-refcount/end_run_\\$(date +%s)_\\$RANDOM ; /run/if_refcount0_shutdown.sh " > /run/refcount_and_delay_shutdown.sh`, - `echo "flock /run/refcount-lock /run/delay_shutdown.sh" > /run/safe_delay_shutdown.sh`, - `echo "flock /run/refcount-lock /run/refcount_and_delay_shutdown.sh" > /run/safe_refcount_and_delay_shutdown.sh`, + // - when the amount of started jobs (start_run_* files) equal the amount of finished jobs (end_run_* files), we shutdown in 5 minutes (with a reaper script installed later) + `echo "${bumpShutdown}" > /run/delay_shutdown.sh`, "chmod +x /run/delay_shutdown.sh", - "chmod +x /run/refcount_and_delay_shutdown.sh", - "chmod +x /run/if_refcount0_shutdown.sh", - "chmod +x /run/safe_refcount_and_delay_shutdown.sh", - "chmod +x /run/safe_if_refcount0_shutdown.sh", - "export ACTIONS_RUNNER_HOOK_JOB_STARTED=/run/safe_delay_shutdown.sh", - "export ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/run/safe_refcount_and_delay_shutdown.sh", + "export ACTIONS_RUNNER_HOOK_JOB_STARTED=/run/delay_shutdown.sh", + "export ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/run/delay_shutdown.sh", "mkdir -p actions-runner && cd actions-runner", - 'echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/run/safe_delay_shutdown.sh" > .env', - 'echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/run/safe_refcount_and_delay_shutdown.sh" > .env', + 'echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/run/delay_shutdown.sh" > .env', + 'echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/run/delay_shutdown.sh" > .env', `GH_RUNNER_VERSION=${githubActionRunnerVersion}`, 'case $(uname -m) in aarch64) ARCH="arm64" ;; amd64|x86_64) ARCH="x64" ;; esac && export RUNNER_ARCH=${ARCH}', "curl -O -L https://github.com/actions/runner/releases/download/v${GH_RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${GH_RUNNER_VERSION}.tar.gz", diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index aa1d3f26ea7..7f9f89ca4da 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -1,7 +1,7 @@ name: CI (ARM) on: push: - branches: [master] + branches: [disabled] # [master] workflow_dispatch: inputs: runner_action: @@ -32,7 +32,10 @@ jobs: needs: setup runs-on: master-arm steps: - - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - { + uses: actions/checkout@v4, + with: { ref: "${{ github.event.pull_request.head.sha }}" }, + } - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" @@ -48,7 +51,10 @@ jobs: needs: build runs-on: master-arm steps: - - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - { + uses: actions/checkout@v4, + with: { ref: "${{ github.event.pull_request.head.sha }}" }, + } - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" @@ -56,7 +62,7 @@ jobs: - name: Test working-directory: ./yarn-project/end-to-end/ timeout-minutes: 15 - run: earthly -P --no-output +uniswap-trade-on-l1-from-l2 --e2e_mode=cache + run: earthly -P --no-output +uniswap-trade-on-l1-from-l2 notify: needs: [e2e] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73dc79e18af..ad8870e6ec1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,10 +11,12 @@ on: runner_action: description: "The action to take with the self-hosted runner (start, stop, restart)." required: false + concurrency: # force parallelism in master group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true + jobs: setup: uses: ./.github/workflows/setup-runner.yml @@ -64,10 +66,8 @@ jobs: matrix: test: ${{ fromJson( needs.build.outputs.e2e_list )}} steps: - - { - uses: actions/checkout@v4, - with: { ref: "${{ github.event.pull_request.head.sha }}" }, - } + - uses: actions/checkout@v4 + with: { ref: "${{ github.event.pull_request.head.sha }}" } - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" @@ -76,10 +76,35 @@ jobs: - name: Test working-directory: ./yarn-project/end-to-end/ timeout-minutes: 25 - run: earthly-ci -P --no-output +${{ matrix.test }} --e2e_mode=cache - # TODO - # - name: Upload logs - # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }} + + # bench-summary: + # needs: e2e + # runs-on: ${{ inputs.username || github.actor }}-x86 + # steps: + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 100 # Downloading base benchmark from master requires access to history + # ref: "${{ github.event.pull_request.head.sha }}" + # - uses: ./.github/ci-setup-action + # with: + # dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + # concurrency_key: build-${{ inputs.username || github.actor }}-x86 + # - name: "Build and upload bench aggregate file" + # working-directory: ./yarn-project/scripts + # run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} +bench-aggregate + # - name: "Download base benchmark" + # if: ${{ github.event_name == 'pull_request' }} + # run: scripts/logs/download_base_benchmark_from_s3.sh + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # BENCH_FOLDER: ./yarn-project/scripts/bench + # PULL_REQUEST: "${{ github.event.pull_request.number }}" + # - name: "Generate summary comment if pull request" + # if: ${{ github.event_name == 'pull_request' }} + # working-directory: ./yarn-project/scripts + # run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment # barretenberg (prover) native and AVM (public VM) tests # only ran on x86 for resource reasons (memory intensive) @@ -153,6 +178,39 @@ jobs: timeout-minutes: 25 run: earthly-ci --no-output ./yarn-project/+test + prover-client-test: + needs: noir-projects + runs-on: ${{ github.actor }}-x86 + steps: + - { + uses: actions/checkout@v4, + with: { ref: "${{ github.event.pull_request.head.sha }}" }, + } + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_key: prover-client-test-${{ github.actor }}-x86 + - name: "Prover Client Tests" + timeout-minutes: 25 + run: earthly-ci --no-output ./yarn-project/+prover-client-test + + docs-preview: + needs: setup + runs-on: ${{ inputs.username || github.actor }}-x86 + if: github.event.number + steps: + - { + uses: actions/checkout@v4, + with: { ref: "${{ github.event.pull_request.head.sha }}" }, + } + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_key: docs-preview-${{ inputs.username || github.actor }}-x86 + - name: "Docs Preview" + timeout-minutes: 25 + run: earthly --no-output ./docs/+deploy-preview --PR=${{ github.event.number }} --AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} --NETLIFY_SITE_ID=${{ secrets.NETLIFY_SITE_ID }} + # push benchmarking binaries to dockerhub registry bb-bench-binaries: needs: setup @@ -215,6 +273,7 @@ jobs: - bb-bench - yarn-project-formatting - yarn-project-test + - prover-client-test if: always() steps: - run: | @@ -237,6 +296,7 @@ jobs: bb-bench, yarn-project-formatting, yarn-project-test, + prover-client-test, ] runs-on: ubuntu-latest if: ${{ github.ref == 'refs/heads/master' && failure() }} diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml new file mode 100644 index 00000000000..2ad60d1bf63 --- /dev/null +++ b/.github/workflows/publish-docs.yml @@ -0,0 +1,18 @@ +name: Publish Docs +on: + workflow_dispatch: + inputs: + tag: + description: The tag to build from (leave empty to build a nightly release from master) + required: true + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag || env.GITHUB_REF }} + + - timeout-minutes: 25 + run: earthly --no-output ./docs/+deploy-prod --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} --NETLIFY_SITE_ID=${{ secrets.NETLIFY_SITE_ID }} diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index ce4badd3ed4..a294ba2f653 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -22,7 +22,7 @@ jobs: token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} command: manifest - - name: Dispatch to publish workflow + - name: Dispatch to publish BB workflow uses: benc-uk/workflow-dispatch@v1 if: ${{ steps.release.outputs.tag_name }} with: @@ -30,3 +30,12 @@ jobs: ref: master token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} inputs: '{ "tag": "${{ steps.release.outputs.tag_name }}", "publish": true }' + + - name: Dispatch to publish docs workflow + uses: benc-uk/workflow-dispatch@v1 + if: ${{ steps.release.outputs.tag_name }} + with: + workflow: publish-docs.yml + ref: master + token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + inputs: '{ "tag": "${{ steps.release.outputs.tag_name }}" }' diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml index a29ccc6d5e5..dff8f10cff3 100644 --- a/.github/workflows/setup-runner.yml +++ b/.github/workflows/setup-runner.yml @@ -78,11 +78,11 @@ jobs: run: | # Compare the checked-out CI configuration files with the reference files if ! git diff --no-index .github/workflows/ci.yml merge-commit-pipeline-files/.github/workflows/ci.yml; then - echo "Error: ci.yml changes in master (or PR base). Please merge these changes." + echo "Error: ci.yml changes in master (or PR base). Please merge these changes. This is to prevent surprises from Github Action's merge behavior." exit 1 fi if ! git diff --no-index .github/workflows/setup-runner.yml merge-commit-pipeline-files/.github/workflows/setup-runner.yml; then - echo "Error: setup-runner.yml changes in master (or PR base). Please merge these changes." + echo "Error: setup-runner.yml changes in master (or PR base). Please merge these changes. This is to prevent surprises from Github Action's merge behavior." exit 1 fi diff --git a/.gitignore b/.gitignore index fcee2d1838e..8e28a8bac2f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ cmake-build-debug .DS_Store **/*.dockerignore + +# Earthly +.arg +.secret diff --git a/.noir-sync-commit b/.noir-sync-commit index 78cfb1ccf97..c7d7dc99a11 100644 --- a/.noir-sync-commit +++ b/.noir-sync-commit @@ -1 +1 @@ -1ec9cdc7013e867db3672d27e3a6104e4b7e7eef \ No newline at end of file +d4c68066ab35ce1c52510cf0c038fb627a0677c3 diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 2af8fe1250f..a3061298efc 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,7 +1,7 @@ { - ".": "0.35.1", + ".": "0.36.0", "yarn-project/cli": "0.35.1", - "yarn-project/aztec": "0.35.1", - "barretenberg": "0.35.1", - "barretenberg/ts": "0.35.1" + "yarn-project/aztec": "0.36.0", + "barretenberg": "0.36.0", + "barretenberg/ts": "0.36.0" } diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 988ff79b475..a872182ad77 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -37,7 +37,7 @@ // Displays code coverage report information within vscode "ryanluker.vscode-coverage-gutters", // Spell checking - "streetsidesoftware.code-spell-checker", + "streetsidesoftware.code-spell-checker" // End C++/Circuits extensions /////////////////////////////////////// ], @@ -58,7 +58,7 @@ // Most features are disabled in `settings.json` // which confict with `clangd` // Since we ignore GDB, we no longer need this extension - "ms-vscode.cpptools", + "ms-vscode.cpptools" // End C++/Circuits unwanted extensions /////////////////////////////////////// ] diff --git a/.vscode/settings.json b/.vscode/settings.json index ea41bbe2bc2..48a4cde9de8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -136,7 +136,7 @@ // Clangd. Note that this setting may be overridden by user settings // to the default value "clangd". // - "clangd.path": "clangd-15", + "clangd.path": "clangd-16", // // C/C++ (should be disabled) // @@ -165,6 +165,5 @@ "**/target/**": true, "**/l1-contracts/lib/**": true, "**/barretenberg/cpp/build*/**": true - }, - "cmake.sourceDirectory": "/mnt/user-data/adam/aztec-packages/barretenberg/cpp" + } } diff --git a/CHANGELOG.md b/CHANGELOG.md index 18ca6ac8737..1f4fa357c90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,276 @@ # Changelog +## [0.36.0](https://github.com/AztecProtocol/aztec-packages/compare/aztec-packages-v0.35.1...aztec-packages-v0.36.0) (2024-04-30) + + +### ⚠ BREAKING CHANGES + +* remove `Opcode::Brillig` from ACIR ([#5995](https://github.com/AztecProtocol/aztec-packages/issues/5995)) +* delete field note ([#5959](https://github.com/AztecProtocol/aztec-packages/issues/5959)) +* remove slow updates tree ([#5954](https://github.com/AztecProtocol/aztec-packages/issues/5954)) +* Add `as_array` and remove `_slice` variants of hash functions (https://github.com/noir-lang/noir/pull/4675) +* reserve keyword `super` (https://github.com/noir-lang/noir/pull/4836) +* **aztec-nr:** unencrypted logs go behind context ([#5871](https://github.com/AztecProtocol/aztec-packages/issues/5871)) +* move fixtures to @aztec/circuits.js/testing/fixtures ([#5826](https://github.com/AztecProtocol/aztec-packages/issues/5826)) +* contract interfaces and better function calls ([#5687](https://github.com/AztecProtocol/aztec-packages/issues/5687)) +* change backend width to 4 ([#5374](https://github.com/AztecProtocol/aztec-packages/issues/5374)) + +### Features + +* `variable_base_scalar_mul` blackbox func ([#6039](https://github.com/AztecProtocol/aztec-packages/issues/6039)) ([81142fe](https://github.com/AztecProtocol/aztec-packages/commit/81142fe799338e6ed73b30eeac4468c1345f6fab)) +* **acir_gen:** Brillig stdlib (https://github.com/noir-lang/noir/pull/4848) ([8f73f18](https://github.com/AztecProtocol/aztec-packages/commit/8f73f18f3c07de0fd5e247ade5a48109c37c1bc5)) +* Add `#[inline(tag)]` attribute and codegen (https://github.com/noir-lang/noir/pull/4913) ([e615a83](https://github.com/AztecProtocol/aztec-packages/commit/e615a831a12b78644b798e12395d970bf5601948)) +* Add `min` and `max` functions to the stdlib (https://github.com/noir-lang/noir/pull/4839) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Add `NARGO_FOREIGN_CALL_TIMEOUT` environment variable (https://github.com/noir-lang/noir/pull/4780) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Add comptime Interpreter (https://github.com/noir-lang/noir/pull/4821) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Add key registry to deployment (e2e & sandbox) ([#5875](https://github.com/AztecProtocol/aztec-packages/issues/5875)) ([0881cd3](https://github.com/AztecProtocol/aztec-packages/commit/0881cd3083af70271bceda695d0c8ad21212c172)), closes [#5611](https://github.com/AztecProtocol/aztec-packages/issues/5611) +* Add promiseWithResolvers ([#5808](https://github.com/AztecProtocol/aztec-packages/issues/5808)) ([afeef17](https://github.com/AztecProtocol/aztec-packages/commit/afeef17e14054f8ee95a6244c1b165435fddaa50)) +* Add proving queue ([#5754](https://github.com/AztecProtocol/aztec-packages/issues/5754)) ([a0a9668](https://github.com/AztecProtocol/aztec-packages/commit/a0a9668d933907a89f21077fd700b6d2f44e6c74)) +* Add side effect counter to logs ([#5718](https://github.com/AztecProtocol/aztec-packages/issues/5718)) ([d7486a6](https://github.com/AztecProtocol/aztec-packages/commit/d7486a6b0b26b5264a1b02c1134d82abfb497aa0)) +* Add the storage layout to the contract artifact ([#5952](https://github.com/AztecProtocol/aztec-packages/issues/5952)) ([88ee0af](https://github.com/AztecProtocol/aztec-packages/commit/88ee0af9987063d63afb49c4f61ab5ae5f7c1b73)) +* Add TimeoutError ([#5751](https://github.com/AztecProtocol/aztec-packages/issues/5751)) ([741fdf1](https://github.com/AztecProtocol/aztec-packages/commit/741fdf16e7f0b3f116c724505afa8e604bc51bf1)) +* Add variable size sha256 (https://github.com/noir-lang/noir/pull/4920) ([e615a83](https://github.com/AztecProtocol/aztec-packages/commit/e615a831a12b78644b798e12395d970bf5601948)) +* Add variable size sha256 (https://github.com/noir-lang/noir/pull/4920) ([078aa61](https://github.com/AztecProtocol/aztec-packages/commit/078aa61b06557aba74ac9cce557ee6bd05040feb)) +* AES oracle ([#5996](https://github.com/AztecProtocol/aztec-packages/issues/5996)) ([8e0a563](https://github.com/AztecProtocol/aztec-packages/commit/8e0a56306ba45ea1eaaa25ee47d84b7334e0bbe3)), closes [#5895](https://github.com/AztecProtocol/aztec-packages/issues/5895) +* AES oracle padding ([#6013](https://github.com/AztecProtocol/aztec-packages/issues/6013)) ([4b563cd](https://github.com/AztecProtocol/aztec-packages/commit/4b563cd79f16f513a05c1595a4f2673cdaa7600a)) +* Allow numeric generics to non inlined ACIR functions (https://github.com/noir-lang/noir/pull/4834) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Avm mem trace validation ([#6025](https://github.com/AztecProtocol/aztec-packages/issues/6025)) ([3a3afb5](https://github.com/AztecProtocol/aztec-packages/commit/3a3afb57ab8b6b3f11b7a7799d557436638c8cd3)), closes [#5950](https://github.com/AztecProtocol/aztec-packages/issues/5950) +* **avm:** Avm circuit FDIV opcode ([#5958](https://github.com/AztecProtocol/aztec-packages/issues/5958)) ([fed5b6d](https://github.com/AztecProtocol/aztec-packages/commit/fed5b6dd1ee310fc90404a3e5ec9eb02ad7dbc10)), closes [#5953](https://github.com/AztecProtocol/aztec-packages/issues/5953) +* **avm:** CAST opcode implementation ([#5477](https://github.com/AztecProtocol/aztec-packages/issues/5477)) ([a821bcc](https://github.com/AztecProtocol/aztec-packages/commit/a821bccef7b1894140f0495510d7c6b4eefde821)), closes [#5466](https://github.com/AztecProtocol/aztec-packages/issues/5466) +* **avm:** Negative tests ([#5919](https://github.com/AztecProtocol/aztec-packages/issues/5919)) ([8a5ece7](https://github.com/AztecProtocol/aztec-packages/commit/8a5ece7548a86d099ac6a166f04882624b8d95fd)) +* **avm:** Shift relations ([#5716](https://github.com/AztecProtocol/aztec-packages/issues/5716)) ([a516637](https://github.com/AztecProtocol/aztec-packages/commit/a51663707b96914b0a300440611748ce44fbe933)) +* Avoiding redundant computation in PG ([#5844](https://github.com/AztecProtocol/aztec-packages/issues/5844)) ([9f57733](https://github.com/AztecProtocol/aztec-packages/commit/9f5773353aa0261fa07a81704bcadcee513d42c5)) +* Brillig pointer codegen and execution ([#5737](https://github.com/AztecProtocol/aztec-packages/issues/5737)) ([a7b9d20](https://github.com/AztecProtocol/aztec-packages/commit/a7b9d20a962c33d8585502fd00739138c6d79aca)) +* Bump lmdb ([#5783](https://github.com/AztecProtocol/aztec-packages/issues/5783)) ([f7d5cf2](https://github.com/AztecProtocol/aztec-packages/commit/f7d5cf2c683ee7840885ac176b9e838b4e3ab6e2)) +* Change backend width to 4 ([#5374](https://github.com/AztecProtocol/aztec-packages/issues/5374)) ([3f24fc2](https://github.com/AztecProtocol/aztec-packages/commit/3f24fc2cdb56eff6da6e47062d2a2a3dc0fa4bd2)) +* Circuit simulator for Ultra and GoblinUltra verifiers ([#1195](https://github.com/AztecProtocol/aztec-packages/issues/1195)) ([0032a3a](https://github.com/AztecProtocol/aztec-packages/commit/0032a3a55dea5e4c9051dbc36607288f8ca1be4a)) +* Computing sym key for incoming ciphertext ([#6020](https://github.com/AztecProtocol/aztec-packages/issues/6020)) ([1904fa8](https://github.com/AztecProtocol/aztec-packages/commit/1904fa864ff8c546d4d849436c6ca7a7606fb3d2)) +* Configure prover as separate process ([#5973](https://github.com/AztecProtocol/aztec-packages/issues/5973)) ([c0dd7b2](https://github.com/AztecProtocol/aztec-packages/commit/c0dd7b21779b99f1b9d3ed43623d3de25a332699)) +* Contract interfaces and better function calls ([#5687](https://github.com/AztecProtocol/aztec-packages/issues/5687)) ([274f7d9](https://github.com/AztecProtocol/aztec-packages/commit/274f7d935230ce21d062644f6ec5f7cd0f58ae62)) +* Decoded return values ([#5762](https://github.com/AztecProtocol/aztec-packages/issues/5762)) ([03e693a](https://github.com/AztecProtocol/aztec-packages/commit/03e693a0db52a0c0b02c403f9ded2e28f3c7ced2)) +* Delete field note ([#5959](https://github.com/AztecProtocol/aztec-packages/issues/5959)) ([ae18396](https://github.com/AztecProtocol/aztec-packages/commit/ae183960a96d14d1eac2876bc070ed09f75b8f25)) +* **docs:** Nuke CLI from docs ([#5936](https://github.com/AztecProtocol/aztec-packages/issues/5936)) ([9af68d8](https://github.com/AztecProtocol/aztec-packages/commit/9af68d8bd59a84e014567b429e9c9b4aed7fee74)) +* Dynamic assertion payloads v2 ([#5949](https://github.com/AztecProtocol/aztec-packages/issues/5949)) ([405bdf6](https://github.com/AztecProtocol/aztec-packages/commit/405bdf6a297b81e0c3fda303cf2b1480eaea69f1)) +* **experimental:** Add `comptime` keyword (https://github.com/noir-lang/noir/pull/4840) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Handle `BrilligCall` opcodes in the debugger (https://github.com/noir-lang/noir/pull/4897) ([3b91791](https://github.com/AztecProtocol/aztec-packages/commit/3b9179118369137880277f1444f0e3f94b3f5e79)) +* Implement `Eq` trait on `BoundedVec` (https://github.com/noir-lang/noir/pull/4830) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Implement canonical key registry 5609 and implement shared mutable getter from another contract 5689 ([#5723](https://github.com/AztecProtocol/aztec-packages/issues/5723)) ([15b569f](https://github.com/AztecProtocol/aztec-packages/commit/15b569f24e55d374bfb5a54c8771118653e5e77c)) +* Implement recursive verification in the parity circuits ([#6006](https://github.com/AztecProtocol/aztec-packages/issues/6006)) ([a5b6dac](https://github.com/AztecProtocol/aztec-packages/commit/a5b6dacd5512d7a035655845381b2c720b1e550a)) +* Keshas skipping plus conditions for grand prod relations ([#5766](https://github.com/AztecProtocol/aztec-packages/issues/5766)) ([d8fcfb5](https://github.com/AztecProtocol/aztec-packages/commit/d8fcfb590f788b911111010e20458797d76f5779)) +* Lalrpop lexer prototype (https://github.com/noir-lang/noir/pull/4656) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Make public l1tol2 message consumption take leafIndex ([#5805](https://github.com/AztecProtocol/aztec-packages/issues/5805)) ([302e3bb](https://github.com/AztecProtocol/aztec-packages/commit/302e3bbb2d7a7d54f362026edb314f3d3596b6d6)) +* More robust spot shutdown + CI commandline ([#5825](https://github.com/AztecProtocol/aztec-packages/issues/5825)) ([12064f9](https://github.com/AztecProtocol/aztec-packages/commit/12064f95abb3125933eb55996abb978c4aeaad53)) +* Naive structured execution trace ([#5853](https://github.com/AztecProtocol/aztec-packages/issues/5853)) ([23aab17](https://github.com/AztecProtocol/aztec-packages/commit/23aab171b17d0dfb840621a74266496ac270b3e8)) +* **nargo:** Handle call stacks for multiple Acir calls (https://github.com/noir-lang/noir/pull/4711) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Narrow ABI encoding errors down to target problem argument/field (https://github.com/noir-lang/noir/pull/4798) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* **p2p:** DiscV5 Peer Discovery ([#5652](https://github.com/AztecProtocol/aztec-packages/issues/5652)) ([0e81642](https://github.com/AztecProtocol/aztec-packages/commit/0e8164239b6a1180fd292e37faf1a0e64aa9cff4)) +* Prove the public kernel circuits ([#5778](https://github.com/AztecProtocol/aztec-packages/issues/5778)) ([f9a843a](https://github.com/AztecProtocol/aztec-packages/commit/f9a843a00ff41ef39b958ae7f5a24bdbc1b1add2)) +* Prove then verify flow for honk ([#5957](https://github.com/AztecProtocol/aztec-packages/issues/5957)) ([099346e](https://github.com/AztecProtocol/aztec-packages/commit/099346ebbab9428f57bfffdc03e8bede5c2e2bed)) +* Re-introducing update command ([#5946](https://github.com/AztecProtocol/aztec-packages/issues/5946)) ([13153d0](https://github.com/AztecProtocol/aztec-packages/commit/13153d02c8b0eb9cae1b7c0436fe1a1ddb49734f)) +* Remove slow updates tree ([#5954](https://github.com/AztecProtocol/aztec-packages/issues/5954)) ([52a1631](https://github.com/AztecProtocol/aztec-packages/commit/52a1631b59297ce062eda14a10e99e552f7fa706)) +* Reserve keyword `super` (https://github.com/noir-lang/noir/pull/4836) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Serialize public kernel private inputs ([#5971](https://github.com/AztecProtocol/aztec-packages/issues/5971)) ([0c712b9](https://github.com/AztecProtocol/aztec-packages/commit/0c712b9c0f69bad0da3910add5adba40622d3cea)) +* Simplify `BoundedVec::eq` (https://github.com/noir-lang/noir/pull/4838) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Squashing transient note hashes and nullifiers ([#6059](https://github.com/AztecProtocol/aztec-packages/issues/6059)) ([2b8b2c3](https://github.com/AztecProtocol/aztec-packages/commit/2b8b2c3bcbed425027f343bd2b90fd6380e2ddc4)) +* Sync from aztec-packages (https://github.com/noir-lang/noir/pull/4792) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Sync from aztec-packages (https://github.com/noir-lang/noir/pull/4833) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Sync from aztec-packages (https://github.com/noir-lang/noir/pull/4902) ([3b91791](https://github.com/AztecProtocol/aztec-packages/commit/3b9179118369137880277f1444f0e3f94b3f5e79)) +* Validate globals in public kernel ([#6031](https://github.com/AztecProtocol/aztec-packages/issues/6031)) ([82b17c8](https://github.com/AztecProtocol/aztec-packages/commit/82b17c8f0e9207db803fd3b824e63bac25ea69f6)) +* Verify public data reads ([#5701](https://github.com/AztecProtocol/aztec-packages/issues/5701)) ([323f59f](https://github.com/AztecProtocol/aztec-packages/commit/323f59f55bcd64e32725d1ed5aab72d5b9dbe31d)) +* Wire gas from public execution to kernels ([#5941](https://github.com/AztecProtocol/aztec-packages/issues/5941)) ([6894fc7](https://github.com/AztecProtocol/aztec-packages/commit/6894fc759cc4cd4e77d297fe6164cd39478ece4a)) + + +### Bug Fixes + +* `test_native.sh` not running all noir tests ([#6075](https://github.com/AztecProtocol/aztec-packages/issues/6075)) ([cc7676e](https://github.com/AztecProtocol/aztec-packages/commit/cc7676e87a7002f14b1b77b7c13f88f71355ec5b)) +* Args + selector in deploy.nr ([#5948](https://github.com/AztecProtocol/aztec-packages/issues/5948)) ([100744f](https://github.com/AztecProtocol/aztec-packages/commit/100744f89b676a03990c2d29aa0b48da77be5d8d)) +* **avm-simulator:** L1TOL2MESSAGEEXISTS opcode ([#5807](https://github.com/AztecProtocol/aztec-packages/issues/5807)) ([71b60f3](https://github.com/AztecProtocol/aztec-packages/commit/71b60f32c3b3781dda1c79bb6a926050bad7bb55)) +* **avm:** Comments and assert ([#5956](https://github.com/AztecProtocol/aztec-packages/issues/5956)) ([ae50219](https://github.com/AztecProtocol/aztec-packages/commit/ae502199b84999418d461ed5d0d6fca0c60494c5)) +* **avm:** Do not scale CALLDATACOPY base cost with size ([#5879](https://github.com/AztecProtocol/aztec-packages/issues/5879)) ([99e12b1](https://github.com/AztecProtocol/aztec-packages/commit/99e12b1abd7e66e871b41572a54cee63b5300d96)) +* Bigint corruption in lmdb ([#6002](https://github.com/AztecProtocol/aztec-packages/issues/6002)) ([703e0c1](https://github.com/AztecProtocol/aztec-packages/commit/703e0c1e2c2a5703410ff5fd4c1a135131254a53)) +* Calculate tx fee using current constants in public kernel ([#6066](https://github.com/AztecProtocol/aztec-packages/issues/6066)) ([c359d79](https://github.com/AztecProtocol/aztec-packages/commit/c359d796e72c215edf1af06c54d9287ee87df425)) +* Catch panics from EC point creation (e.g. the point is at infinity) (https://github.com/noir-lang/noir/pull/4790) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Check if a runner is available + safer refcount for spot life ([#5793](https://github.com/AztecProtocol/aztec-packages/issues/5793)) ([67077a1](https://github.com/AztecProtocol/aztec-packages/commit/67077a11250cb28dbef890009668341524b85d8b)) +* **ci:** Always run merge-check ([#6065](https://github.com/AztecProtocol/aztec-packages/issues/6065)) ([b43b84f](https://github.com/AztecProtocol/aztec-packages/commit/b43b84f5993e6950c5f081c3c77e9549dc7fddbe)) +* **ci:** Deploy_npm script ([#5817](https://github.com/AztecProtocol/aztec-packages/issues/5817)) ([df1c3c4](https://github.com/AztecProtocol/aztec-packages/commit/df1c3c4c706a44847b25a66d27544eedc508cf62)) +* **ci:** Merge check fails ungracefully on spot issues ([#5887](https://github.com/AztecProtocol/aztec-packages/issues/5887)) ([3683f0b](https://github.com/AztecProtocol/aztec-packages/commit/3683f0bb034ea59258c587d70d0517ee2ed00b91)) +* **ci:** Race condition when making spot in multiple PRs ([#5798](https://github.com/AztecProtocol/aztec-packages/issues/5798)) ([18e75b8](https://github.com/AztecProtocol/aztec-packages/commit/18e75b85bcd6eec53cee3a5da854a8d27e3f186e)) +* Deploy L1 contracts before starting node ([#5969](https://github.com/AztecProtocol/aztec-packages/issues/5969)) ([1908139](https://github.com/AztecProtocol/aztec-packages/commit/190813911c5e4fc7533525478ceca4162170fa6b)) +* **docs:** Fix admonition in contract class protocol spec ([#6017](https://github.com/AztecProtocol/aztec-packages/issues/6017)) ([12bfc15](https://github.com/AztecProtocol/aztec-packages/commit/12bfc15923ee4b7b57e50ac714953cb8129e7d5d)) +* **docs:** Fix formatting in protocol specs ([#5882](https://github.com/AztecProtocol/aztec-packages/issues/5882)) ([07fc143](https://github.com/AztecProtocol/aztec-packages/commit/07fc1434ac780f8a35533775e26ef2bd9e190816)) +* **docs:** Tutorial fixes ([#5600](https://github.com/AztecProtocol/aztec-packages/issues/5600)) ([6421467](https://github.com/AztecProtocol/aztec-packages/commit/642146705857cf34eb0f9feab665977fb2d8fb02)) +* Don't refcount spot ([#5812](https://github.com/AztecProtocol/aztec-packages/issues/5812)) ([98e8da0](https://github.com/AztecProtocol/aztec-packages/commit/98e8da094dbac1c06f800f82bd89181a6b9039b5)) +* Don't reuse brillig with slice arguments ([#5800](https://github.com/AztecProtocol/aztec-packages/issues/5800)) ([be9f24c](https://github.com/AztecProtocol/aztec-packages/commit/be9f24c16484b26a1eb88bcf35b785553160995d)) +* **experimental:** Skip over comptime functions in scan pass (https://github.com/noir-lang/noir/pull/4893) ([2e64428](https://github.com/AztecProtocol/aztec-packages/commit/2e64428af9525bd8c390931061505f7b48d729a4)) +* Fix and reenable fees e2e tests ([#5877](https://github.com/AztecProtocol/aztec-packages/issues/5877)) ([165e62f](https://github.com/AztecProtocol/aztec-packages/commit/165e62f38239f25cc6595bb43f435e9f4673fd83)) +* Fix curve parameters for bigints (https://github.com/noir-lang/noir/pull/4900) ([2e64428](https://github.com/AztecProtocol/aztec-packages/commit/2e64428af9525bd8c390931061505f7b48d729a4)) +* Fix panic when returning a zeroed unit value (https://github.com/noir-lang/noir/pull/4797) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Fix relation skipping for sumcheck ([#6092](https://github.com/AztecProtocol/aztec-packages/issues/6092)) ([1449c33](https://github.com/AztecProtocol/aztec-packages/commit/1449c338ca79f8d72b71484546aa46ddebb21779)) +* Hotfix stopped instance terminate ([#6037](https://github.com/AztecProtocol/aztec-packages/issues/6037)) ([005c71c](https://github.com/AztecProtocol/aztec-packages/commit/005c71cff4b592f89833c5556d827e55b7678b7b)) +* Issue 4682 and add solver for unconstrained bigintegers (https://github.com/noir-lang/noir/pull/4729) ([beab8c9](https://github.com/AztecProtocol/aztec-packages/commit/beab8c93857536e07fa37994213fc664a5864013)) +* Make discv5 test deterministic ([#5968](https://github.com/AztecProtocol/aztec-packages/issues/5968)) ([41749a5](https://github.com/AztecProtocol/aztec-packages/commit/41749a5148b9b5360659e06155cd09d8d7f2a78e)) +* MemoryFifo return null when empty and timeout 0 ([#5753](https://github.com/AztecProtocol/aztec-packages/issues/5753)) ([27129e6](https://github.com/AztecProtocol/aztec-packages/commit/27129e6d483e787abea5084c029e560d5d4b5b0e)) +* Merge-check ([#5873](https://github.com/AztecProtocol/aztec-packages/issues/5873)) ([c999dae](https://github.com/AztecProtocol/aztec-packages/commit/c999dae76580ea63486aaa17edb774736dc32b89)) +* Move fixtures to @aztec/circuits.js/testing/fixtures ([#5826](https://github.com/AztecProtocol/aztec-packages/issues/5826)) ([fb7a617](https://github.com/AztecProtocol/aztec-packages/commit/fb7a6175b185725e607d28a8930a15d88f84e117)) +* Nested array equality (https://github.com/noir-lang/noir/pull/4903) ([3b91791](https://github.com/AztecProtocol/aztec-packages/commit/3b9179118369137880277f1444f0e3f94b3f5e79)) +* Proper field inversion for bigints (https://github.com/noir-lang/noir/pull/4802) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Refuse to start sequencer without a prover ([#6000](https://github.com/AztecProtocol/aztec-packages/issues/6000)) ([b30d0b6](https://github.com/AztecProtocol/aztec-packages/commit/b30d0b6481b0f0b2241f1fcc9ec9bc0f82308ce9)) +* Remove tx.origin ([#5765](https://github.com/AztecProtocol/aztec-packages/issues/5765)) ([c8784d7](https://github.com/AztecProtocol/aztec-packages/commit/c8784d7994937bfae9d23f5d17eb914bae92d8dc)), closes [#5756](https://github.com/AztecProtocol/aztec-packages/issues/5756) +* Reset the noir-gates-diff report on master (https://github.com/noir-lang/noir/pull/4878) ([2e64428](https://github.com/AztecProtocol/aztec-packages/commit/2e64428af9525bd8c390931061505f7b48d729a4)) +* Revert "feat: Sync from noir" ([#6034](https://github.com/AztecProtocol/aztec-packages/issues/6034)) ([6383a09](https://github.com/AztecProtocol/aztec-packages/commit/6383a09ce5d9ab581af5d458bdcb65f92d9011fb)) +* **Revert:** "refactor: purge unconstrained functions where possible" ([#5911](https://github.com/AztecProtocol/aztec-packages/issues/5911)) ([c36246b](https://github.com/AztecProtocol/aztec-packages/commit/c36246bb692bf9a3d8e338bbc26a3ce801f0e389)) +* Set gas settings in bench ([#5796](https://github.com/AztecProtocol/aztec-packages/issues/5796)) ([86d8176](https://github.com/AztecProtocol/aztec-packages/commit/86d8176279fdc07d3b0eed91f226985e2b15f54e)) +* Start spot label ([#5810](https://github.com/AztecProtocol/aztec-packages/issues/5810)) ([96da333](https://github.com/AztecProtocol/aztec-packages/commit/96da3334af2b4b1815f9d3eb9839840c0b76d5bc)) +* Start-spot.yml ([#5824](https://github.com/AztecProtocol/aztec-packages/issues/5824)) ([3cf9c2c](https://github.com/AztecProtocol/aztec-packages/commit/3cf9c2c908b361437050e97fcdf67359727eff8b)) +* Temporarily exclude bytecode from class id computation ([#5857](https://github.com/AztecProtocol/aztec-packages/issues/5857)) ([55ff125](https://github.com/AztecProtocol/aztec-packages/commit/55ff1251c2c1c02ecbbaadfa38764c5847fee910)) +* Update noir-gates-diff commit to use master reference report (https://github.com/noir-lang/noir/pull/4891) ([2e64428](https://github.com/AztecProtocol/aztec-packages/commit/2e64428af9525bd8c390931061505f7b48d729a4)) +* Use correct gates diff commit now that master has been reset ([#6004](https://github.com/AztecProtocol/aztec-packages/issues/6004)) ([d8e5af4](https://github.com/AztecProtocol/aztec-packages/commit/d8e5af4eb023f68140d8cebd39d1d15b4683a4a3)) + + +### Miscellaneous + +* `create_fixed_base_constraint` cleanup ([#6047](https://github.com/AztecProtocol/aztec-packages/issues/6047)) ([e1d6526](https://github.com/AztecProtocol/aztec-packages/commit/e1d6526b34f03458f258c0f0fa6967b5f20035f4)) +* `TransparentNote` cleanup ([#5904](https://github.com/AztecProtocol/aztec-packages/issues/5904)) ([febf00f](https://github.com/AztecProtocol/aztec-packages/commit/febf00fb841407d54f42634560146568c383f80a)) +* Add `as_array` and remove `_slice` variants of hash functions (https://github.com/noir-lang/noir/pull/4675) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Add benchmarks for serializing a dummy program (https://github.com/noir-lang/noir/pull/4813) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Add error conversion from `InterpreterError` (https://github.com/noir-lang/noir/pull/4896) ([e615a83](https://github.com/AztecProtocol/aztec-packages/commit/e615a831a12b78644b798e12395d970bf5601948)) +* Add error conversion from `InterpreterError` (https://github.com/noir-lang/noir/pull/4896) ([078aa61](https://github.com/AztecProtocol/aztec-packages/commit/078aa61b06557aba74ac9cce557ee6bd05040feb)) +* Add Hir -> Ast conversion (https://github.com/noir-lang/noir/pull/4788) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Add target for individual e2e tests ([#6009](https://github.com/AztecProtocol/aztec-packages/issues/6009)) ([e2842a6](https://github.com/AztecProtocol/aztec-packages/commit/e2842a6e02bd42792c8ee8ca03316e5aa4902f5b)) +* Adding devcontainer with create aztec app ([#5752](https://github.com/AztecProtocol/aztec-packages/issues/5752)) ([c72f34e](https://github.com/AztecProtocol/aztec-packages/commit/c72f34ef0ff582269db245643909e02b66a4d37a)) +* Adding devcontainer with create aztec app ([#5849](https://github.com/AztecProtocol/aztec-packages/issues/5849)) ([eb1cfef](https://github.com/AztecProtocol/aztec-packages/commit/eb1cfefc4ff11802a97127f10ab30fb5487335fd)) +* Allow expressions in constant generation ([#5839](https://github.com/AztecProtocol/aztec-packages/issues/5839)) ([cb1e25b](https://github.com/AztecProtocol/aztec-packages/commit/cb1e25b8c6f203d8a7e4beb2f027d72bee981695)) +* **avm-simulator:** Remove AvmContext::raw_* external calls ([#5869](https://github.com/AztecProtocol/aztec-packages/issues/5869)) ([0c9d0b4](https://github.com/AztecProtocol/aztec-packages/commit/0c9d0b4e611472e0e8718449f6d8f2451e0391a0)) +* **avm:** More test cleanup ([#5771](https://github.com/AztecProtocol/aztec-packages/issues/5771)) ([23d0070](https://github.com/AztecProtocol/aztec-packages/commit/23d0070095bf7d32cfdcf97e7aea348753bb7492)) +* **avm:** Negative unit tests for AVM CAST opcode ([#5907](https://github.com/AztecProtocol/aztec-packages/issues/5907)) ([4465e3b](https://github.com/AztecProtocol/aztec-packages/commit/4465e3be870963ea435d9a0cd063397020442f0b)), closes [#5908](https://github.com/AztecProtocol/aztec-packages/issues/5908) +* **avm:** Re-enable proof in some unit tests ([#6056](https://github.com/AztecProtocol/aztec-packages/issues/6056)) ([0ebee28](https://github.com/AztecProtocol/aztec-packages/commit/0ebee28b14042417956a02a3247af68f4f13dcf5)), closes [#6019](https://github.com/AztecProtocol/aztec-packages/issues/6019) +* **aztec-nr:** Unencrypted logs go behind context ([#5871](https://github.com/AztecProtocol/aztec-packages/issues/5871)) ([6a5ad7c](https://github.com/AztecProtocol/aztec-packages/commit/6a5ad7ccfe7fc17237a5a6237493c1fbb6af6d53)) +* Bump `rustls` to v0.21.11 (https://github.com/noir-lang/noir/pull/4895) ([2e64428](https://github.com/AztecProtocol/aztec-packages/commit/2e64428af9525bd8c390931061505f7b48d729a4)) +* Bump MSRV to `1.74.1` (https://github.com/noir-lang/noir/pull/4873) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Bump public call depth ([#5845](https://github.com/AztecProtocol/aztec-packages/issues/5845)) ([b61502f](https://github.com/AztecProtocol/aztec-packages/commit/b61502f372e5b09d8bff138fdb74e3175b5186ff)) +* Bundle spot runner + target more spot types ([#6012](https://github.com/AztecProtocol/aztec-packages/issues/6012)) ([d51c8b8](https://github.com/AztecProtocol/aztec-packages/commit/d51c8b8698187b4a69aadf1ce47f1565d71d2827)) +* Check working copy is clean before extract-repo ([#5851](https://github.com/AztecProtocol/aztec-packages/issues/5851)) ([8ff9767](https://github.com/AztecProtocol/aztec-packages/commit/8ff9767c213d172ee1568aeedaa7265ead7b5466)) +* **ci:** Address start-runner edge-cases ([#5888](https://github.com/AztecProtocol/aztec-packages/issues/5888)) ([564b893](https://github.com/AztecProtocol/aztec-packages/commit/564b893486375e88945bdeb63364bca374f376fb)) +* **ci:** Back to on-demand ([#5998](https://github.com/AztecProtocol/aztec-packages/issues/5998)) ([f2f15f0](https://github.com/AztecProtocol/aztec-packages/commit/f2f15f0808c7b03a3ef90afae4b71015cfe1b9fd)) +* **ci:** Better docker prune ([#5889](https://github.com/AztecProtocol/aztec-packages/issues/5889)) ([b5a8e02](https://github.com/AztecProtocol/aztec-packages/commit/b5a8e02edf44bacc3415e478b75b182c5b352ca2)) +* **ci:** Don't interleave docker prunes ([#5914](https://github.com/AztecProtocol/aztec-packages/issues/5914)) ([2b51fee](https://github.com/AztecProtocol/aztec-packages/commit/2b51fee7dee663ee4a8cc54b5a5412d862d04862)) +* **ci:** Don't use redirected earthly ([#5909](https://github.com/AztecProtocol/aztec-packages/issues/5909)) ([2e55713](https://github.com/AztecProtocol/aztec-packages/commit/2e557130ace2f6db555fa27eab80ccfc18f0d88e)) +* **ci:** Fix alerts on msrv issues (https://github.com/noir-lang/noir/pull/4816) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* **ci:** Fix concurrency key ([#5962](https://github.com/AztecProtocol/aztec-packages/issues/5962)) ([7eb164f](https://github.com/AztecProtocol/aztec-packages/commit/7eb164f28a65426482557cc5dfcb31b9e7c23ab9)) +* **ci:** Hotfix runners not starting ([067e460](https://github.com/AztecProtocol/aztec-packages/commit/067e4607019c17dad7c3861734c4bee0e849fbad)) +* **ci:** Make syncing out to Noir manual ([#5997](https://github.com/AztecProtocol/aztec-packages/issues/5997)) ([1801db8](https://github.com/AztecProtocol/aztec-packages/commit/1801db88640b0e012fa32650bf8074587709ef83)) +* **ci:** Move yarn-project-test to new CI ([#5850](https://github.com/AztecProtocol/aztec-packages/issues/5850)) ([d8254ef](https://github.com/AztecProtocol/aztec-packages/commit/d8254efe958898d28dbe25474b9eb21cebb4ed2c)) +* **ci:** Notify internal Slack channel when CI breaks on master ([#5788](https://github.com/AztecProtocol/aztec-packages/issues/5788)) ([70b3f3f](https://github.com/AztecProtocol/aztec-packages/commit/70b3f3f1aebbb626014d54e121e841938407bdaf)) +* **ci:** Notify on ARM failures ([#5847](https://github.com/AztecProtocol/aztec-packages/issues/5847)) ([bdb59cb](https://github.com/AztecProtocol/aztec-packages/commit/bdb59cb4dc2c691798be3d1ef46d422f8fcd930d)) +* **ci:** Prevent haywire logs ([#5966](https://github.com/AztecProtocol/aztec-packages/issues/5966)) ([b12f609](https://github.com/AztecProtocol/aztec-packages/commit/b12f60994fdd54cb4d8e18e444c207e319f9d6a6)) +* **ci:** Reenable deploy tests ([#6011](https://github.com/AztecProtocol/aztec-packages/issues/6011)) ([087a624](https://github.com/AztecProtocol/aztec-packages/commit/087a624689ca34de4ac6dca759cf5e644a163b37)) +* **ci:** Reenable spot ([348b34f](https://github.com/AztecProtocol/aztec-packages/commit/348b34f868e98c1e6dc388b164b0df6ee131ae6c)) +* **ci:** Remove devnet deployments (for now) ([#5912](https://github.com/AztecProtocol/aztec-packages/issues/5912)) ([d9c1ee9](https://github.com/AztecProtocol/aztec-packages/commit/d9c1ee938ea8ff94639f29e457bd5e04ab2b9e8a)) +* **ci:** Use on-demand for now ([#5933](https://github.com/AztecProtocol/aztec-packages/issues/5933)) ([f77636f](https://github.com/AztecProtocol/aztec-packages/commit/f77636f686d5416c9e2f893ed40730a08b48a5ee)) +* Clean up and clarify some translator flavor logic ([#5965](https://github.com/AztecProtocol/aztec-packages/issues/5965)) ([242b364](https://github.com/AztecProtocol/aztec-packages/commit/242b364aacdf662cd6dab6254562ab5f61a58731)) +* Clean up stopped instances ([#6030](https://github.com/AztecProtocol/aztec-packages/issues/6030)) ([1318bd5](https://github.com/AztecProtocol/aztec-packages/commit/1318bd5493e65ac8f478d74bc1537dea2facd575)) +* **debugger:** Docs (https://github.com/noir-lang/noir/pull/4145) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Delete dead code (https://github.com/noir-lang/noir/pull/4906) ([e615a83](https://github.com/AztecProtocol/aztec-packages/commit/e615a831a12b78644b798e12395d970bf5601948)) +* Delete dead code (https://github.com/noir-lang/noir/pull/4906) ([078aa61](https://github.com/AztecProtocol/aztec-packages/commit/078aa61b06557aba74ac9cce557ee6bd05040feb)) +* Delete flake.lock (https://github.com/noir-lang/noir/pull/4855) ([e615a83](https://github.com/AztecProtocol/aztec-packages/commit/e615a831a12b78644b798e12395d970bf5601948)) +* Delete flake.lock (https://github.com/noir-lang/noir/pull/4855) ([078aa61](https://github.com/AztecProtocol/aztec-packages/commit/078aa61b06557aba74ac9cce557ee6bd05040feb)) +* Delete unnecessary Prover.toml file (https://github.com/noir-lang/noir/pull/4829) ([beab8c9](https://github.com/AztecProtocol/aztec-packages/commit/beab8c93857536e07fa37994213fc664a5864013)) +* Delete unused brillig methods (https://github.com/noir-lang/noir/pull/4887) ([8f73f18](https://github.com/AztecProtocol/aztec-packages/commit/8f73f18f3c07de0fd5e247ade5a48109c37c1bc5)) +* Do not aggregate note decryption time for benchmarks ([#6032](https://github.com/AztecProtocol/aztec-packages/issues/6032)) ([658a880](https://github.com/AztecProtocol/aztec-packages/commit/658a880fe40273e16cb65bbc18ede7740895baf4)) +* Do not bootstrap cache if working copy is dirty ([#6033](https://github.com/AztecProtocol/aztec-packages/issues/6033)) ([3671932](https://github.com/AztecProtocol/aztec-packages/commit/367193253670a1d61ffa440d94dad4b9d068e72f)) +* **docs:** Fix migration notes ([#6083](https://github.com/AztecProtocol/aztec-packages/issues/6083)) ([e1f3e32](https://github.com/AztecProtocol/aztec-packages/commit/e1f3e320f15003282ca5b5ea707471cfcd1b6354)) +* **docs:** Fix wrong Nargo.toml workspace examples (https://github.com/noir-lang/noir/pull/4822) ([beab8c9](https://github.com/AztecProtocol/aztec-packages/commit/beab8c93857536e07fa37994213fc664a5864013)) +* **docs:** Remove 'yellow paper' reference from protocol specs ([#5872](https://github.com/AztecProtocol/aztec-packages/issues/5872)) ([b348ec1](https://github.com/AztecProtocol/aztec-packages/commit/b348ec149b7df0d4620a79d501834a6590078160)) +* **docs:** Remove link to play.noir-lang.org (https://github.com/noir-lang/noir/pull/4872) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* **experimental:** Add scan pass and `into_expression` for comptime interpreter (https://github.com/noir-lang/noir/pull/4884) ([2e64428](https://github.com/AztecProtocol/aztec-packages/commit/2e64428af9525bd8c390931061505f7b48d729a4)) +* **experimental:** Improve variable not defined error message in comptime interpreter (https://github.com/noir-lang/noir/pull/4889) ([2e64428](https://github.com/AztecProtocol/aztec-packages/commit/2e64428af9525bd8c390931061505f7b48d729a4)) +* Extend SharedMutable tests ([#6005](https://github.com/AztecProtocol/aztec-packages/issues/6005)) ([4cee8e0](https://github.com/AztecProtocol/aztec-packages/commit/4cee8e0644780e527395da452a831055ec41a4c7)) +* Fix alerts on rust msrv (https://github.com/noir-lang/noir/pull/4817) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Fix and reenable e2e account init fees test ([#5878](https://github.com/AztecProtocol/aztec-packages/issues/5878)) ([cec8191](https://github.com/AztecProtocol/aztec-packages/commit/cec819178635b41c3b310431afa435bea207d925)) +* Fix formatting and serialization that fails CI run ([#5942](https://github.com/AztecProtocol/aztec-packages/issues/5942)) ([da67f18](https://github.com/AztecProtocol/aztec-packages/commit/da67f181b09439e2e2e04209ed3d84c21c7cc6bf)) +* Fix typo in error message ([#5139](https://github.com/AztecProtocol/aztec-packages/issues/5139)) ([b194f83](https://github.com/AztecProtocol/aztec-packages/commit/b194f83188f0e874a1f4c67a512370d3efcf883b)) +* Flag account init test as flakey ([ea030e5](https://github.com/AztecProtocol/aztec-packages/commit/ea030e534b965d154b00ececd5974606dd85f217)) +* Flag two failing e2e tests as flakey until we fix them ([901ae87](https://github.com/AztecProtocol/aztec-packages/commit/901ae87795ba39420258e5d70b92221f11f7d20e)) +* Improve `compute_note_hash_and_nullifier` autogeneration and `NoteProcessor` warnings ([#5838](https://github.com/AztecProtocol/aztec-packages/issues/5838)) ([566f25c](https://github.com/AztecProtocol/aztec-packages/commit/566f25c25744501ce1ae31243820ef549d9b1f30)) +* Improved naming in `TxExecutionRequest` ([#6014](https://github.com/AztecProtocol/aztec-packages/issues/6014)) ([f2364d4](https://github.com/AztecProtocol/aztec-packages/commit/f2364d40f850414029ed967eb05c48b5be2ffff6)) +* Integrate new key store ([#5731](https://github.com/AztecProtocol/aztec-packages/issues/5731)) ([ab9fe78](https://github.com/AztecProtocol/aztec-packages/commit/ab9fe780e8a9fc3187a02b37ddbefa609d3bff8f)), closes [#5720](https://github.com/AztecProtocol/aztec-packages/issues/5720) +* Introducing re-export for poseidon2 ([#5898](https://github.com/AztecProtocol/aztec-packages/issues/5898)) ([03a87b8](https://github.com/AztecProtocol/aztec-packages/commit/03a87b8d97b72f8144ef83b679eed564048d4683)), closes [#5863](https://github.com/AztecProtocol/aztec-packages/issues/5863) +* Lift run-e2e to yarn-project earthfile ([#6018](https://github.com/AztecProtocol/aztec-packages/issues/6018)) ([b7900b8](https://github.com/AztecProtocol/aztec-packages/commit/b7900b88a66bfd9d75b92ed05a4236dda41b2013)) +* Migrate blacklist token to use shared mutable ([#5885](https://github.com/AztecProtocol/aztec-packages/issues/5885)) ([26c1eec](https://github.com/AztecProtocol/aztec-packages/commit/26c1eecc76613c7c7883031691672ba36fb16152)) +* More explicit `self` parameter in `Into` trait (https://github.com/noir-lang/noir/pull/4867) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* No implicit overrides ([#5792](https://github.com/AztecProtocol/aztec-packages/issues/5792)) ([0fafaef](https://github.com/AztecProtocol/aztec-packages/commit/0fafaef8eb92ba261c1aefe1daab2539caab0bea)) +* Nuking CLI ([#5865](https://github.com/AztecProtocol/aztec-packages/issues/5865)) ([c48c913](https://github.com/AztecProtocol/aztec-packages/commit/c48c91349dd592520ee33d4c45bc2d3913883541)), closes [#5894](https://github.com/AztecProtocol/aztec-packages/issues/5894) +* Nuking unused keys.nr ([#5910](https://github.com/AztecProtocol/aztec-packages/issues/5910)) ([1d3af93](https://github.com/AztecProtocol/aztec-packages/commit/1d3af93f26d7b09389debe4b7046ae3359ff1893)) +* Optimize poseidon2 implementation (https://github.com/noir-lang/noir/pull/4807) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Patch jest to not use JSON serialization in message passing ([#5883](https://github.com/AztecProtocol/aztec-packages/issues/5883)) ([1c24c8e](https://github.com/AztecProtocol/aztec-packages/commit/1c24c8e53f190c7b1ac3b4d8896abb4ab6b5712b)) +* Prepare ScheduledValueChange for mutable delays. ([#6085](https://github.com/AztecProtocol/aztec-packages/issues/6085)) ([cfa850b](https://github.com/AztecProtocol/aztec-packages/commit/cfa850bbbad9ff54a7efd9fd7a045a6d3f158ebf)) +* ProvingKey has ProverPolynomials ([#5940](https://github.com/AztecProtocol/aztec-packages/issues/5940)) ([0a64279](https://github.com/AztecProtocol/aztec-packages/commit/0a64279ba1b2b3bb6627c675b8a0b116be17f579)) +* Purge unconstrained functions where possible ([#5819](https://github.com/AztecProtocol/aztec-packages/issues/5819)) ([ce84161](https://github.com/AztecProtocol/aztec-packages/commit/ce8416174f360a4a00cc70c20c8f2d99354aec2e)), closes [#5451](https://github.com/AztecProtocol/aztec-packages/issues/5451) +* Purging portal addresses ([#5842](https://github.com/AztecProtocol/aztec-packages/issues/5842)) ([4faccad](https://github.com/AztecProtocol/aztec-packages/commit/4faccad569e39228b0f3fbf741fc95e3a189e276)) +* Redo typo PR by dockercui ([#5930](https://github.com/AztecProtocol/aztec-packages/issues/5930)) ([b23e42f](https://github.com/AztecProtocol/aztec-packages/commit/b23e42f5f897936bb9607ba94e57f31723d9984b)) +* Redo typo PR by satyambnsal ([#5929](https://github.com/AztecProtocol/aztec-packages/issues/5929)) ([d28b1cb](https://github.com/AztecProtocol/aztec-packages/commit/d28b1cbc0364c1d760187ffa7263c147e9295dd4)) +* Redo typo PR by socialsister ([#5931](https://github.com/AztecProtocol/aztec-packages/issues/5931)) ([e817f78](https://github.com/AztecProtocol/aztec-packages/commit/e817f78158e895807151f6b451cb506cab9c2510)) +* Redo typo script ([#5926](https://github.com/AztecProtocol/aztec-packages/issues/5926)) ([41fa87e](https://github.com/AztecProtocol/aztec-packages/commit/41fa87e1216eeb6ff774eb1925797f9ae721c70b)) +* Reenable account init fees e2e ([#5938](https://github.com/AztecProtocol/aztec-packages/issues/5938)) ([49c45c3](https://github.com/AztecProtocol/aztec-packages/commit/49c45c38f01e5a2034f81506089640d93c87744d)) +* Refactor e2e tests to use the new simulate fn ([#5854](https://github.com/AztecProtocol/aztec-packages/issues/5854)) ([e7d2aff](https://github.com/AztecProtocol/aztec-packages/commit/e7d2aff3a1922dc685bc859901dffdb83933dff2)) +* Refactor public cross chain tests for speed ([#6082](https://github.com/AztecProtocol/aztec-packages/issues/6082)) ([6065a6c](https://github.com/AztecProtocol/aztec-packages/commit/6065a6c4157a2d356964f4c5476425da55e09728)) +* Refactor recursive verifier tests ([#6063](https://github.com/AztecProtocol/aztec-packages/issues/6063)) ([94a2d61](https://github.com/AztecProtocol/aztec-packages/commit/94a2d61d10d8e21d0080b7ea3a8b283f8dd0162f)) +* Refactor token blacklist test for speed ([#6054](https://github.com/AztecProtocol/aztec-packages/issues/6054)) ([ab36d7e](https://github.com/AztecProtocol/aztec-packages/commit/ab36d7e42ccd6403c5b6967c4e2b319ab7b85d37)) +* Release Noir(0.28.0) (https://github.com/noir-lang/noir/pull/4776) ([3b91791](https://github.com/AztecProtocol/aztec-packages/commit/3b9179118369137880277f1444f0e3f94b3f5e79)) +* Remove `Opcode::Brillig` from ACIR ([#5995](https://github.com/AztecProtocol/aztec-packages/issues/5995)) ([ffd5f46](https://github.com/AztecProtocol/aztec-packages/commit/ffd5f460fce8b1f12265730f97c8cfcd3a4774ca)) +* Remove `SecondaryAttribute::Event` (https://github.com/noir-lang/noir/pull/4868) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Remove empty yarn.lock ([#5835](https://github.com/AztecProtocol/aztec-packages/issues/5835)) ([c3dd039](https://github.com/AztecProtocol/aztec-packages/commit/c3dd039e5d2a779cc9bda1c0ac46306563914578)) +* Remove get_portal_address oracle ([#5816](https://github.com/AztecProtocol/aztec-packages/issues/5816)) ([67c2823](https://github.com/AztecProtocol/aztec-packages/commit/67c2823e3bb302d4d7a28bca03de468c92336680)) +* Remove initialisation of logger in `acvm_js` tests (https://github.com/noir-lang/noir/pull/4850) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Remove l1 gas ([#6069](https://github.com/AztecProtocol/aztec-packages/issues/6069)) ([0e3705f](https://github.com/AztecProtocol/aztec-packages/commit/0e3705f2591c1da36778c316d8b7ab914f5d6757)) +* Remove private kernel snapshot test ([#5829](https://github.com/AztecProtocol/aztec-packages/issues/5829)) ([9434784](https://github.com/AztecProtocol/aztec-packages/commit/9434784b12f5e5402e93596110ee2e131317b251)) +* Remove pub wildcard import of ast into `noirc_frontend` root (https://github.com/noir-lang/noir/pull/4862) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Remove unnecessary casts in `BoundedVec` (https://github.com/noir-lang/noir/pull/4831) ([beab8c9](https://github.com/AztecProtocol/aztec-packages/commit/beab8c93857536e07fa37994213fc664a5864013)) +* Rename 'global' to 'function' in the monomorphization pass (https://github.com/noir-lang/noir/pull/4774) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Rename capture to end_setup ([#6008](https://github.com/AztecProtocol/aztec-packages/issues/6008)) ([61e61ab](https://github.com/AztecProtocol/aztec-packages/commit/61e61aba60ec02d12141ef396c4e827c800d57bf)) +* Renaming `noir-compiler` as `builder` ([#5951](https://github.com/AztecProtocol/aztec-packages/issues/5951)) ([185e57d](https://github.com/AztecProtocol/aztec-packages/commit/185e57d51e8bbf6194628ce62db3dd44f11634a4)) +* Reorganize gas fields in structs ([#5828](https://github.com/AztecProtocol/aztec-packages/issues/5828)) ([e26d342](https://github.com/AztecProtocol/aztec-packages/commit/e26d342c5646425da31d95c50ce94025e5c6d053)) +* Replace queue with facade over CircuitProver ([#5972](https://github.com/AztecProtocol/aztec-packages/issues/5972)) ([dafb3ed](https://github.com/AztecProtocol/aztec-packages/commit/dafb3edc799b2adaf285ffe57b41630040c68449)) +* Replace relative paths to noir-protocol-circuits ([b723534](https://github.com/AztecProtocol/aztec-packages/commit/b723534db2fcbd3399aca722354df7c45ee8a84f)) +* Replace relative paths to noir-protocol-circuits ([20057b2](https://github.com/AztecProtocol/aztec-packages/commit/20057b25bbf9c6b007fe3595eca7a2cff872aa52)) +* Replace relative paths to noir-protocol-circuits ([543ff13](https://github.com/AztecProtocol/aztec-packages/commit/543ff131c32cd005de2e83fe2af59b132c5896de)) +* Replace relative paths to noir-protocol-circuits ([d0622cf](https://github.com/AztecProtocol/aztec-packages/commit/d0622cffa2dfffdf8bd96cc34627a78aeb8a72e5)) +* Replace relative paths to noir-protocol-circuits ([41d6e81](https://github.com/AztecProtocol/aztec-packages/commit/41d6e81426090c5b8c50787123bac826a732204d)) +* Replace relative paths to noir-protocol-circuits ([c0c8e3f](https://github.com/AztecProtocol/aztec-packages/commit/c0c8e3f880076d20cca96a3c92a1484abdcc66a0)) +* Replace relative paths to noir-protocol-circuits ([8b33a58](https://github.com/AztecProtocol/aztec-packages/commit/8b33a58e095815d5b131ab3fbd668c4e88680e13)) +* Replace relative paths to noir-protocol-circuits ([ce4a010](https://github.com/AztecProtocol/aztec-packages/commit/ce4a010c7c075cb68bed91e0123d4fcecc7c6938)) +* Replace use of PublicContext with interface ([#5840](https://github.com/AztecProtocol/aztec-packages/issues/5840)) ([834067f](https://github.com/AztecProtocol/aztec-packages/commit/834067f12b07a36b9348a368b83d61d789c5c22b)) +* Reset noir-gates-diff report on master ([#6003](https://github.com/AztecProtocol/aztec-packages/issues/6003)) ([7f01f7d](https://github.com/AztecProtocol/aztec-packages/commit/7f01f7d16230fe011a3f52db9e477a958796b202)) +* Revert "Check working copy is clean before extract-repo ([#5851](https://github.com/AztecProtocol/aztec-packages/issues/5851))" ([ec21fb8](https://github.com/AztecProtocol/aztec-packages/commit/ec21fb8251c34d535fd0c5e08f354cfa22c25320)) +* **revert:** "chore(ci): don't use redirected earthly" ([#6062](https://github.com/AztecProtocol/aztec-packages/issues/6062)) ([26cba9e](https://github.com/AztecProtocol/aztec-packages/commit/26cba9e4ef9b63bf100e451b66cfe3ea62ab416c)) +* Rework workspace structure for utils crates (https://github.com/noir-lang/noir/pull/4886) ([e615a83](https://github.com/AztecProtocol/aztec-packages/commit/e615a831a12b78644b798e12395d970bf5601948)) +* Rework workspace structure for utils crates (https://github.com/noir-lang/noir/pull/4886) ([078aa61](https://github.com/AztecProtocol/aztec-packages/commit/078aa61b06557aba74ac9cce557ee6bd05040feb)) +* Run clippy (https://github.com/noir-lang/noir/pull/4810) ([84c930a](https://github.com/AztecProtocol/aztec-packages/commit/84c930a912ca9ed0d9c0ce2436309a4e9a840bcb)) +* Run flakey e2e tests on CI but allow failure ([#5937](https://github.com/AztecProtocol/aztec-packages/issues/5937)) ([a074251](https://github.com/AztecProtocol/aztec-packages/commit/a07425184d08d647588e3778221740e724b1b052)) +* Run noir projects tests in earthly ([#6024](https://github.com/AztecProtocol/aztec-packages/issues/6024)) ([e950433](https://github.com/AztecProtocol/aztec-packages/commit/e9504333dcb25c3f9bd1344743a0e12e7719ab2e)) +* Simplify computation of pow for each sumcheck round ([#5903](https://github.com/AztecProtocol/aztec-packages/issues/5903)) ([74a9d5d](https://github.com/AztecProtocol/aztec-packages/commit/74a9d5d6736a4376e40e501765974b9686ca738e)) +* Temporarily skip failing gas tests ([#5874](https://github.com/AztecProtocol/aztec-packages/issues/5874)) ([ad55af0](https://github.com/AztecProtocol/aztec-packages/commit/ad55af0d44b3c818d5e42fe75bb72fa95e88c309)) +* Update noir README (https://github.com/noir-lang/noir/pull/4856) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Update NoirJS tutorial (https://github.com/noir-lang/noir/pull/4861) ([cea5107](https://github.com/AztecProtocol/aztec-packages/commit/cea51073be4ecc65a3c8d36cfe107df8390b853f)) +* Use BrilligCall for unconstrained main and update AVM transpiler ([#5797](https://github.com/AztecProtocol/aztec-packages/issues/5797)) ([3fb94c0](https://github.com/AztecProtocol/aztec-packages/commit/3fb94c0cd5ffba20a99b97c0088ae5ef357c205d)) +* Use new mock.get_last_params() for public storage writes ([#5823](https://github.com/AztecProtocol/aztec-packages/issues/5823)) ([6b0f919](https://github.com/AztecProtocol/aztec-packages/commit/6b0f919d83209a83e5d1900942a160424b30fe22)) +* Using poseidon2 when computing a nullifier ([#5906](https://github.com/AztecProtocol/aztec-packages/issues/5906)) ([3a10e5e](https://github.com/AztecProtocol/aztec-packages/commit/3a10e5e75b8053dfea13a4901873d42ca01ca7c2)), closes [#5832](https://github.com/AztecProtocol/aztec-packages/issues/5832) [#1205](https://github.com/AztecProtocol/aztec-packages/issues/1205) +* Validate instance deployer address every time we request it ([#5848](https://github.com/AztecProtocol/aztec-packages/issues/5848)) ([2422891](https://github.com/AztecProtocol/aztec-packages/commit/2422891fa021cfb4c83b91849ff1f22baa93a4b9)) +* Workaround earthly flake ([#5811](https://github.com/AztecProtocol/aztec-packages/issues/5811)) ([dd3a521](https://github.com/AztecProtocol/aztec-packages/commit/dd3a521b59b950871645306179d23a3f332ef6f3)) +* Yarn build:dev don't clear terminal ([#5970](https://github.com/AztecProtocol/aztec-packages/issues/5970)) ([b3fdb3b](https://github.com/AztecProtocol/aztec-packages/commit/b3fdb3b59e887974b89db0eb209e16b0630b1360)) + + +### Documentation + +* Addition around Nargo.toml search ([#5943](https://github.com/AztecProtocol/aztec-packages/issues/5943)) ([d1350da](https://github.com/AztecProtocol/aztec-packages/commit/d1350da9e3d78fa53ccd5663219f70c67df4c66d)) +* Aztec smart contract tutorial - crowdfunding ([#5786](https://github.com/AztecProtocol/aztec-packages/issues/5786)) ([91cc0a4](https://github.com/AztecProtocol/aztec-packages/commit/91cc0a424031b9b8346cc9182f303d1468b1179b)) +* Gas and accounting ([#5855](https://github.com/AztecProtocol/aztec-packages/issues/5855)) ([d0b3f06](https://github.com/AztecProtocol/aztec-packages/commit/d0b3f06ff29d5e5ac99097cb1ab2906190eec5c3)) +* Migration notes for GasOpts in public calls ([#5822](https://github.com/AztecProtocol/aztec-packages/issues/5822)) ([edeea3d](https://github.com/AztecProtocol/aztec-packages/commit/edeea3dfe425b83b36c981dde3ce169e33aaece9)) +* Remove mentions of slow updates ([#5884](https://github.com/AztecProtocol/aztec-packages/issues/5884)) ([029d1e5](https://github.com/AztecProtocol/aztec-packages/commit/029d1e5d4ff679f73dce72779cb316a1d8c7eda8)) +* Shared state ([#5963](https://github.com/AztecProtocol/aztec-packages/issues/5963)) ([86c106f](https://github.com/AztecProtocol/aztec-packages/commit/86c106f122b3fe0daa5853f7824bb68abadf70d0)) +* Update emit_event.md ([#5964](https://github.com/AztecProtocol/aztec-packages/issues/5964)) ([616a8f3](https://github.com/AztecProtocol/aztec-packages/commit/616a8f328f893ab563b1d90c5c627572cf838968)) +* Update info around VERSION ([#5891](https://github.com/AztecProtocol/aztec-packages/issues/5891)) ([e1eb98e](https://github.com/AztecProtocol/aztec-packages/commit/e1eb98e85e6ef6ca87f502036426457c8c2a7efc)) + ## [0.35.1](https://github.com/AztecProtocol/aztec-packages/compare/aztec-packages-v0.35.0...aztec-packages-v0.35.1) (2024-04-16) diff --git a/Earthfile b/Earthfile index 8ecbc266c06..5a42a251dfd 100644 --- a/Earthfile +++ b/Earthfile @@ -1,5 +1,5 @@ VERSION 0.8 -FROM ubuntu:lunar +FROM ubuntu:noble build-ci: BUILD ./avm-transpiler/+build @@ -16,15 +16,22 @@ build-ci: BUILD ./yarn-project/+end-to-end BUILD ./yarn-project/+aztec -build-ci-small: - BUILD ./yarn-project/end-to-end/+e2e-escrow-contract - build: # yarn-project has the entry point to Aztec BUILD ./yarn-project/+build test-end-to-end: - BUILD ./yarn-project/end-to-end/+test-all + BUILD ./yarn-project/end-to-end+e2e-tests bench: RUN echo hi + +release-meta: + COPY .release-please-manifest.json /usr/src/.release-please-manifest.json + SAVE ARTIFACT /usr/src /usr/src + +scripts: + FROM ubuntu:lunar + RUN apt-get update && apt-get install -y awscli + COPY scripts /usr/src/scripts + SAVE ARTIFACT /usr/src/scripts scripts \ No newline at end of file diff --git a/avm-transpiler/Earthfile b/avm-transpiler/Earthfile index b2c77f9b3f9..4c3102dec8c 100644 --- a/avm-transpiler/Earthfile +++ b/avm-transpiler/Earthfile @@ -1,18 +1,17 @@ VERSION 0.8 IMPORT ../noir AS noir -# we rely on noir source, which this image has -FROM noir+nargo -# move noir contents to /usr/src/noir -RUN mv /usr/src /noir && mkdir /usr/src && mv /noir /usr/src -# work in avm-transpiler -WORKDIR /usr/src/avm-transpiler +source: + # we rely on noir source, which this image has + FROM noir+nargo -RUN apt-get update && apt-get install -y git + # move noir contents to /usr/src/noir + RUN mv /usr/src /noir && mkdir /usr/src && mv /noir /usr/src + # work in avm-transpiler + WORKDIR /usr/src/avm-transpiler -COPY --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml . + COPY --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml . -source: # for debugging rebuilds RUN echo CONTENT HASH $(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') | tee .content-hash @@ -21,7 +20,7 @@ build: RUN ./scripts/bootstrap_native.sh SAVE ARTIFACT target/release/avm-transpiler avm-transpiler -run: +run: #TODO needed? FROM ubuntu:focal COPY +build/avm-transpiler /usr/src/avm-transpiler diff --git a/avm-transpiler/src/utils.rs b/avm-transpiler/src/utils.rs index 97667a386c0..af0bc6bc9cc 100644 --- a/avm-transpiler/src/utils.rs +++ b/avm-transpiler/src/utils.rs @@ -15,16 +15,17 @@ pub fn extract_brillig_from_acir_program(program: &Program) -> &[BrilligOpcode] assert_eq!( program.functions.len(), 1, - "An AVM program should have only a single ACIR function flagged as 'BrilligCall'" + "An AVM program should have only a single ACIR function with a 'BrilligCall'" ); - let opcodes = &program.functions[0].opcodes; + let main_function = &program.functions[0]; + let opcodes = &main_function.opcodes; assert_eq!( opcodes.len(), 1, - "An AVM program should have only a single ACIR function flagged as 'BrilligCall'" + "An AVM program should only have a single `BrilligCall`" ); match opcodes[0] { - Opcode::BrilligCall { .. } => {} + Opcode::BrilligCall { id, .. } => assert_eq!(id, 0, "The ID of the `BrilligCall` must be 0 as we have a single `Brillig` function"), _ => panic!("Tried to extract a Brillig program from its ACIR wrapper opcode, but the opcode doesn't contain Brillig!"), } assert_eq!( diff --git a/aztec-up/README.md b/aztec-up/README.md index 630d0e0fc5c..4a8ebf22fa2 100644 --- a/aztec-up/README.md +++ b/aztec-up/README.md @@ -14,6 +14,7 @@ a users `PATH` variable in their shell startup script so they can be found. - `aztec-nargo` - A build of `nargo` from `noir` that is guaranteed to be version aligned. Provides compiler, lsp and more. - `aztec-sandbox` - A wrapper around docker-compose that launches services needed for sandbox testing. - `aztec-up` - A tool to upgrade the aztec toolchain to the latest, or specific versions. +- `aztec-builder` - A useful tool for projects to generate ABIs and update their dependencies. After installed, you can use `aztec-up` to upgrade or install specific versions. diff --git a/aztec-up/bin/aztec-cli b/aztec-up/bin/aztec-builder similarity index 80% rename from aztec-up/bin/aztec-cli rename to aztec-up/bin/aztec-builder index 3624338b5c3..a676553772f 100755 --- a/aztec-up/bin/aztec-cli +++ b/aztec-up/bin/aztec-builder @@ -5,4 +5,4 @@ export ENV_VARS_TO_INJECT="PXE_URL PRIVATE_KEY DEBUG" export PXE_URL=${PXE_URL:-"http://host.docker.internal:8080"} export ETHEREUM_HOST=${ETHEREUM_HOST:-"http://host.docker.internal:8545"} -$(dirname $0)/.aztec-run aztecprotocol/cli $@ +$(dirname $0)/.aztec-run aztecprotocol/aztec-builder $@ diff --git a/barretenberg/.gitrepo b/barretenberg/.gitrepo index ee434d12668..0b27d459758 100644 --- a/barretenberg/.gitrepo +++ b/barretenberg/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/AztecProtocol/barretenberg branch = master - commit = 31ec6089135e12bf85240d50bc8bac066918dfa0 - parent = 1449c338ca79f8d72b71484546aa46ddebb21779 + commit = 11d32452a6e82ea4d4738ba36be8d99ad977d15f + parent = f95de6b498d34e138cd55f88340917c6881eec6b method = merge cmdver = 0.4.6 diff --git a/barretenberg/CHANGELOG.md b/barretenberg/CHANGELOG.md index d37b74ee13c..b6ba610067e 100644 --- a/barretenberg/CHANGELOG.md +++ b/barretenberg/CHANGELOG.md @@ -1,5 +1,52 @@ # Changelog +## [0.36.0](https://github.com/AztecProtocol/aztec-packages/compare/barretenberg-v0.35.1...barretenberg-v0.36.0) (2024-04-30) + + +### ⚠ BREAKING CHANGES + +* remove `Opcode::Brillig` from ACIR ([#5995](https://github.com/AztecProtocol/aztec-packages/issues/5995)) +* change backend width to 4 ([#5374](https://github.com/AztecProtocol/aztec-packages/issues/5374)) + +### Features + +* `variable_base_scalar_mul` blackbox func ([#6039](https://github.com/AztecProtocol/aztec-packages/issues/6039)) ([81142fe](https://github.com/AztecProtocol/aztec-packages/commit/81142fe799338e6ed73b30eeac4468c1345f6fab)) +* Avm mem trace validation ([#6025](https://github.com/AztecProtocol/aztec-packages/issues/6025)) ([3a3afb5](https://github.com/AztecProtocol/aztec-packages/commit/3a3afb57ab8b6b3f11b7a7799d557436638c8cd3)), closes [#5950](https://github.com/AztecProtocol/aztec-packages/issues/5950) +* **avm:** Avm circuit FDIV opcode ([#5958](https://github.com/AztecProtocol/aztec-packages/issues/5958)) ([fed5b6d](https://github.com/AztecProtocol/aztec-packages/commit/fed5b6dd1ee310fc90404a3e5ec9eb02ad7dbc10)), closes [#5953](https://github.com/AztecProtocol/aztec-packages/issues/5953) +* **avm:** CAST opcode implementation ([#5477](https://github.com/AztecProtocol/aztec-packages/issues/5477)) ([a821bcc](https://github.com/AztecProtocol/aztec-packages/commit/a821bccef7b1894140f0495510d7c6b4eefde821)), closes [#5466](https://github.com/AztecProtocol/aztec-packages/issues/5466) +* **avm:** Negative tests ([#5919](https://github.com/AztecProtocol/aztec-packages/issues/5919)) ([8a5ece7](https://github.com/AztecProtocol/aztec-packages/commit/8a5ece7548a86d099ac6a166f04882624b8d95fd)) +* **avm:** Shift relations ([#5716](https://github.com/AztecProtocol/aztec-packages/issues/5716)) ([a516637](https://github.com/AztecProtocol/aztec-packages/commit/a51663707b96914b0a300440611748ce44fbe933)) +* Avoiding redundant computation in PG ([#5844](https://github.com/AztecProtocol/aztec-packages/issues/5844)) ([9f57733](https://github.com/AztecProtocol/aztec-packages/commit/9f5773353aa0261fa07a81704bcadcee513d42c5)) +* Change backend width to 4 ([#5374](https://github.com/AztecProtocol/aztec-packages/issues/5374)) ([3f24fc2](https://github.com/AztecProtocol/aztec-packages/commit/3f24fc2cdb56eff6da6e47062d2a2a3dc0fa4bd2)) +* Circuit simulator for Ultra and GoblinUltra verifiers ([#1195](https://github.com/AztecProtocol/aztec-packages/issues/1195)) ([0032a3a](https://github.com/AztecProtocol/aztec-packages/commit/0032a3a55dea5e4c9051dbc36607288f8ca1be4a)) +* Dynamic assertion payloads v2 ([#5949](https://github.com/AztecProtocol/aztec-packages/issues/5949)) ([405bdf6](https://github.com/AztecProtocol/aztec-packages/commit/405bdf6a297b81e0c3fda303cf2b1480eaea69f1)) +* Implement recursive verification in the parity circuits ([#6006](https://github.com/AztecProtocol/aztec-packages/issues/6006)) ([a5b6dac](https://github.com/AztecProtocol/aztec-packages/commit/a5b6dacd5512d7a035655845381b2c720b1e550a)) +* Keshas skipping plus conditions for grand prod relations ([#5766](https://github.com/AztecProtocol/aztec-packages/issues/5766)) ([d8fcfb5](https://github.com/AztecProtocol/aztec-packages/commit/d8fcfb590f788b911111010e20458797d76f5779)) +* Naive structured execution trace ([#5853](https://github.com/AztecProtocol/aztec-packages/issues/5853)) ([23aab17](https://github.com/AztecProtocol/aztec-packages/commit/23aab171b17d0dfb840621a74266496ac270b3e8)) +* Prove then verify flow for honk ([#5957](https://github.com/AztecProtocol/aztec-packages/issues/5957)) ([099346e](https://github.com/AztecProtocol/aztec-packages/commit/099346ebbab9428f57bfffdc03e8bede5c2e2bed)) + + +### Bug Fixes + +* **avm:** Comments and assert ([#5956](https://github.com/AztecProtocol/aztec-packages/issues/5956)) ([ae50219](https://github.com/AztecProtocol/aztec-packages/commit/ae502199b84999418d461ed5d0d6fca0c60494c5)) +* Fix relation skipping for sumcheck ([#6092](https://github.com/AztecProtocol/aztec-packages/issues/6092)) ([1449c33](https://github.com/AztecProtocol/aztec-packages/commit/1449c338ca79f8d72b71484546aa46ddebb21779)) +* Remove tx.origin ([#5765](https://github.com/AztecProtocol/aztec-packages/issues/5765)) ([c8784d7](https://github.com/AztecProtocol/aztec-packages/commit/c8784d7994937bfae9d23f5d17eb914bae92d8dc)), closes [#5756](https://github.com/AztecProtocol/aztec-packages/issues/5756) + + +### Miscellaneous + +* `create_fixed_base_constraint` cleanup ([#6047](https://github.com/AztecProtocol/aztec-packages/issues/6047)) ([e1d6526](https://github.com/AztecProtocol/aztec-packages/commit/e1d6526b34f03458f258c0f0fa6967b5f20035f4)) +* **avm:** Negative unit tests for AVM CAST opcode ([#5907](https://github.com/AztecProtocol/aztec-packages/issues/5907)) ([4465e3b](https://github.com/AztecProtocol/aztec-packages/commit/4465e3be870963ea435d9a0cd063397020442f0b)), closes [#5908](https://github.com/AztecProtocol/aztec-packages/issues/5908) +* **avm:** Re-enable proof in some unit tests ([#6056](https://github.com/AztecProtocol/aztec-packages/issues/6056)) ([0ebee28](https://github.com/AztecProtocol/aztec-packages/commit/0ebee28b14042417956a02a3247af68f4f13dcf5)), closes [#6019](https://github.com/AztecProtocol/aztec-packages/issues/6019) +* Clean up and clarify some translator flavor logic ([#5965](https://github.com/AztecProtocol/aztec-packages/issues/5965)) ([242b364](https://github.com/AztecProtocol/aztec-packages/commit/242b364aacdf662cd6dab6254562ab5f61a58731)) +* Do not bootstrap cache if working copy is dirty ([#6033](https://github.com/AztecProtocol/aztec-packages/issues/6033)) ([3671932](https://github.com/AztecProtocol/aztec-packages/commit/367193253670a1d61ffa440d94dad4b9d068e72f)) +* ProvingKey has ProverPolynomials ([#5940](https://github.com/AztecProtocol/aztec-packages/issues/5940)) ([0a64279](https://github.com/AztecProtocol/aztec-packages/commit/0a64279ba1b2b3bb6627c675b8a0b116be17f579)) +* Purging portal addresses ([#5842](https://github.com/AztecProtocol/aztec-packages/issues/5842)) ([4faccad](https://github.com/AztecProtocol/aztec-packages/commit/4faccad569e39228b0f3fbf741fc95e3a189e276)) +* Refactor recursive verifier tests ([#6063](https://github.com/AztecProtocol/aztec-packages/issues/6063)) ([94a2d61](https://github.com/AztecProtocol/aztec-packages/commit/94a2d61d10d8e21d0080b7ea3a8b283f8dd0162f)) +* Remove `Opcode::Brillig` from ACIR ([#5995](https://github.com/AztecProtocol/aztec-packages/issues/5995)) ([ffd5f46](https://github.com/AztecProtocol/aztec-packages/commit/ffd5f460fce8b1f12265730f97c8cfcd3a4774ca)) +* Remove l1 gas ([#6069](https://github.com/AztecProtocol/aztec-packages/issues/6069)) ([0e3705f](https://github.com/AztecProtocol/aztec-packages/commit/0e3705f2591c1da36778c316d8b7ab914f5d6757)) +* Simplify computation of pow for each sumcheck round ([#5903](https://github.com/AztecProtocol/aztec-packages/issues/5903)) ([74a9d5d](https://github.com/AztecProtocol/aztec-packages/commit/74a9d5d6736a4376e40e501765974b9686ca738e)) + ## [0.35.1](https://github.com/AztecProtocol/aztec-packages/compare/barretenberg-v0.35.0...barretenberg-v0.35.1) (2024-04-16) diff --git a/barretenberg/acir_tests/headless-test/src/index.ts b/barretenberg/acir_tests/headless-test/src/index.ts index 5c91dc8fa15..1902e510e87 100644 --- a/barretenberg/acir_tests/headless-test/src/index.ts +++ b/barretenberg/acir_tests/headless-test/src/index.ts @@ -51,7 +51,7 @@ const readWitnessFile = (path: string): Uint8Array => { // Set up the command-line interface const program = new Command(); program.option("-v, --verbose", "verbose logging"); -program.option("-c, --crs-path ", "ignored (here for compatability)"); +program.option("-c, --crs-path ", "ignored (here for compatibility)"); program .command("prove_and_verify") diff --git a/barretenberg/acir_tests/run_acir_tests.sh b/barretenberg/acir_tests/run_acir_tests.sh index 88189a43438..e75f8fb1a6b 100755 --- a/barretenberg/acir_tests/run_acir_tests.sh +++ b/barretenberg/acir_tests/run_acir_tests.sh @@ -35,7 +35,7 @@ export BIN CRS_PATH VERBOSE BRANCH cd acir_tests # Convert them to array -SKIP_ARRAY=(diamond_deps_0 workspace workspace_default_member) +SKIP_ARRAY=(diamond_deps_0 workspace workspace_default_member witness_compression) function test() { cd $1 diff --git a/barretenberg/acir_tests/sol-test/src/index.js b/barretenberg/acir_tests/sol-test/src/index.js index b63921d2709..59d43bea54f 100644 --- a/barretenberg/acir_tests/sol-test/src/index.js +++ b/barretenberg/acir_tests/sol-test/src/index.js @@ -137,7 +137,7 @@ const readPublicInputs = (proofAsFields) => { * Get Anvil * * Creates an anvil instance on a random port, and returns the instance and the port - * If the port is alredy allocated, it will try again + * If the port is already allocated, it will try again * @returns {[ChildProcess, Number]} [anvil, port] */ const getAnvil = async () => { diff --git a/barretenberg/cpp/CMakeLists.txt b/barretenberg/cpp/CMakeLists.txt index f33ff1034bd..8b0b82e934f 100644 --- a/barretenberg/cpp/CMakeLists.txt +++ b/barretenberg/cpp/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.24 FATAL_ERROR) project( Barretenberg DESCRIPTION "BN254 elliptic curve library, and PLONK SNARK prover" - VERSION 0.35.1 # x-release-please-version + VERSION 0.36.0 # x-release-please-version LANGUAGES CXX C ) # Insert version into `bb` config file diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json index 24538c08c0b..a42cf23517c 100644 --- a/barretenberg/cpp/CMakePresets.json +++ b/barretenberg/cpp/CMakePresets.json @@ -256,7 +256,7 @@ "generator": "Ninja", "toolchainFile": "cmake/toolchains/wasm32-wasi.cmake", "environment": { - "WASI_SDK_PREFIX": "${sourceDir}/src/wasi-sdk", + "WASI_SDK_PREFIX": "/opt/wasi-sdk", "CC": "$env{WASI_SDK_PREFIX}/bin/clang", "CXX": "$env{WASI_SDK_PREFIX}/bin/clang++", "AR": "$env{WASI_SDK_PREFIX}/bin/llvm-ar", diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index 346d1be3d07..e4057ec965c 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -1,80 +1,13 @@ VERSION 0.8 -build-base: - ARG TARGETARCH - FROM --platform=linux/$TARGETARCH ubuntu:lunar - RUN apt-get update && apt-get install -y \ - build-essential \ - curl \ - git \ - cmake \ - lsb-release \ - wget \ - software-properties-common \ - gnupg \ - ninja-build \ - npm \ - libssl-dev \ - jq \ - bash \ - libstdc++6 \ - clang-format \ - clang-16 - - IF [ $TARGETARCH = arm64 ] - # publish arm after, assumes x86 already exists, becomes multiplatform image - SAVE IMAGE --push aztecprotocol/bb-ubuntu-lunar - FROM --platform=linux/amd64 aztecprotocol/bb-ubuntu-lunar:x86-latest - SAVE IMAGE --push aztecprotocol/bb-ubuntu-lunar - ELSE - SAVE IMAGE --push aztecprotocol/bb-ubuntu-lunar:x86-latest - END - -build-wasi-sdk-image: - WORKDIR / - RUN git clone --recursive https://github.com/WebAssembly/wasi-sdk.git - WORKDIR /wasi-sdk - RUN git checkout 9389ea5eeec98afc61039683ae92c6147fee9c54 - ENV NINJA_FLAGS=-v - ENV MAKEFLAGS="-j$(nproc)" - RUN make build/llvm.BUILT - RUN make build/wasi-libc.BUILT - RUN make build/compiler-rt.BUILT - RUN make build/libcxx.BUILT - RUN make build/config.BUILT - SAVE ARTIFACT build/install/opt/wasi-sdk - -build-wasi-sdk: - ARG TARGETARCH - # Wrapper just exists share files. - FROM scratch - WORKDIR /usr/src - COPY +get-wasi-sdk-image/wasi-sdk wasi-sdk - SAVE ARTIFACT wasi-sdk - SAVE IMAGE --push aztecprotocol/cache:wasi-sdk-threads-v21.0-$TARGETARCH - -get-wasi-sdk-threads: - ARG TARGETARCH - # If this is failing, we need to run earthly --push +build-wasi-sdk - FROM aztecprotocol/cache:wasi-sdk-threads-v21.0-$TARGETARCH - SAVE ARTIFACT wasi-sdk - -get-wasi-sdk: - # NOTE: currently only works with x86 - # TODO Align with above - FROM +source - COPY ./scripts/install-wasi-sdk.sh ./scripts/ - RUN ./scripts/install-wasi-sdk.sh - # TODO better output name to mirror wasi-sdk - SAVE ARTIFACT src/wasi-sdk-20.0 wasi-sdk - wasmtime: - FROM aztecprotocol/bb-ubuntu-lunar + FROM ubuntu:noble + RUN apt update && apt install -y curl xz-utils RUN curl https://wasmtime.dev/install.sh -sSf | bash SAVE ARTIFACT /root/.wasmtime/bin/wasmtime source: - FROM aztecprotocol/bb-ubuntu-lunar + FROM ../../build-images+build WORKDIR /usr/src/barretenberg # cpp source COPY --dir src/barretenberg src/CMakeLists.txt src @@ -106,19 +39,17 @@ preset-wasm: FROM +preset-wasm-threads SAVE ARTIFACT build/bin ELSE - COPY +get-wasi-sdk/wasi-sdk src/wasi-sdk RUN cmake --preset wasm -Bbuild && cmake --build build --target barretenberg.wasm - RUN src/wasi-sdk/bin/llvm-strip ./build/bin/barretenberg.wasm + RUN /opt/wasi-sdk/bin/llvm-strip ./build/bin/barretenberg.wasm SAVE ARTIFACT build/bin SAVE IMAGE --cache-hint END preset-wasm-threads: FROM +source - COPY +get-wasi-sdk-threads/wasi-sdk src/wasi-sdk RUN cmake --preset wasm-threads -Bbuild && cmake --build build --target barretenberg.wasm # TODO(https://github.com/AztecProtocol/barretenberg/issues/941) We currently do not strip barretenberg threaded wasm, for stack traces. - # RUN src/wasi-sdk/bin/llvm-strip ./build/bin/barretenberg.wasm + # RUN /opt/wasi-sdk/bin/llvm-strip ./build/bin/barretenberg.wasm SAVE ARTIFACT build/bin preset-gcc: @@ -189,7 +120,7 @@ bench-binaries: # Runs on the bench image, sent from the builder runner bench-ultra-honk: BUILD +wasmtime # prefetch - FROM +source + FROM ubuntu:noble COPY --dir +bench-binaries/* . # install SRS needed for proving COPY --dir ./srs_db/+build/. srs_db @@ -201,7 +132,7 @@ bench-ultra-honk: bench-client-ivc: BUILD +wasmtime # prefetch - FROM +source + FROM ubuntu:noble COPY --dir +bench-binaries/* . # install SRS needed for proving COPY --dir ./srs_db/+build/. srs_db diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 2eaafa29571..a323eb601cc 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -31,9 +31,6 @@ fi # Download ignition transcripts. (cd ./srs_db && ./download_ignition.sh 0) -# Install wasi-sdk. -./scripts/install-wasi-sdk.sh - # Attempt to just pull artefacts from CI and exit on success. [ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit @@ -82,7 +79,7 @@ AVAILABLE_MEMORY=0 case "$(uname)" in Linux*) # Check available memory on Linux - AVAILABLE_MEMORY=$(awk '/MemFree/ { printf $2 }' /proc/meminfo) + AVAILABLE_MEMORY=$(awk '/MemTotal/ { printf $2 }' /proc/meminfo) ;; *) echo "Parallel builds not supported on this operating system" @@ -90,11 +87,11 @@ case "$(uname)" in esac # This value may be too low. # If builds fail with an amount of free memory greater than this value then it should be increased. -MIN_PARALLEL_BUILD_MEMORY=32000000 +MIN_PARALLEL_BUILD_MEMORY=32854492 if [[ AVAILABLE_MEMORY -lt MIN_PARALLEL_BUILD_MEMORY ]]; then echo "System does not have enough memory for parallel builds, falling back to sequential" - build_native + build_native build_wasm build_wasm_threads else diff --git a/barretenberg/cpp/cmake/threading.cmake b/barretenberg/cpp/cmake/threading.cmake index ff60f240a16..fffffb6f83c 100644 --- a/barretenberg/cpp/cmake/threading.cmake +++ b/barretenberg/cpp/cmake/threading.cmake @@ -4,7 +4,7 @@ if(MULTITHREADING) add_link_options(-pthread) if(WASM) add_compile_options(--target=wasm32-wasi-threads) - add_link_options(--target=wasm32-wasi-threads) + add_link_options(--target=wasm32-wasi-threads -Wl,--shared-memory) endif() #add_compile_options(-fsanitize=thread) #add_link_options(-fsanitize=thread) diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang index 568f0fcd9e4..012cae9403b 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang +++ b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang @@ -20,7 +20,7 @@ RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 WORKDIR /usr/src/barretenberg/cpp COPY . . -# Build bb binary and targets needed for benchmarking. +# Build bb binary and targets needed for benchmarking. # Everything else is built as part linux-clang-assert. # Benchmark targets want to run without asserts, so get built alongside bb. RUN cmake --preset clang16 diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert index bef8e342756..3cff8c5023c 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert +++ b/barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert @@ -14,10 +14,9 @@ RUN apt update && apt install -y \ libssl-dev \ jq \ bash \ - libstdc++6 \ - clang-format + libstdc++6 -RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 +RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 && apt install -y clang-format-16 WORKDIR /usr/src/barretenberg/cpp COPY . . diff --git a/barretenberg/cpp/format.sh b/barretenberg/cpp/format.sh index ae314e96a6f..4b1b9e7cbd7 100755 --- a/barretenberg/cpp/format.sh +++ b/barretenberg/cpp/format.sh @@ -4,22 +4,22 @@ set -e if [ "$1" == "staged" ]; then echo Formatting barretenberg staged files... for FILE in $(git diff-index --diff-filter=d --relative --cached --name-only HEAD | grep -e '\.\(cpp\|hpp\|tcc\)$'); do - clang-format -i $FILE + clang-format-16 -i $FILE sed -i.bak 's/\r$//' $FILE && rm ${FILE}.bak git add $FILE done elif [ "$1" == "check" ]; then for FILE in $(find ./src -iname *.hpp -o -iname *.cpp -o -iname *.tcc | grep -v src/msgpack-c); do - clang-format --dry-run --Werror $FILE + clang-format-16 --dry-run --Werror $FILE done elif [ -n "$1" ]; then for FILE in $(git diff-index --relative --name-only $1 | grep -e '\.\(cpp\|hpp\|tcc\)$'); do - clang-format -i $FILE + clang-format-16 -i $FILE sed -i.bak 's/\r$//' $FILE && rm ${FILE}.bak done else for FILE in $(find ./src -iname *.hpp -o -iname *.cpp -o -iname *.tcc | grep -v src/msgpack-c); do - clang-format -i $FILE + clang-format-16 -i $FILE sed -i.bak 's/\r$//' $FILE && rm ${FILE}.bak done fi diff --git a/barretenberg/cpp/scripts/install-wasi-sdk.sh b/barretenberg/cpp/scripts/install-wasi-sdk.sh index 22224212170..650ab723d22 100755 --- a/barretenberg/cpp/scripts/install-wasi-sdk.sh +++ b/barretenberg/cpp/scripts/install-wasi-sdk.sh @@ -35,4 +35,8 @@ else curl -s -L https://wasi-sdk.s3.eu-west-2.amazonaws.com/yamt-wasi-sysroot-20.0.threads.tgz | tar zxf - fi # TODO(https://github.com/AztecProtocol/barretenberg/issues/906): in the future this should use earthly and a 'SAVE ARTIFACT wasi-sdk AS LOCAL wasi-sdk' -mv wasi-sdk-20.0+threads wasi-sdk +if [ "$(id -u)" -eq 0 ]; then + mv wasi-sdk-20.0+threads /opt/wasi-sdk +else + sudo mv wasi-sdk-20.0+threads /opt/wasi-sdk +fi diff --git a/barretenberg/cpp/scripts/line_count.py b/barretenberg/cpp/scripts/line_count.py new file mode 100755 index 00000000000..450eb14ccfa --- /dev/null +++ b/barretenberg/cpp/scripts/line_count.py @@ -0,0 +1,101 @@ +# Dependency: The line counting utility cloc https://github.com/AlDanial/cloc +import csv +from os import listdir, system +from pathlib import Path + +BASE_DIR = Path("src/barretenberg") +PER_FILE_REPORT_PATH = Path("build/per_file_report.csv") + +# validate that directory structure hasn't changed since last run +all_dirs = listdir("src/barretenberg") +last_all_dirs = ['bb', 'benchmark', 'commitment_schemes', 'common', 'crypto', 'dsl', 'ecc', 'eccvm', 'env', 'examples', 'flavor', 'goblin', 'grumpkin_srs_gen', 'honk', 'numeric', 'plonk', 'polynomials', 'protogalaxy', 'relations', 'serialize', + 'smt_verification', 'solidity_helpers', 'srs', 'stdlib', 'sumcheck', 'transcript', 'ultra_honk', 'wasi', 'circuit_checker', 'client_ivc', 'translator_vm', 'vm', 'execution_trace', 'plonk_honk_shared', 'stdlib_circuit_builders', 'proof_system', 'build'] +assert (all_dirs == last_all_dirs) + +# mark directories that will be covered in an audit of Barretenberg +# calculated the total number of lines weighted by complexity, with maximum complexity equal to 1 +dirs_to_audit = [ + 'bb', + # 'benchmark', + 'commitment_schemes', + # 'common', + 'crypto', + # 'dsl', + 'ecc', + 'eccvm', + # 'env', + # 'examples', + 'flavor', + 'goblin', + 'grumpkin_srs_gen', + 'honk', + 'numeric', + # 'plonk', + 'polynomials', + 'protogalaxy', + 'relations', + # 'serialize', + # 'smt_verification', + # 'solidity_helpers', + 'srs', + 'stdlib', + 'sumcheck', + 'transcript', + 'ultra_honk', + # 'wasi', + 'circuit_checker', + 'client_ivc', + 'translator_vm', + 'vm', + 'execution_trace', + 'plonk_honk_shared', + 'stdlib_circuit_builders', + 'proof_system' +] +weights = {directory: 1 for directory in dirs_to_audit} +weights["circuit_checker"] = 0.3 +weights["srs"] = 0.3 +weights["polynomials"] = 0.5 +weights["numeric"] = 0.3 +weights["ecc"] = 0.5 +weights["crypto"] = 0.5 +weights["bb"] = 0.3 + +TOTAL_NUM_CODE_LINES = 0 +# use cloc to count the lines in every file to be audited in the current agreement +system( + f"cloc --include-lang='C++','C/C++ Header' --by-file --csv --out='{PER_FILE_REPORT_PATH}' {BASE_DIR}") +with open(PER_FILE_REPORT_PATH, newline='') as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + if row['language'] != 'SUM': + path = Path(row['filename']).relative_to(BASE_DIR).parts[0] + if path in dirs_to_audit: + TOTAL_NUM_CODE_LINES += int(row['code']) * weights[path] + +TO_AUDIT_NOW = [ + "src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp", + "src/barretenberg/stdlib/primitives/bigfield/bigfield_impl.hpp", + "src/barretenberg/stdlib/primitives/bigfield/bigfield.test.cpp", + "src/barretenberg/stdlib/primitives/field/field.hpp", + "src/barretenberg/stdlib/primitives/field/field.cpp", + "src/barretenberg/stdlib/primitives/field/field.test.cpp", + "src/barretenberg/stdlib/primitives/byte_array/byte_array.hpp", + "src/barretenberg/stdlib/primitives/byte_array/byte_array.cpp", + "src/barretenberg/stdlib/primitives/byte_array/byte_array.test.cpp", + "src/barretenberg/relations/delta_range_constraint_relation.hpp", + "src/barretenberg/relations/ultra_arithmetic_relation.hpp", + "src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp", + "src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.cpp"] +counts = {} +with open(PER_FILE_REPORT_PATH, newline='') as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + if row["filename"] in TO_AUDIT_NOW: + counts[row["filename"]] = row["code"] +total = 0 +for filename, count in counts.items(): + total += int(count) + +# print the percentage to be audited +print(f"Audit covers {total/float(TOTAL_NUM_CODE_LINES):.2%}") diff --git a/barretenberg/cpp/scripts/strip-wasm.sh b/barretenberg/cpp/scripts/strip-wasm.sh index 18e3cf78d02..9d0c4c36cff 100755 --- a/barretenberg/cpp/scripts/strip-wasm.sh +++ b/barretenberg/cpp/scripts/strip-wasm.sh @@ -1,4 +1,4 @@ #!/bin/sh -./src/wasi-sdk-20.0/bin/llvm-strip ./build-wasm/bin/barretenberg.wasm +/opt/wasi-sdk/bin/llvm-strip ./build-wasm/bin/barretenberg.wasm # TODO(https://github.com/AztecProtocol/barretenberg/issues/941) We currently do not strip barretenberg threaded wasm, for stack traces. -# ./src/wasi-sdk-20.0/bin/llvm-strip ./build-wasm-threads/bin/barretenberg.wasm +# /opt/wasi-sdk/bin/llvm-strip ./build-wasm-threads/bin/barretenberg.wasm diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 57f24f01fdf..7f7b588c58d 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -23,8 +23,7 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") endif() if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 18) # We target clang18 and need this, eventually warning should be fixed or this will be unconditional. - add_compile_options(-Wno-vla-cxx-extension -pthread) - add_link_options(-Wl,--shared-memory) + add_compile_options(-Wno-vla-cxx-extension) endif() endif() diff --git a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt index 1fce39f3e92..0ada6b922c3 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(ipa_bench) add_subdirectory(client_ivc_bench) add_subdirectory(pippenger_bench) add_subdirectory(plonk_bench) +add_subdirectory(simulator_bench) add_subdirectory(protogalaxy_bench) add_subdirectory(protogalaxy_rounds_bench) add_subdirectory(relations_bench) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/basics_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/basics_bench/CMakeLists.txt index 1740ea9ecb6..3b27b458bc5 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/basics_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/basics_bench/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(basics_bench ecc) \ No newline at end of file +barretenberg_module(basics_bench ecc srs) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/basics_bench/basics.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/basics_bench/basics.bench.cpp index 03614693396..5aa278d66f9 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/basics_bench/basics.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/basics_bench/basics.bench.cpp @@ -20,9 +20,11 @@ * sequential_copy: 3.3 * */ +#include "barretenberg/commitment_schemes/commitment_key.hpp" #include "barretenberg/common/op_count.hpp" #include "barretenberg/common/thread.hpp" #include "barretenberg/ecc/curves/bn254/bn254.hpp" +#include "barretenberg/srs/global_crs.hpp" #include using namespace benchmark; @@ -304,7 +306,7 @@ void projective_point_doubling(State& state) *@details ~50000 ns * @param state */ -void scalar_multiplication(State& state) +void scalar_multiplication_bench(State& state) { numeric::RNG& engine = numeric::get_debug_randomness(); Curve::Element element = Curve::Element::random_element(&engine); @@ -364,6 +366,47 @@ void sequential_copy(State& state) } } } + +/** + * @brief Load srs for pippenger + * + */ +static void DoPippengerSetup(const benchmark::State&) +{ + bb::srs::init_crs_factory("../srs_db/ignition"); +} + +/** + * @brief Run pippenger benchmarks (can be used with wasmtime) + * + *@details(Wasmtime) ----------------------------------------------- +Benchmark Time CPU Iterations +-------------------------------------------------------------------- +pippenger/16/iterations:5 133089 us 1.3309e+11 us 5 +pippenger/17/iterations:5 255069 us 2.5507e+11 us 5 +pippenger/18/iterations:5 481599 us 4.8160e+11 us 5 +pippenger/19/iterations:5 892886 us 8.9289e+11 us 5 +pippenger/20/iterations:5 1706423 us 1.7064e+12 us 5 + */ +void pippenger(State& state) +{ + numeric::RNG& engine = numeric::get_debug_randomness(); + for (auto _ : state) { + state.PauseTiming(); + size_t num_cycles = 1 << static_cast(state.range(0)); + Polynomial pol(num_cycles); + for (size_t i = 0; i < num_cycles; i++) { + *(uint256_t*)&pol[i] = engine.get_random_uint256(); + pol[i].self_reduce_once(); + pol[i].self_reduce_once(); + pol[i].self_reduce_once(); + } + + auto ck = std::make_shared>(num_cycles); + state.ResumeTiming(); + benchmark::DoNotOptimize(ck->commit(pol)); + } +} } // namespace BENCHMARK(parallel_for_field_element_addition)->Unit(kMicrosecond)->DenseRange(0, MAX_REPETITION_LOG); @@ -377,7 +420,8 @@ BENCHMARK(ff_reduce)->Unit(kMicrosecond)->DenseRange(12, 29); BENCHMARK(projective_point_addition)->Unit(kMicrosecond)->DenseRange(12, 22); BENCHMARK(projective_point_accidental_doubling)->Unit(kMicrosecond)->DenseRange(12, 22); BENCHMARK(projective_point_doubling)->Unit(kMicrosecond)->DenseRange(12, 22); -BENCHMARK(scalar_multiplication)->Unit(kMicrosecond)->DenseRange(12, 18); +BENCHMARK(scalar_multiplication_bench)->Unit(kMicrosecond)->DenseRange(12, 18); BENCHMARK(cycle_waste)->Unit(kMicrosecond)->DenseRange(20, 30); BENCHMARK(sequential_copy)->Unit(kMicrosecond)->DenseRange(20, 25); +BENCHMARK(pippenger)->Unit(kMicrosecond)->DenseRange(16, 20)->Setup(DoPippengerSetup)->Iterations(5); BENCHMARK_MAIN(); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/CMakeLists.txt new file mode 100644 index 00000000000..9f91d3093e0 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/CMakeLists.txt @@ -0,0 +1 @@ + barretenberg_module(simulator_bench stdlib_honk_recursion stdlib_sha256 crypto_merkle_tree) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/simulator.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/simulator.bench.cpp new file mode 100644 index 00000000000..a8aee11c099 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/simulator_bench/simulator.bench.cpp @@ -0,0 +1,134 @@ +#include "barretenberg/goblin/goblin.hpp" +#include "barretenberg/goblin/mock_circuits.hpp" +#include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp" +#include + +using namespace benchmark; +using namespace bb; + +namespace { +template class SimulatorFixture : public benchmark::Fixture { + + public: + using Flavor = typename RecursiveFlavor::NativeFlavor; + using ProverInstance = ProverInstance_; + using Builder = typename Flavor::CircuitBuilder; + using VerificationKey = typename Flavor::VerificationKey; + using CircuitSimulator = typename RecursiveFlavor::CircuitBuilder; + using SimulatingVerifier = stdlib::recursion::honk::UltraRecursiveVerifier_; + + struct VerifierInput { + HonkProof proof; + std::shared_ptr verification_key; + }; + + void SetUp([[maybe_unused]] const ::benchmark::State& state) override + { + bb::srs::init_crs_factory("../srs_db/ignition"); + } + + /** + * @brief Create a Honk proof (either Ultra or GoblinUltra) for a non-trivial circuit. + * + * @param large determines whether the circuit is 2^17 or 2^19 + */ + static VerifierInput create_proof(bool large = false) + { + + auto builder = construct_mock_function_circuit(large); + auto instance = std::make_shared(builder); + UltraProver_ prover(instance); + auto ultra_proof = prover.construct_proof(); + auto verification_key = std::make_shared(instance->proving_key); + return { ultra_proof, verification_key }; + } + + /** + * @brief Populate the builder with non-trivial operations that mock a circuit encountered in practice. + * + * @param large determines whether the circuit is 2^17 or 2^19 + */ + static Builder construct_mock_function_circuit(bool large = false) + { + using InnerCurve = bb::stdlib::bn254; + using fr_ct = InnerCurve::ScalarField; + using point_ct = InnerCurve::AffineElement; + using fr = typename InnerCurve::ScalarFieldNative; + using point = typename InnerCurve::GroupNative::affine_element; + Builder builder; + + // Perform a batch mul which will add some arbitrary goblin-style ECC op gates if the circuit arithmetic is + // goblinisied otherwise it will add the conventional nonnative gates + size_t num_points = 5; + std::vector circuit_points; + std::vector circuit_scalars; + for (size_t i = 0; i < num_points; ++i) { + circuit_points.push_back(point_ct::from_witness(&builder, point::random_element())); + circuit_scalars.push_back(fr_ct::from_witness(&builder, fr::random_element())); + } + point_ct::batch_mul(circuit_points, circuit_scalars); + + // Determine number of times to execute the below operations that constitute the mock circuit logic. Note + // that the circuit size does not scale linearly with number of iterations due to e.g. amortization of lookup + + const size_t NUM_ITERATIONS_LARGE = 12; // results in circuit size 2^19 (502238 gates) + const size_t NUM_ITERATIONS_MEDIUM = 3; // results in circuit size 2^17 (124843 gates) + const size_t NUM_ITERATIONS = large ? NUM_ITERATIONS_LARGE : NUM_ITERATIONS_MEDIUM; + + stdlib::generate_sha256_test_circuit(builder, NUM_ITERATIONS); // min gates: ~39k + stdlib::generate_ecdsa_verification_test_circuit(builder, NUM_ITERATIONS); // min gates: ~41k + stdlib::generate_merkle_membership_test_circuit(builder, NUM_ITERATIONS); // min gates: ~29k + + return builder; + } +}; + +BENCHMARK_TEMPLATE_F(SimulatorFixture, GoblinNative, bb::GoblinUltraRecursiveFlavor_) +(benchmark::State& state) +{ + auto verifier_input = SimulatorFixture::create_proof(); + for (auto _ : state) { + UltraVerifier_ ultra_verifier{ verifier_input.verification_key }; + ultra_verifier.verify_proof((verifier_input.proof)); + } +} + +BENCHMARK_TEMPLATE_F(SimulatorFixture, GoblinSimulated, bb::GoblinUltraRecursiveFlavor_) +(benchmark::State& state) +{ + auto verifier_input = SimulatorFixture::create_proof(); + for (auto _ : state) { + CircuitSimulator simulator; + SimulatingVerifier ultra_verifier{ &simulator, verifier_input.verification_key }; + ultra_verifier.verify_proof((verifier_input.proof)); + } +} + +BENCHMARK_TEMPLATE_F(SimulatorFixture, UltraNative, bb::UltraRecursiveFlavor_) +(benchmark::State& state) +{ + auto verifier_input = SimulatorFixture::create_proof(); + for (auto _ : state) { + UltraVerifier_ ultra_verifier{ verifier_input.verification_key }; + ultra_verifier.verify_proof((verifier_input.proof)); + } +} + +BENCHMARK_TEMPLATE_F(SimulatorFixture, UltraSimulated, bb::UltraRecursiveFlavor_) +(benchmark::State& state) +{ + auto verifier_input = SimulatorFixture::create_proof(); + for (auto _ : state) { + CircuitSimulator simulator; + SimulatingVerifier ultra_verifier{ &simulator, verifier_input.verification_key }; + ultra_verifier.verify_proof((verifier_input.proof)); + } +} + +BENCHMARK_REGISTER_F(SimulatorFixture, GoblinSimulated)->Unit(benchmark::kMillisecond); +BENCHMARK_REGISTER_F(SimulatorFixture, UltraSimulated)->Unit(benchmark::kMillisecond); +BENCHMARK_REGISTER_F(SimulatorFixture, GoblinNative)->Unit(benchmark::kMillisecond); +BENCHMARK_REGISTER_F(SimulatorFixture, UltraNative)->Unit(benchmark::kMillisecond); + +} // namespace +BENCHMARK_MAIN(); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/circuit_checker/circuit_checker.hpp b/barretenberg/cpp/src/barretenberg/circuit_checker/circuit_checker.hpp index ec6ceed7e6d..8d232e75f3c 100644 --- a/barretenberg/cpp/src/barretenberg/circuit_checker/circuit_checker.hpp +++ b/barretenberg/cpp/src/barretenberg/circuit_checker/circuit_checker.hpp @@ -12,7 +12,8 @@ concept IsCheckable = bb::IsAnyOf, StandardCircuitBuilder_, UltraCircuitBuilder, - GoblinUltraCircuitBuilder>; + GoblinUltraCircuitBuilder, + CircuitSimulatorBN254>; /** * @brief The unified interface for check circuit functionality implemented in the specialized CircuitChecker classes @@ -28,6 +29,8 @@ class CircuitChecker { return UltraCircuitChecker::check(builder); } else if constexpr (IsStandardBuilder) { return StandardCircuitChecker::check(builder); + } else if constexpr (IsSimulator) { + return SimulatorCircuitChecker::check(builder); } else { return false; } diff --git a/barretenberg/cpp/src/barretenberg/common/fuzzer.hpp b/barretenberg/cpp/src/barretenberg/common/fuzzer.hpp index aaa2e71f676..51e9f202463 100644 --- a/barretenberg/cpp/src/barretenberg/common/fuzzer.hpp +++ b/barretenberg/cpp/src/barretenberg/common/fuzzer.hpp @@ -90,10 +90,10 @@ class FastRandom { */ template concept SimpleRng = requires(T a) { - { - a.next() - } -> std::convertible_to; - }; + { + a.next() + } -> std::convertible_to; +}; /** * @brief Concept for forcing ArgumentSizes to be size_t * @@ -101,27 +101,27 @@ concept SimpleRng = requires(T a) { */ template concept InstructionArgumentSizes = requires { - { - std::make_tuple(T::CONSTANT, - T::WITNESS, - T::CONSTANT_WITNESS, - T::ADD, - T::SUBTRACT, - T::MULTIPLY, - T::DIVIDE, - T::ADD_TWO, - T::MADD, - T::MULT_MADD, - T::MSUB_DIV, - T::SQR, - T::SQR_ADD, - T::SUBTRACT_WITH_CONSTRAINT, - T::DIVIDE_WITH_CONSTRAINTS, - T::SLICE, - T::ASSERT_ZERO, - T::ASSERT_NOT_ZERO) - } -> std::same_as>; - }; + { + std::make_tuple(T::CONSTANT, + T::WITNESS, + T::CONSTANT_WITNESS, + T::ADD, + T::SUBTRACT, + T::MULTIPLY, + T::DIVIDE, + T::ADD_TWO, + T::MADD, + T::MULT_MADD, + T::MSUB_DIV, + T::SQR, + T::SQR_ADD, + T::SUBTRACT_WITH_CONSTRAINT, + T::DIVIDE_WITH_CONSTRAINTS, + T::SLICE, + T::ASSERT_ZERO, + T::ASSERT_NOT_ZERO) + } -> std::same_as>; +}; /** * @brief Concept for Havoc Configurations @@ -129,13 +129,12 @@ concept InstructionArgumentSizes = requires { * @tparam T */ template -concept HavocConfigConstraint = - requires { - { - std::make_tuple(T::GEN_MUTATION_COUNT_LOG, T::GEN_STRUCTURAL_MUTATION_PROBABILITY) - } -> std::same_as>; - T::GEN_MUTATION_COUNT_LOG <= 7; - }; +concept HavocConfigConstraint = requires { + { + std::make_tuple(T::GEN_MUTATION_COUNT_LOG, T::GEN_STRUCTURAL_MUTATION_PROBABILITY) + } -> std::same_as>; + T::GEN_MUTATION_COUNT_LOG <= 7; +}; /** * @brief Concept specifying the class used by the fuzzer * @@ -143,12 +142,12 @@ concept HavocConfigConstraint = */ template concept ArithmeticFuzzHelperConstraint = requires { - typename T::ArgSizes; - typename T::Instruction; - typename T::ExecutionState; - typename T::ExecutionHandler; - InstructionArgumentSizes; - }; + typename T::ArgSizes; + typename T::Instruction; + typename T::ExecutionState; + typename T::ExecutionHandler; + InstructionArgumentSizes; +}; /** * @brief Fuzzer uses only composers with check_circuit function @@ -157,10 +156,10 @@ concept ArithmeticFuzzHelperConstraint = requires { */ template concept CheckableComposer = requires(T a) { - { - CircuitChecker::check(a) - } -> std::same_as; - }; + { + CircuitChecker::check(a) + } -> std::same_as; +}; /** * @brief The fuzzer can use a postprocessing function that is specific to the type being fuzzed @@ -171,10 +170,10 @@ concept CheckableComposer = requires(T a) { */ template concept PostProcessingEnabled = requires(Composer composer, Context context) { - { - T::postProcess(&composer, context) - } -> std::same_as; - }; + { + T::postProcess(&composer, context) + } -> std::same_as; +}; /** * @brief This concept is used when we want to limit the number of executions of certain instructions (for example, @@ -184,9 +183,9 @@ concept PostProcessingEnabled = requires(Composer composer, Context context) { */ template concept InstructionWeightsEnabled = requires { - typename T::InstructionWeights; - T::InstructionWeights::_LIMIT; - }; + typename T::InstructionWeights; + T::InstructionWeights::_LIMIT; +}; /** * @brief Mutate the value of a field element diff --git a/barretenberg/cpp/src/barretenberg/common/moody/blockingconcurrentqueue.h b/barretenberg/cpp/src/barretenberg/common/moody/blockingconcurrentqueue.h deleted file mode 100644 index 60d3c5ce725..00000000000 --- a/barretenberg/cpp/src/barretenberg/common/moody/blockingconcurrentqueue.h +++ /dev/null @@ -1,561 +0,0 @@ -// Provides an efficient blocking version of moodycamel::ConcurrentQueue. -// ©2015-2020 Cameron Desrochers. Distributed under the terms of the simplified -// BSD license, available at the top of concurrentqueue.h. -// Also dual-licensed under the Boost Software License (see LICENSE.md) -// Uses Jeff Preshing's semaphore implementation (under the terms of its -// separate zlib license, see lightweightsemaphore.h). - -#pragma once - -#include "concurrentqueue.h" -#include "lightweightsemaphore.h" - -#include -#include -#include -#include -#include - -namespace moodycamel { -// This is a blocking version of the queue. It has an almost identical interface to -// the normal non-blocking version, with the addition of various wait_dequeue() methods -// and the removal of producer-specific dequeue methods. -template class BlockingConcurrentQueue { - private: - typedef ::moodycamel::ConcurrentQueue ConcurrentQueue; - typedef ::moodycamel::LightweightSemaphore LightweightSemaphore; - - public: - typedef typename ConcurrentQueue::producer_token_t producer_token_t; - typedef typename ConcurrentQueue::consumer_token_t consumer_token_t; - - typedef typename ConcurrentQueue::index_t index_t; - typedef typename ConcurrentQueue::size_t size_t; - typedef typename std::make_signed::type ssize_t; - - static const size_t BLOCK_SIZE = ConcurrentQueue::BLOCK_SIZE; - static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = - ConcurrentQueue::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD; - static const size_t EXPLICIT_INITIAL_INDEX_SIZE = ConcurrentQueue::EXPLICIT_INITIAL_INDEX_SIZE; - static const size_t IMPLICIT_INITIAL_INDEX_SIZE = ConcurrentQueue::IMPLICIT_INITIAL_INDEX_SIZE; - static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = ConcurrentQueue::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; - static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = - ConcurrentQueue::EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE; - static const size_t MAX_SUBQUEUE_SIZE = ConcurrentQueue::MAX_SUBQUEUE_SIZE; - - public: - // Creates a queue with at least `capacity` element slots; note that the - // actual number of elements that can be inserted without additional memory - // allocation depends on the number of producers and the block size (e.g. if - // the block size is equal to `capacity`, only a single block will be allocated - // up-front, which means only a single producer will be able to enqueue elements - // without an extra allocation -- blocks aren't shared between producers). - // This method is not thread safe -- it is up to the user to ensure that the - // queue is fully constructed before it starts being used by other threads (this - // includes making the memory effects of construction visible, possibly with a - // memory barrier). - explicit BlockingConcurrentQueue(size_t capacity = 6 * BLOCK_SIZE) - : inner(capacity) - , sema(create(0, (int)Traits::MAX_SEMA_SPINS), - &BlockingConcurrentQueue::template destroy) - { - assert(reinterpret_cast((BlockingConcurrentQueue*)1) == - &((BlockingConcurrentQueue*)1)->inner && - "BlockingConcurrentQueue must have ConcurrentQueue as its first member"); - if (!sema) { - MOODYCAMEL_THROW(std::bad_alloc()); - } - } - - BlockingConcurrentQueue(size_t minCapacity, size_t maxExplicitProducers, size_t maxImplicitProducers) - : inner(minCapacity, maxExplicitProducers, maxImplicitProducers) - , sema(create(0, (int)Traits::MAX_SEMA_SPINS), - &BlockingConcurrentQueue::template destroy) - { - assert(reinterpret_cast((BlockingConcurrentQueue*)1) == - &((BlockingConcurrentQueue*)1)->inner && - "BlockingConcurrentQueue must have ConcurrentQueue as its first member"); - if (!sema) { - MOODYCAMEL_THROW(std::bad_alloc()); - } - } - - // Disable copying and copy assignment - BlockingConcurrentQueue(BlockingConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; - BlockingConcurrentQueue& operator=(BlockingConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; - - // Moving is supported, but note that it is *not* a thread-safe operation. - // Nobody can use the queue while it's being moved, and the memory effects - // of that move must be propagated to other threads before they can use it. - // Note: When a queue is moved, its tokens are still valid but can only be - // used with the destination queue (i.e. semantically they are moved along - // with the queue itself). - BlockingConcurrentQueue(BlockingConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT : inner(std::move(other.inner)), - sema(std::move(other.sema)) - {} - - inline BlockingConcurrentQueue& operator=(BlockingConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT - { - return swap_internal(other); - } - - // Swaps this queue's state with the other's. Not thread-safe. - // Swapping two queues does not invalidate their tokens, however - // the tokens that were created for one queue must be used with - // only the swapped queue (i.e. the tokens are tied to the - // queue's movable state, not the object itself). - inline void swap(BlockingConcurrentQueue& other) MOODYCAMEL_NOEXCEPT { swap_internal(other); } - - private: - BlockingConcurrentQueue& swap_internal(BlockingConcurrentQueue& other) - { - if (this == &other) { - return *this; - } - - inner.swap(other.inner); - sema.swap(other.sema); - return *this; - } - - public: - // Enqueues a single item (by copying it). - // Allocates memory if required. Only fails if memory allocation fails (or implicit - // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, - // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(T const& item) - { - if ((details::likely)(inner.enqueue(item))) { - sema->signal(); - return true; - } - return false; - } - - // Enqueues a single item (by moving it, if possible). - // Allocates memory if required. Only fails if memory allocation fails (or implicit - // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, - // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(T&& item) - { - if ((details::likely)(inner.enqueue(std::move(item)))) { - sema->signal(); - return true; - } - return false; - } - - // Enqueues a single item (by copying it) using an explicit producer token. - // Allocates memory if required. Only fails if memory allocation fails (or - // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(producer_token_t const& token, T const& item) - { - if ((details::likely)(inner.enqueue(token, item))) { - sema->signal(); - return true; - } - return false; - } - - // Enqueues a single item (by moving it, if possible) using an explicit producer token. - // Allocates memory if required. Only fails if memory allocation fails (or - // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(producer_token_t const& token, T&& item) - { - if ((details::likely)(inner.enqueue(token, std::move(item)))) { - sema->signal(); - return true; - } - return false; - } - - // Enqueues several items. - // Allocates memory if required. Only fails if memory allocation fails (or - // implicit production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - // is 0, or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Note: Use std::make_move_iterator if the elements should be moved instead of copied. - // Thread-safe. - template inline bool enqueue_bulk(It itemFirst, size_t count) - { - if ((details::likely)(inner.enqueue_bulk(std::forward(itemFirst), count))) { - sema->signal((LightweightSemaphore::ssize_t)(ssize_t)count); - return true; - } - return false; - } - - // Enqueues several items using an explicit producer token. - // Allocates memory if required. Only fails if memory allocation fails - // (or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Note: Use std::make_move_iterator if the elements should be moved - // instead of copied. - // Thread-safe. - template inline bool enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) - { - if ((details::likely)(inner.enqueue_bulk(token, std::forward(itemFirst), count))) { - sema->signal((LightweightSemaphore::ssize_t)(ssize_t)count); - return true; - } - return false; - } - - // Enqueues a single item (by copying it). - // Does not allocate memory. Fails if not enough room to enqueue (or implicit - // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - // is 0). - // Thread-safe. - inline bool try_enqueue(T const& item) - { - if (inner.try_enqueue(item)) { - sema->signal(); - return true; - } - return false; - } - - // Enqueues a single item (by moving it, if possible). - // Does not allocate memory (except for one-time implicit producer). - // Fails if not enough room to enqueue (or implicit production is - // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). - // Thread-safe. - inline bool try_enqueue(T&& item) - { - if (inner.try_enqueue(std::move(item))) { - sema->signal(); - return true; - } - return false; - } - - // Enqueues a single item (by copying it) using an explicit producer token. - // Does not allocate memory. Fails if not enough room to enqueue. - // Thread-safe. - inline bool try_enqueue(producer_token_t const& token, T const& item) - { - if (inner.try_enqueue(token, item)) { - sema->signal(); - return true; - } - return false; - } - - // Enqueues a single item (by moving it, if possible) using an explicit producer token. - // Does not allocate memory. Fails if not enough room to enqueue. - // Thread-safe. - inline bool try_enqueue(producer_token_t const& token, T&& item) - { - if (inner.try_enqueue(token, std::move(item))) { - sema->signal(); - return true; - } - return false; - } - - // Enqueues several items. - // Does not allocate memory (except for one-time implicit producer). - // Fails if not enough room to enqueue (or implicit production is - // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). - // Note: Use std::make_move_iterator if the elements should be moved - // instead of copied. - // Thread-safe. - template inline bool try_enqueue_bulk(It itemFirst, size_t count) - { - if (inner.try_enqueue_bulk(std::forward(itemFirst), count)) { - sema->signal((LightweightSemaphore::ssize_t)(ssize_t)count); - return true; - } - return false; - } - - // Enqueues several items using an explicit producer token. - // Does not allocate memory. Fails if not enough room to enqueue. - // Note: Use std::make_move_iterator if the elements should be moved - // instead of copied. - // Thread-safe. - template inline bool try_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) - { - if (inner.try_enqueue_bulk(token, std::forward(itemFirst), count)) { - sema->signal((LightweightSemaphore::ssize_t)(ssize_t)count); - return true; - } - return false; - } - - // Attempts to dequeue from the queue. - // Returns false if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template inline bool try_dequeue(U& item) - { - if (sema->tryWait()) { - while (!inner.try_dequeue(item)) { - continue; - } - return true; - } - return false; - } - - // Attempts to dequeue from the queue using an explicit consumer token. - // Returns false if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template inline bool try_dequeue(consumer_token_t& token, U& item) - { - if (sema->tryWait()) { - while (!inner.try_dequeue(token, item)) { - continue; - } - return true; - } - return false; - } - - // Attempts to dequeue several elements from the queue. - // Returns the number of items actually dequeued. - // Returns 0 if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template inline size_t try_dequeue_bulk(It itemFirst, size_t max) - { - size_t count = 0; - max = (size_t)sema->tryWaitMany((LightweightSemaphore::ssize_t)(ssize_t)max); - while (count != max) { - count += inner.template try_dequeue_bulk(itemFirst, max - count); - } - return count; - } - - // Attempts to dequeue several elements from the queue using an explicit consumer token. - // Returns the number of items actually dequeued. - // Returns 0 if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template inline size_t try_dequeue_bulk(consumer_token_t& token, It itemFirst, size_t max) - { - size_t count = 0; - max = (size_t)sema->tryWaitMany((LightweightSemaphore::ssize_t)(ssize_t)max); - while (count != max) { - count += inner.template try_dequeue_bulk(token, itemFirst, max - count); - } - return count; - } - - // Blocks the current thread until there's something to dequeue, then - // dequeues it. - // Never allocates. Thread-safe. - template inline void wait_dequeue(U& item) - { - while (!sema->wait()) { - continue; - } - while (!inner.try_dequeue(item)) { - continue; - } - } - - // Blocks the current thread until either there's something to dequeue - // or the timeout (specified in microseconds) expires. Returns false - // without setting `item` if the timeout expires, otherwise assigns - // to `item` and returns true. - // Using a negative timeout indicates an indefinite timeout, - // and is thus functionally equivalent to calling wait_dequeue. - // Never allocates. Thread-safe. - template inline bool wait_dequeue_timed(U& item, std::int64_t timeout_usecs) - { - if (!sema->wait(timeout_usecs)) { - return false; - } - while (!inner.try_dequeue(item)) { - continue; - } - return true; - } - - // Blocks the current thread until either there's something to dequeue - // or the timeout expires. Returns false without setting `item` if the - // timeout expires, otherwise assigns to `item` and returns true. - // Never allocates. Thread-safe. - template - inline bool wait_dequeue_timed(U& item, std::chrono::duration const& timeout) - { - return wait_dequeue_timed(item, std::chrono::duration_cast(timeout).count()); - } - - // Blocks the current thread until there's something to dequeue, then - // dequeues it using an explicit consumer token. - // Never allocates. Thread-safe. - template inline void wait_dequeue(consumer_token_t& token, U& item) - { - while (!sema->wait()) { - continue; - } - while (!inner.try_dequeue(token, item)) { - continue; - } - } - - // Blocks the current thread until either there's something to dequeue - // or the timeout (specified in microseconds) expires. Returns false - // without setting `item` if the timeout expires, otherwise assigns - // to `item` and returns true. - // Using a negative timeout indicates an indefinite timeout, - // and is thus functionally equivalent to calling wait_dequeue. - // Never allocates. Thread-safe. - template inline bool wait_dequeue_timed(consumer_token_t& token, U& item, std::int64_t timeout_usecs) - { - if (!sema->wait(timeout_usecs)) { - return false; - } - while (!inner.try_dequeue(token, item)) { - continue; - } - return true; - } - - // Blocks the current thread until either there's something to dequeue - // or the timeout expires. Returns false without setting `item` if the - // timeout expires, otherwise assigns to `item` and returns true. - // Never allocates. Thread-safe. - template - inline bool wait_dequeue_timed(consumer_token_t& token, U& item, std::chrono::duration const& timeout) - { - return wait_dequeue_timed(token, item, std::chrono::duration_cast(timeout).count()); - } - - // Attempts to dequeue several elements from the queue. - // Returns the number of items actually dequeued, which will - // always be at least one (this method blocks until the queue - // is non-empty) and at most max. - // Never allocates. Thread-safe. - template inline size_t wait_dequeue_bulk(It itemFirst, size_t max) - { - size_t count = 0; - max = (size_t)sema->waitMany((LightweightSemaphore::ssize_t)(ssize_t)max); - while (count != max) { - count += inner.template try_dequeue_bulk(itemFirst, max - count); - } - return count; - } - - // Attempts to dequeue several elements from the queue. - // Returns the number of items actually dequeued, which can - // be 0 if the timeout expires while waiting for elements, - // and at most max. - // Using a negative timeout indicates an indefinite timeout, - // and is thus functionally equivalent to calling wait_dequeue_bulk. - // Never allocates. Thread-safe. - template inline size_t wait_dequeue_bulk_timed(It itemFirst, size_t max, std::int64_t timeout_usecs) - { - size_t count = 0; - max = (size_t)sema->waitMany((LightweightSemaphore::ssize_t)(ssize_t)max, timeout_usecs); - while (count != max) { - count += inner.template try_dequeue_bulk(itemFirst, max - count); - } - return count; - } - - // Attempts to dequeue several elements from the queue. - // Returns the number of items actually dequeued, which can - // be 0 if the timeout expires while waiting for elements, - // and at most max. - // Never allocates. Thread-safe. - template - inline size_t wait_dequeue_bulk_timed(It itemFirst, size_t max, std::chrono::duration const& timeout) - { - return wait_dequeue_bulk_timed( - itemFirst, max, std::chrono::duration_cast(timeout).count()); - } - - // Attempts to dequeue several elements from the queue using an explicit consumer token. - // Returns the number of items actually dequeued, which will - // always be at least one (this method blocks until the queue - // is non-empty) and at most max. - // Never allocates. Thread-safe. - template inline size_t wait_dequeue_bulk(consumer_token_t& token, It itemFirst, size_t max) - { - size_t count = 0; - max = (size_t)sema->waitMany((LightweightSemaphore::ssize_t)(ssize_t)max); - while (count != max) { - count += inner.template try_dequeue_bulk(token, itemFirst, max - count); - } - return count; - } - - // Attempts to dequeue several elements from the queue using an explicit consumer token. - // Returns the number of items actually dequeued, which can - // be 0 if the timeout expires while waiting for elements, - // and at most max. - // Using a negative timeout indicates an indefinite timeout, - // and is thus functionally equivalent to calling wait_dequeue_bulk. - // Never allocates. Thread-safe. - template - inline size_t wait_dequeue_bulk_timed(consumer_token_t& token, It itemFirst, size_t max, std::int64_t timeout_usecs) - { - size_t count = 0; - max = (size_t)sema->waitMany((LightweightSemaphore::ssize_t)(ssize_t)max, timeout_usecs); - while (count != max) { - count += inner.template try_dequeue_bulk(token, itemFirst, max - count); - } - return count; - } - - // Attempts to dequeue several elements from the queue using an explicit consumer token. - // Returns the number of items actually dequeued, which can - // be 0 if the timeout expires while waiting for elements, - // and at most max. - // Never allocates. Thread-safe. - template - inline size_t wait_dequeue_bulk_timed(consumer_token_t& token, - It itemFirst, - size_t max, - std::chrono::duration const& timeout) - { - return wait_dequeue_bulk_timed( - token, itemFirst, max, std::chrono::duration_cast(timeout).count()); - } - - // Returns an estimate of the total number of elements currently in the queue. This - // estimate is only accurate if the queue has completely stabilized before it is called - // (i.e. all enqueue and dequeue operations have completed and their memory effects are - // visible on the calling thread, and no further operations start while this method is - // being called). - // Thread-safe. - inline size_t size_approx() const { return (size_t)sema->availableApprox(); } - - // Returns true if the underlying atomic variables used by - // the queue are lock-free (they should be on most platforms). - // Thread-safe. - static constexpr bool is_lock_free() { return ConcurrentQueue::is_lock_free(); } - - private: - template static inline U* create(A1&& a1, A2&& a2) - { - void* p = (Traits::malloc)(sizeof(U)); - return p != nullptr ? new (p) U(std::forward(a1), std::forward(a2)) : nullptr; - } - - template static inline void destroy(U* p) - { - if (p != nullptr) { - p->~U(); - } - (Traits::free)(p); - } - - private: - ConcurrentQueue inner; - std::unique_ptr sema; -}; - -template -inline void swap(BlockingConcurrentQueue& a, BlockingConcurrentQueue& b) MOODYCAMEL_NOEXCEPT -{ - a.swap(b); -} - -} // end namespace moodycamel \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/common/moody/concurrentqueue.h b/barretenberg/cpp/src/barretenberg/common/moody/concurrentqueue.h deleted file mode 100644 index fc23142efcf..00000000000 --- a/barretenberg/cpp/src/barretenberg/common/moody/concurrentqueue.h +++ /dev/null @@ -1,3988 +0,0 @@ -// Provides a C++11 implementation of a multi-producer, multi-consumer lock-free queue. -// An overview, including benchmark results, is provided here: -// http://moodycamel.com/blog/2014/a-fast-general-purpose-lock-free-queue-for-c++ -// The full design is also described in excruciating detail at: -// http://moodycamel.com/blog/2014/detailed-design-of-a-lock-free-queue - -// Simplified BSD license: -// Copyright (c) 2013-2020, Cameron Desrochers. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// - Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// - Redistributions in binary form must reproduce the above copyright notice, this list of -// conditions and the following disclaimer in the documentation and/or other materials -// provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT -// OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR -// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Also dual-licensed under the Boost Software License (see LICENSE.md) - -#pragma once - -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) -// Disable -Wconversion warnings (spuriously triggered when Traits::size_t and -// Traits::index_t are set to < 32 bits, causing integer promotion, causing warnings -// upon assigning any computed values) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wconversion" - -#ifdef MCDBGQ_USE_RELACY -#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" -#endif -#endif - -#if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17) -// VS2019 with /W4 warns about constant conditional expressions but unless /std=c++17 or higher -// does not support `if constexpr`, so we have no choice but to simply disable the warning -#pragma warning(push) -#pragma warning(disable : 4127) // conditional expression is constant -#endif - -#if defined(__APPLE__) -#include "TargetConditionals.h" -#endif - -#ifdef MCDBGQ_USE_RELACY -#include "relacy/relacy_std.hpp" -#include "relacy_shims.h" -// We only use malloc/free anyway, and the delete macro messes up `= delete` method declarations. -// We'll override the default trait malloc ourselves without a macro. -#undef new -#undef delete -#undef malloc -#undef free -#else -#include // Requires C++11. Sorry VS2010. -#include -#endif -#include // for max_align_t -#include -#include -#include -#include -#include -#include -#include // for CHAR_BIT -#include -#include // partly for __WINPTHREADS_VERSION if on MinGW-w64 w/ POSIX threading -#include // used for thread exit synchronization - -// Platform-specific definitions of a numeric thread ID type and an invalid value -namespace moodycamel { -namespace details { -template struct thread_id_converter { - typedef thread_id_t thread_id_numeric_size_t; - typedef thread_id_t thread_id_hash_t; - static thread_id_hash_t prehash(thread_id_t const& x) { return x; } -}; -} // namespace details -} // namespace moodycamel -#if defined(MCDBGQ_USE_RELACY) -namespace moodycamel { -namespace details { -typedef std::uint32_t thread_id_t; -static const thread_id_t invalid_thread_id = 0xFFFFFFFFU; -static const thread_id_t invalid_thread_id2 = 0xFFFFFFFEU; -static inline thread_id_t thread_id() -{ - return rl::thread_index(); -} -} // namespace details -} // namespace moodycamel -#elif defined(_WIN32) || defined(__WINDOWS__) || defined(__WIN32__) -// No sense pulling in windows.h in a header, we'll manually declare the function -// we use and rely on backwards-compatibility for this not to break -extern "C" __declspec(dllimport) unsigned long __stdcall GetCurrentThreadId(void); -namespace moodycamel { -namespace details { -static_assert(sizeof(unsigned long) == sizeof(std::uint32_t), - "Expected size of unsigned long to be 32 bits on Windows"); -typedef std::uint32_t thread_id_t; -static const thread_id_t invalid_thread_id = 0; // See http://blogs.msdn.com/b/oldnewthing/archive/2004/02/23/78395.aspx -static const thread_id_t invalid_thread_id2 = - 0xFFFFFFFFU; // Not technically guaranteed to be invalid, but is never used in practice. Note that all Win32 thread - // IDs are presently multiples of 4. -static inline thread_id_t thread_id() -{ - return static_cast(::GetCurrentThreadId()); -} -} // namespace details -} // namespace moodycamel -#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE) || \ - defined(MOODYCAMEL_NO_THREAD_LOCAL) -namespace moodycamel { -namespace details { -static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, - "std::thread::id is expected to be either 4 or 8 bytes"); - -typedef std::thread::id thread_id_t; -static const thread_id_t invalid_thread_id; // Default ctor creates invalid ID - -// Note we don't define a invalid_thread_id2 since std::thread::id doesn't have one; it's -// only used if MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is defined anyway, which it won't -// be. -static inline thread_id_t thread_id() -{ - return std::this_thread::get_id(); -} - -template struct thread_id_size {}; -template <> struct thread_id_size<4> { - typedef std::uint32_t numeric_t; -}; -template <> struct thread_id_size<8> { - typedef std::uint64_t numeric_t; -}; - -template <> struct thread_id_converter { - typedef thread_id_size::numeric_t thread_id_numeric_size_t; -#ifndef __APPLE__ - typedef std::size_t thread_id_hash_t; -#else - typedef thread_id_numeric_size_t thread_id_hash_t; -#endif - - static thread_id_hash_t prehash(thread_id_t const& x) - { -#ifndef __APPLE__ - return std::hash()(x); -#else - return *reinterpret_cast(&x); -#endif - } -}; -} -} -#else -// Use a nice trick from this answer: http://stackoverflow.com/a/8438730/21475 -// In order to get a numeric thread ID in a platform-independent way, we use a thread-local -// static variable's address as a thread identifier :-) -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#define MOODYCAMEL_THREADLOCAL __thread -#elif defined(_MSC_VER) -#define MOODYCAMEL_THREADLOCAL __declspec(thread) -#else -// Assume C++11 compliant compiler -#define MOODYCAMEL_THREADLOCAL thread_local -#endif -namespace moodycamel { -namespace details { -typedef std::uintptr_t thread_id_t; -static const thread_id_t invalid_thread_id = 0; // Address can't be nullptr -static const thread_id_t invalid_thread_id2 = - 1; // Member accesses off a null pointer are also generally invalid. Plus it's not aligned. -inline thread_id_t thread_id() -{ - static MOODYCAMEL_THREADLOCAL int x; - return reinterpret_cast(&x); -} -} -} -#endif - -// Constexpr if -#ifndef MOODYCAMEL_CONSTEXPR_IF -#if (defined(_MSC_VER) && defined(_HAS_CXX17) && _HAS_CXX17) || __cplusplus > 201402L -#define MOODYCAMEL_CONSTEXPR_IF if constexpr -#define MOODYCAMEL_MAYBE_UNUSED [[maybe_unused]] -#else -#define MOODYCAMEL_CONSTEXPR_IF if -#define MOODYCAMEL_MAYBE_UNUSED -#endif -#endif - -// Exceptions -#ifndef MOODYCAMEL_EXCEPTIONS_ENABLED -#if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || \ - (!defined(_MSC_VER) && !defined(__GNUC__)) -#define MOODYCAMEL_EXCEPTIONS_ENABLED -#endif -#endif -#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED -#define MOODYCAMEL_TRY try -#define MOODYCAMEL_CATCH(...) catch (__VA_ARGS__) -#define MOODYCAMEL_RETHROW throw -#define MOODYCAMEL_THROW(expr) throw(expr) -#else -#define MOODYCAMEL_TRY MOODYCAMEL_CONSTEXPR_IF(true) -#define MOODYCAMEL_CATCH(...) else MOODYCAMEL_CONSTEXPR_IF(false) -#define MOODYCAMEL_RETHROW -#define MOODYCAMEL_THROW(expr) -#endif - -#ifndef MOODYCAMEL_NOEXCEPT -#if !defined(MOODYCAMEL_EXCEPTIONS_ENABLED) -#define MOODYCAMEL_NOEXCEPT -#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) true -#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) true -#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1800 -// VS2012's std::is_nothrow_[move_]constructible is broken and returns true when it shouldn't :-( -// We have to assume *all* non-trivial constructors may throw on VS2012! -#define MOODYCAMEL_NOEXCEPT _NOEXCEPT -#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) \ - (std::is_rvalue_reference::value && std::is_move_constructible::value \ - ? std::is_trivially_move_constructible::value \ - : std::is_trivially_copy_constructible::value) -#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) \ - ((std::is_rvalue_reference::value && std::is_move_assignable::value \ - ? std::is_trivially_move_assignable::value || std::is_nothrow_move_assignable::value \ - : std::is_trivially_copy_assignable::value || std::is_nothrow_copy_assignable::value) && \ - MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)) -#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1900 -#define MOODYCAMEL_NOEXCEPT _NOEXCEPT -#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) \ - (std::is_rvalue_reference::value && std::is_move_constructible::value \ - ? std::is_trivially_move_constructible::value || std::is_nothrow_move_constructible::value \ - : std::is_trivially_copy_constructible::value || std::is_nothrow_copy_constructible::value) -#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) \ - ((std::is_rvalue_reference::value && std::is_move_assignable::value \ - ? std::is_trivially_move_assignable::value || std::is_nothrow_move_assignable::value \ - : std::is_trivially_copy_assignable::value || std::is_nothrow_copy_assignable::value) && \ - MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)) -#else -#define MOODYCAMEL_NOEXCEPT noexcept -#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) noexcept(expr) -#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) noexcept(expr) -#endif -#endif - -#ifndef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED -#ifdef MCDBGQ_USE_RELACY -#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED -#else -// VS2013 doesn't support `thread_local`, and MinGW-w64 w/ POSIX threading has a crippling bug: -// http://sourceforge.net/p/mingw-w64/bugs/445 g++ <=4.7 doesn't support thread_local either. Finally, iOS/ARM doesn't -// have support for it either, and g++/ARM allows it to compile but it's unconfirmed to actually work -#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && \ - (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && \ - (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && \ - (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) -// Assume `thread_local` is fully supported in all other C++11 compilers/platforms -#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED // tentatively enabled for now; years ago several users report having - // problems with it on -#endif -#endif -#endif - -// VS2012 doesn't support deleted functions. -// In this case, we declare the function normally but don't define it. A link error will be generated if the function is -// called. -#ifndef MOODYCAMEL_DELETE_FUNCTION -#if defined(_MSC_VER) && _MSC_VER < 1800 -#define MOODYCAMEL_DELETE_FUNCTION -#else -#define MOODYCAMEL_DELETE_FUNCTION = delete -#endif -#endif - -namespace moodycamel { -namespace details { -#ifndef MOODYCAMEL_ALIGNAS -// VS2013 doesn't support alignas or alignof, and align() requires a constant literal -#if defined(_MSC_VER) && _MSC_VER <= 1800 -#define MOODYCAMEL_ALIGNAS(alignment) __declspec(align(alignment)) -#define MOODYCAMEL_ALIGNOF(obj) __alignof(obj) -#define MOODYCAMEL_ALIGNED_TYPE_LIKE(T, obj) typename details::Vs2013Aligned::value, T>::type -template struct Vs2013Aligned {}; // default, unsupported alignment -template struct Vs2013Aligned<1, T> { - typedef __declspec(align(1)) T type; -}; -template struct Vs2013Aligned<2, T> { - typedef __declspec(align(2)) T type; -}; -template struct Vs2013Aligned<4, T> { - typedef __declspec(align(4)) T type; -}; -template struct Vs2013Aligned<8, T> { - typedef __declspec(align(8)) T type; -}; -template struct Vs2013Aligned<16, T> { - typedef __declspec(align(16)) T type; -}; -template struct Vs2013Aligned<32, T> { - typedef __declspec(align(32)) T type; -}; -template struct Vs2013Aligned<64, T> { - typedef __declspec(align(64)) T type; -}; -template struct Vs2013Aligned<128, T> { - typedef __declspec(align(128)) T type; -}; -template struct Vs2013Aligned<256, T> { - typedef __declspec(align(256)) T type; -}; -#else -template struct identity { - typedef T type; -}; -#define MOODYCAMEL_ALIGNAS(alignment) alignas(alignment) -#define MOODYCAMEL_ALIGNOF(obj) alignof(obj) -#define MOODYCAMEL_ALIGNED_TYPE_LIKE(T, obj) alignas(alignof(obj)) typename details::identity::type -#endif -#endif -} // namespace details -} // namespace moodycamel - -// TSAN can false report races in lock-free code. To enable TSAN to be used from projects that use this one, -// we can apply per-function compile-time suppression. -// See https://clang.llvm.org/docs/ThreadSanitizer.html#has-feature-thread-sanitizer -#define MOODYCAMEL_NO_TSAN -#if defined(__has_feature) -#if __has_feature(thread_sanitizer) -#undef MOODYCAMEL_NO_TSAN -#define MOODYCAMEL_NO_TSAN __attribute__((no_sanitize("thread"))) -#endif // TSAN -#endif // TSAN - -// Compiler-specific likely/unlikely hints -namespace moodycamel { -namespace details { -#if defined(__GNUC__) -static inline bool(likely)(bool x) -{ - return __builtin_expect((x), true); -} -static inline bool(unlikely)(bool x) -{ - return __builtin_expect((x), false); -} -#else -static inline bool(likely)(bool x) -{ - return x; -} -static inline bool(unlikely)(bool x) -{ - return x; -} -#endif -} // namespace details -} // namespace moodycamel - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG -#include "internal/concurrentqueue_internal_debug.h" -#endif - -namespace moodycamel { -namespace details { -template struct const_numeric_max { - static_assert(std::is_integral::value, "const_numeric_max can only be used with integers"); - static const T value = std::numeric_limits::is_signed - ? (static_cast(1) << (sizeof(T) * CHAR_BIT - 1)) - static_cast(1) - : static_cast(-1); -}; - -#if defined(__GLIBCXX__) -typedef ::max_align_t std_max_align_t; // libstdc++ forgot to add it to std:: for a while -#else -typedef std::max_align_t std_max_align_t; // Others (e.g. MSVC) insist it can *only* be accessed via std:: -#endif - -// Some platforms have incorrectly set max_align_t to a type with <8 bytes alignment even while supporting -// 8-byte aligned scalar values (*cough* 32-bit iOS). Work around this with our own union. See issue #64. -typedef union { - std_max_align_t x; - long long y; - void* z; -} max_align_t; -} // namespace details - -// Default traits for the ConcurrentQueue. To change some of the -// traits without re-implementing all of them, inherit from this -// struct and shadow the declarations you wish to be different; -// since the traits are used as a template type parameter, the -// shadowed declarations will be used where defined, and the defaults -// otherwise. -struct ConcurrentQueueDefaultTraits { - // General-purpose size type. std::size_t is strongly recommended. - typedef std::size_t size_t; - - // The type used for the enqueue and dequeue indices. Must be at least as - // large as size_t. Should be significantly larger than the number of elements - // you expect to hold at once, especially if you have a high turnover rate; - // for example, on 32-bit x86, if you expect to have over a hundred million - // elements or pump several million elements through your queue in a very - // short space of time, using a 32-bit type *may* trigger a race condition. - // A 64-bit int type is recommended in that case, and in practice will - // prevent a race condition no matter the usage of the queue. Note that - // whether the queue is lock-free with a 64-int type depends on the whether - // std::atomic is lock-free, which is platform-specific. - typedef std::size_t index_t; - - // Internally, all elements are enqueued and dequeued from multi-element - // blocks; this is the smallest controllable unit. If you expect few elements - // but many producers, a smaller block size should be favoured. For few producers - // and/or many elements, a larger block size is preferred. A sane default - // is provided. Must be a power of 2. - static const size_t BLOCK_SIZE = 32; - - // For explicit producers (i.e. when using a producer token), the block is - // checked for being empty by iterating through a list of flags, one per element. - // For large block sizes, this is too inefficient, and switching to an atomic - // counter-based approach is faster. The switch is made for block sizes strictly - // larger than this threshold. - static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = 32; - - // How many full blocks can be expected for a single explicit producer? This should - // reflect that number's maximum for optimal performance. Must be a power of 2. - static const size_t EXPLICIT_INITIAL_INDEX_SIZE = 32; - - // How many full blocks can be expected for a single implicit producer? This should - // reflect that number's maximum for optimal performance. Must be a power of 2. - static const size_t IMPLICIT_INITIAL_INDEX_SIZE = 32; - - // The initial size of the hash table mapping thread IDs to implicit producers. - // Note that the hash is resized every time it becomes half full. - // Must be a power of two, and either 0 or at least 1. If 0, implicit production - // (using the enqueue methods without an explicit producer token) is disabled. - static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = 32; - - // Controls the number of items that an explicit consumer (i.e. one with a token) - // must consume before it causes all consumers to rotate and move on to the next - // internal queue. - static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = 256; - - // The maximum number of elements (inclusive) that can be enqueued to a sub-queue. - // Enqueue operations that would cause this limit to be surpassed will fail. Note - // that this limit is enforced at the block level (for performance reasons), i.e. - // it's rounded up to the nearest block size. - static const size_t MAX_SUBQUEUE_SIZE = details::const_numeric_max::value; - - // The number of times to spin before sleeping when waiting on a semaphore. - // Recommended values are on the order of 1000-10000 unless the number of - // consumer threads exceeds the number of idle cores (in which case try 0-100). - // Only affects instances of the BlockingConcurrentQueue. - static const int MAX_SEMA_SPINS = 10000; - - // Whether to recycle dynamically-allocated blocks into an internal free list or - // not. If false, only pre-allocated blocks (controlled by the constructor - // arguments) will be recycled, and all others will be `free`d back to the heap. - // Note that blocks consumed by explicit producers are only freed on destruction - // of the queue (not following destruction of the token) regardless of this trait. - static const bool RECYCLE_ALLOCATED_BLOCKS = false; - -#ifndef MCDBGQ_USE_RELACY - // Memory allocation can be customized if needed. - // malloc should return nullptr on failure, and handle alignment like std::malloc. -#if defined(malloc) || defined(free) - // Gah, this is 2015, stop defining macros that break standard code already! - // Work around malloc/free being special macros: - static inline void* WORKAROUND_malloc(size_t size) { return malloc(size); } - static inline void WORKAROUND_free(void* ptr) { return free(ptr); } - static inline void*(malloc)(size_t size) { return WORKAROUND_malloc(size); } - static inline void(free)(void* ptr) { return WORKAROUND_free(ptr); } -#else - static inline void* malloc(size_t size) { return std::malloc(size); } - static inline void free(void* ptr) { return std::free(ptr); } -#endif -#else - // Debug versions when running under the Relacy race detector (ignore - // these in user code) - static inline void* malloc(size_t size) { return rl::rl_malloc(size, $); } - static inline void free(void* ptr) { return rl::rl_free(ptr, $); } -#endif -}; - -// When producing or consuming many elements, the most efficient way is to: -// 1) Use one of the bulk-operation methods of the queue with a token -// 2) Failing that, use the bulk-operation methods without a token -// 3) Failing that, create a token and use that with the single-item methods -// 4) Failing that, use the single-parameter methods of the queue -// Having said that, don't create tokens willy-nilly -- ideally there should be -// a maximum of one token per thread (of each kind). -struct ProducerToken; -struct ConsumerToken; - -template class ConcurrentQueue; -template class BlockingConcurrentQueue; -class ConcurrentQueueTests; - -namespace details { -struct ConcurrentQueueProducerTypelessBase { - ConcurrentQueueProducerTypelessBase* next; - std::atomic inactive; - ProducerToken* token; - - ConcurrentQueueProducerTypelessBase() - : next(nullptr) - , inactive(false) - , token(nullptr) - {} -}; - -template struct _hash_32_or_64 { - static inline std::uint32_t hash(std::uint32_t h) - { - // MurmurHash3 finalizer -- see https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp - // Since the thread ID is already unique, all we really want to do is propagate that - // uniqueness evenly across all the bits, so that we can use a subset of the bits while - // reducing collisions significantly - h ^= h >> 16; - h *= 0x85ebca6b; - h ^= h >> 13; - h *= 0xc2b2ae35; - return h ^ (h >> 16); - } -}; -template <> struct _hash_32_or_64<1> { - static inline std::uint64_t hash(std::uint64_t h) - { - h ^= h >> 33; - h *= 0xff51afd7ed558ccd; - h ^= h >> 33; - h *= 0xc4ceb9fe1a85ec53; - return h ^ (h >> 33); - } -}; -template struct hash_32_or_64 : public _hash_32_or_64<(size > 4)> {}; - -static inline size_t hash_thread_id(thread_id_t id) -{ - static_assert(sizeof(thread_id_t) <= 8, "Expected a platform where thread IDs are at most 64-bit values"); - return static_cast(hash_32_or_64::thread_id_hash_t)>::hash( - thread_id_converter::prehash(id))); -} - -template static inline bool circular_less_than(T a, T b) -{ - static_assert(std::is_integral::value && !std::numeric_limits::is_signed, - "circular_less_than is intended to be used only with unsigned integer types"); - return static_cast(a - b) > static_cast(static_cast(1) << (static_cast(sizeof(T) * CHAR_BIT - 1))); - // Note: extra parens around rhs of operator<< is MSVC bug: - // https://developercommunity2.visualstudio.com/t/C4554-triggers-when-both-lhs-and-rhs-is/10034931 - // silencing the bug requires #pragma warning(disable: 4554) around the calling code and has no effect when - // done here. -} - -template static inline char* align_for(char* ptr) -{ - const std::size_t alignment = std::alignment_of::value; - return ptr + (alignment - (reinterpret_cast(ptr) % alignment)) % alignment; -} - -template static inline T ceil_to_pow_2(T x) -{ - static_assert(std::is_integral::value && !std::numeric_limits::is_signed, - "ceil_to_pow_2 is intended to be used only with unsigned integer types"); - - // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 - --x; - x |= x >> 1; - x |= x >> 2; - x |= x >> 4; - for (std::size_t i = 1; i < sizeof(T); i <<= 1) { - x |= x >> (i << 3); - } - ++x; - return x; -} - -template static inline void swap_relaxed(std::atomic& left, std::atomic& right) -{ - T temp = std::move(left.load(std::memory_order_relaxed)); - left.store(std::move(right.load(std::memory_order_relaxed)), std::memory_order_relaxed); - right.store(std::move(temp), std::memory_order_relaxed); -} - -template static inline T const& nomove(T const& x) -{ - return x; -} - -template struct nomove_if { - template static inline T const& eval(T const& x) { return x; } -}; - -template <> struct nomove_if { - template static inline auto eval(U&& x) -> decltype(std::forward(x)) { return std::forward(x); } -}; - -template static inline auto deref_noexcept(It& it) MOODYCAMEL_NOEXCEPT->decltype(*it) -{ - return *it; -} - -#if defined(__clang__) || !defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) -template struct is_trivially_destructible : std::is_trivially_destructible {}; -#else -template struct is_trivially_destructible : std::has_trivial_destructor {}; -#endif - -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED -#ifdef MCDBGQ_USE_RELACY -typedef RelacyThreadExitListener ThreadExitListener; -typedef RelacyThreadExitNotifier ThreadExitNotifier; -#else -class ThreadExitNotifier; - -struct ThreadExitListener { - typedef void (*callback_t)(void*); - callback_t callback; - void* userData; - - ThreadExitListener* next; // reserved for use by the ThreadExitNotifier - ThreadExitNotifier* chain; // reserved for use by the ThreadExitNotifier -}; - -class ThreadExitNotifier { - public: - static void subscribe(ThreadExitListener* listener) - { - auto& tlsInst = instance(); - std::lock_guard guard(mutex()); - listener->next = tlsInst.tail; - listener->chain = &tlsInst; - tlsInst.tail = listener; - } - - static void unsubscribe(ThreadExitListener* listener) - { - std::lock_guard guard(mutex()); - if (!listener->chain) { - return; // race with ~ThreadExitNotifier - } - auto& tlsInst = *listener->chain; - listener->chain = nullptr; - ThreadExitListener** prev = &tlsInst.tail; - for (auto ptr = tlsInst.tail; ptr != nullptr; ptr = ptr->next) { - if (ptr == listener) { - *prev = ptr->next; - break; - } - prev = &ptr->next; - } - } - - private: - ThreadExitNotifier() - : tail(nullptr) - {} - ThreadExitNotifier(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; - ThreadExitNotifier& operator=(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; - - ~ThreadExitNotifier() - { - // This thread is about to exit, let everyone know! - assert(this == &instance() && - "If this assert fails, you likely have a buggy compiler! Change the preprocessor conditions such that " - "MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is no longer defined."); - std::lock_guard guard(mutex()); - for (auto ptr = tail; ptr != nullptr; ptr = ptr->next) { - ptr->chain = nullptr; - ptr->callback(ptr->userData); - } - } - - // Thread-local - static inline ThreadExitNotifier& instance() - { - static thread_local ThreadExitNotifier notifier; - return notifier; - } - - static inline std::mutex& mutex() - { - // Must be static because the ThreadExitNotifier could be destroyed while unsubscribe is called - static std::mutex mutex; - return mutex; - } - - private: - ThreadExitListener* tail; -}; -#endif -#endif - -template struct static_is_lock_free_num { - enum { value = 0 }; -}; -template <> struct static_is_lock_free_num { - enum { value = ATOMIC_CHAR_LOCK_FREE }; -}; -template <> struct static_is_lock_free_num { - enum { value = ATOMIC_SHORT_LOCK_FREE }; -}; -template <> struct static_is_lock_free_num { - enum { value = ATOMIC_INT_LOCK_FREE }; -}; -template <> struct static_is_lock_free_num { - enum { value = ATOMIC_LONG_LOCK_FREE }; -}; -template <> struct static_is_lock_free_num { - enum { value = ATOMIC_LLONG_LOCK_FREE }; -}; -template struct static_is_lock_free : static_is_lock_free_num::type> {}; -template <> struct static_is_lock_free { - enum { value = ATOMIC_BOOL_LOCK_FREE }; -}; -template struct static_is_lock_free { - enum { value = ATOMIC_POINTER_LOCK_FREE }; -}; -} // namespace details - -struct ProducerToken { - template explicit ProducerToken(ConcurrentQueue& queue); - - template explicit ProducerToken(BlockingConcurrentQueue& queue); - - ProducerToken(ProducerToken&& other) MOODYCAMEL_NOEXCEPT : producer(other.producer) - { - other.producer = nullptr; - if (producer != nullptr) { - producer->token = this; - } - } - - inline ProducerToken& operator=(ProducerToken&& other) MOODYCAMEL_NOEXCEPT - { - swap(other); - return *this; - } - - void swap(ProducerToken& other) MOODYCAMEL_NOEXCEPT - { - std::swap(producer, other.producer); - if (producer != nullptr) { - producer->token = this; - } - if (other.producer != nullptr) { - other.producer->token = &other; - } - } - - // A token is always valid unless: - // 1) Memory allocation failed during construction - // 2) It was moved via the move constructor - // (Note: assignment does a swap, leaving both potentially valid) - // 3) The associated queue was destroyed - // Note that if valid() returns true, that only indicates - // that the token is valid for use with a specific queue, - // but not which one; that's up to the user to track. - inline bool valid() const { return producer != nullptr; } - - ~ProducerToken() - { - if (producer != nullptr) { - producer->token = nullptr; - producer->inactive.store(true, std::memory_order_release); - } - } - - // Disable copying and assignment - ProducerToken(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; - ProducerToken& operator=(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; - - private: - template friend class ConcurrentQueue; - friend class ConcurrentQueueTests; - - protected: - details::ConcurrentQueueProducerTypelessBase* producer; -}; - -struct ConsumerToken { - template explicit ConsumerToken(ConcurrentQueue& q); - - template explicit ConsumerToken(BlockingConcurrentQueue& q); - - ConsumerToken(ConsumerToken&& other) MOODYCAMEL_NOEXCEPT : initialOffset(other.initialOffset), - lastKnownGlobalOffset(other.lastKnownGlobalOffset), - itemsConsumedFromCurrent(other.itemsConsumedFromCurrent), - currentProducer(other.currentProducer), - desiredProducer(other.desiredProducer) - {} - - inline ConsumerToken& operator=(ConsumerToken&& other) MOODYCAMEL_NOEXCEPT - { - swap(other); - return *this; - } - - void swap(ConsumerToken& other) MOODYCAMEL_NOEXCEPT - { - std::swap(initialOffset, other.initialOffset); - std::swap(lastKnownGlobalOffset, other.lastKnownGlobalOffset); - std::swap(itemsConsumedFromCurrent, other.itemsConsumedFromCurrent); - std::swap(currentProducer, other.currentProducer); - std::swap(desiredProducer, other.desiredProducer); - } - - // Disable copying and assignment - ConsumerToken(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; - ConsumerToken& operator=(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; - - private: - template friend class ConcurrentQueue; - friend class ConcurrentQueueTests; - - private: // but shared with ConcurrentQueue - std::uint32_t initialOffset; - std::uint32_t lastKnownGlobalOffset; - std::uint32_t itemsConsumedFromCurrent; - details::ConcurrentQueueProducerTypelessBase* currentProducer; - details::ConcurrentQueueProducerTypelessBase* desiredProducer; -}; - -// Need to forward-declare this swap because it's in a namespace. -// See -// http://stackoverflow.com/questions/4492062/why-does-a-c-friend-class-need-a-forward-declaration-only-in-other-namespaces -template -inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, - typename ConcurrentQueue::ImplicitProducerKVP& b) MOODYCAMEL_NOEXCEPT; - -template class ConcurrentQueue { - public: - typedef ::moodycamel::ProducerToken producer_token_t; - typedef ::moodycamel::ConsumerToken consumer_token_t; - - typedef typename Traits::index_t index_t; - typedef typename Traits::size_t size_t; - - static const size_t BLOCK_SIZE = static_cast(Traits::BLOCK_SIZE); - static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = - static_cast(Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD); - static const size_t EXPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::EXPLICIT_INITIAL_INDEX_SIZE); - static const size_t IMPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::IMPLICIT_INITIAL_INDEX_SIZE); - static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = - static_cast(Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE); - static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = - static_cast(Traits::EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE); -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4307) // + integral constant overflow (that's what the ternary expression is for!) -#pragma warning(disable : 4309) // static_cast: Truncation of constant value -#endif - static const size_t MAX_SUBQUEUE_SIZE = - (details::const_numeric_max::value - static_cast(Traits::MAX_SUBQUEUE_SIZE) < BLOCK_SIZE) - ? details::const_numeric_max::value - : ((static_cast(Traits::MAX_SUBQUEUE_SIZE) + (BLOCK_SIZE - 1)) / BLOCK_SIZE * BLOCK_SIZE); -#ifdef _MSC_VER -#pragma warning(pop) -#endif - - static_assert(!std::numeric_limits::is_signed && std::is_integral::value, - "Traits::size_t must be an unsigned integral type"); - static_assert(!std::numeric_limits::is_signed && std::is_integral::value, - "Traits::index_t must be an unsigned integral type"); - static_assert(sizeof(index_t) >= sizeof(size_t), "Traits::index_t must be at least as wide as Traits::size_t"); - static_assert((BLOCK_SIZE > 1) && !(BLOCK_SIZE & (BLOCK_SIZE - 1)), - "Traits::BLOCK_SIZE must be a power of 2 (and at least 2)"); - static_assert((EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD > 1) && - !(EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD & (EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD - 1)), - "Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD must be a power of 2 (and greater than 1)"); - static_assert((EXPLICIT_INITIAL_INDEX_SIZE > 1) && - !(EXPLICIT_INITIAL_INDEX_SIZE & (EXPLICIT_INITIAL_INDEX_SIZE - 1)), - "Traits::EXPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); - static_assert((IMPLICIT_INITIAL_INDEX_SIZE > 1) && - !(IMPLICIT_INITIAL_INDEX_SIZE & (IMPLICIT_INITIAL_INDEX_SIZE - 1)), - "Traits::IMPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); - static_assert((INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) || - !(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE & (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - 1)), - "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be a power of 2"); - static_assert( - INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0 || INITIAL_IMPLICIT_PRODUCER_HASH_SIZE >= 1, - "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be at least 1 (or 0 to disable implicit enqueueing)"); - - public: - // Creates a queue with at least `capacity` element slots; note that the - // actual number of elements that can be inserted without additional memory - // allocation depends on the number of producers and the block size (e.g. if - // the block size is equal to `capacity`, only a single block will be allocated - // up-front, which means only a single producer will be able to enqueue elements - // without an extra allocation -- blocks aren't shared between producers). - // This method is not thread safe -- it is up to the user to ensure that the - // queue is fully constructed before it starts being used by other threads (this - // includes making the memory effects of construction visible, possibly with a - // memory barrier). - explicit ConcurrentQueue(size_t capacity = 32 * BLOCK_SIZE) - : producerListTail(nullptr) - , producerCount(0) - , initialBlockPoolIndex(0) - , nextExplicitConsumerId(0) - , globalExplicitConsumerOffset(0) - { - implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); - populate_initial_implicit_producer_hash(); - populate_initial_block_list(capacity / BLOCK_SIZE + ((capacity & (BLOCK_SIZE - 1)) == 0 ? 0 : 1)); - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - // Track all the producers using a fully-resolved typed list for - // each kind; this makes it possible to debug them starting from - // the root queue object (otherwise wacky casts are needed that - // don't compile in the debugger's expression evaluator). - explicitProducers.store(nullptr, std::memory_order_relaxed); - implicitProducers.store(nullptr, std::memory_order_relaxed); -#endif - } - - // Computes the correct amount of pre-allocated blocks for you based - // on the minimum number of elements you want available at any given - // time, and the maximum concurrent number of each type of producer. - ConcurrentQueue(size_t minCapacity, size_t maxExplicitProducers, size_t maxImplicitProducers) - : producerListTail(nullptr) - , producerCount(0) - , initialBlockPoolIndex(0) - , nextExplicitConsumerId(0) - , globalExplicitConsumerOffset(0) - { - implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); - populate_initial_implicit_producer_hash(); - size_t blocks = (((minCapacity + BLOCK_SIZE - 1) / BLOCK_SIZE) - 1) * (maxExplicitProducers + 1) + - 2 * (maxExplicitProducers + maxImplicitProducers); - populate_initial_block_list(blocks); - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - explicitProducers.store(nullptr, std::memory_order_relaxed); - implicitProducers.store(nullptr, std::memory_order_relaxed); -#endif - } - - // Note: The queue should not be accessed concurrently while it's - // being deleted. It's up to the user to synchronize this. - // This method is not thread safe. - ~ConcurrentQueue() - { - // Destroy producers - auto ptr = producerListTail.load(std::memory_order_relaxed); - while (ptr != nullptr) { - auto next = ptr->next_prod(); - if (ptr->token != nullptr) { - ptr->token->producer = nullptr; - } - destroy(ptr); - ptr = next; - } - - // Destroy implicit producer hash tables - MOODYCAMEL_CONSTEXPR_IF(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE != 0) - { - auto hash = implicitProducerHash.load(std::memory_order_relaxed); - while (hash != nullptr) { - auto prev = hash->prev; - if (prev != nullptr) { // The last hash is part of this object and was not allocated dynamically - for (size_t i = 0; i != hash->capacity; ++i) { - hash->entries[i].~ImplicitProducerKVP(); - } - hash->~ImplicitProducerHash(); - (Traits::free)(hash); - } - hash = prev; - } - } - - // Destroy global free list - auto block = freeList.head_unsafe(); - while (block != nullptr) { - auto next = block->freeListNext.load(std::memory_order_relaxed); - if (block->dynamicallyAllocated) { - destroy(block); - } - block = next; - } - - // Destroy initial free list - destroy_array(initialBlockPool, initialBlockPoolSize); - } - - // Disable copying and copy assignment - ConcurrentQueue(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; - ConcurrentQueue& operator=(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; - - // Moving is supported, but note that it is *not* a thread-safe operation. - // Nobody can use the queue while it's being moved, and the memory effects - // of that move must be propagated to other threads before they can use it. - // Note: When a queue is moved, its tokens are still valid but can only be - // used with the destination queue (i.e. semantically they are moved along - // with the queue itself). - ConcurrentQueue(ConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT - : producerListTail(other.producerListTail.load(std::memory_order_relaxed)), - producerCount(other.producerCount.load(std::memory_order_relaxed)), - initialBlockPoolIndex(other.initialBlockPoolIndex.load(std::memory_order_relaxed)), - initialBlockPool(other.initialBlockPool), - initialBlockPoolSize(other.initialBlockPoolSize), - freeList(std::move(other.freeList)), - nextExplicitConsumerId(other.nextExplicitConsumerId.load(std::memory_order_relaxed)), - globalExplicitConsumerOffset(other.globalExplicitConsumerOffset.load(std::memory_order_relaxed)) - { - // Move the other one into this, and leave the other one as an empty queue - implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); - populate_initial_implicit_producer_hash(); - swap_implicit_producer_hashes(other); - - other.producerListTail.store(nullptr, std::memory_order_relaxed); - other.producerCount.store(0, std::memory_order_relaxed); - other.nextExplicitConsumerId.store(0, std::memory_order_relaxed); - other.globalExplicitConsumerOffset.store(0, std::memory_order_relaxed); - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - explicitProducers.store(other.explicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed); - other.explicitProducers.store(nullptr, std::memory_order_relaxed); - implicitProducers.store(other.implicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed); - other.implicitProducers.store(nullptr, std::memory_order_relaxed); -#endif - - other.initialBlockPoolIndex.store(0, std::memory_order_relaxed); - other.initialBlockPoolSize = 0; - other.initialBlockPool = nullptr; - - reown_producers(); - } - - inline ConcurrentQueue& operator=(ConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT { return swap_internal(other); } - - // Swaps this queue's state with the other's. Not thread-safe. - // Swapping two queues does not invalidate their tokens, however - // the tokens that were created for one queue must be used with - // only the swapped queue (i.e. the tokens are tied to the - // queue's movable state, not the object itself). - inline void swap(ConcurrentQueue& other) MOODYCAMEL_NOEXCEPT { swap_internal(other); } - - private: - ConcurrentQueue& swap_internal(ConcurrentQueue& other) - { - if (this == &other) { - return *this; - } - - details::swap_relaxed(producerListTail, other.producerListTail); - details::swap_relaxed(producerCount, other.producerCount); - details::swap_relaxed(initialBlockPoolIndex, other.initialBlockPoolIndex); - std::swap(initialBlockPool, other.initialBlockPool); - std::swap(initialBlockPoolSize, other.initialBlockPoolSize); - freeList.swap(other.freeList); - details::swap_relaxed(nextExplicitConsumerId, other.nextExplicitConsumerId); - details::swap_relaxed(globalExplicitConsumerOffset, other.globalExplicitConsumerOffset); - - swap_implicit_producer_hashes(other); - - reown_producers(); - other.reown_producers(); - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - details::swap_relaxed(explicitProducers, other.explicitProducers); - details::swap_relaxed(implicitProducers, other.implicitProducers); -#endif - - return *this; - } - - public: - // Enqueues a single item (by copying it). - // Allocates memory if required. Only fails if memory allocation fails (or implicit - // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, - // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(T const& item) - { - MOODYCAMEL_CONSTEXPR_IF(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue(item); - } - - // Enqueues a single item (by moving it, if possible). - // Allocates memory if required. Only fails if memory allocation fails (or implicit - // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, - // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(T&& item) - { - MOODYCAMEL_CONSTEXPR_IF(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue(std::move(item)); - } - - // Enqueues a single item (by copying it) using an explicit producer token. - // Allocates memory if required. Only fails if memory allocation fails (or - // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(producer_token_t const& token, T const& item) { return inner_enqueue(token, item); } - - // Enqueues a single item (by moving it, if possible) using an explicit producer token. - // Allocates memory if required. Only fails if memory allocation fails (or - // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Thread-safe. - inline bool enqueue(producer_token_t const& token, T&& item) - { - return inner_enqueue(token, std::move(item)); - } - - // Enqueues several items. - // Allocates memory if required. Only fails if memory allocation fails (or - // implicit production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - // is 0, or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Note: Use std::make_move_iterator if the elements should be moved instead of copied. - // Thread-safe. - template bool enqueue_bulk(It itemFirst, size_t count) - { - MOODYCAMEL_CONSTEXPR_IF(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue_bulk(itemFirst, count); - } - - // Enqueues several items using an explicit producer token. - // Allocates memory if required. Only fails if memory allocation fails - // (or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). - // Note: Use std::make_move_iterator if the elements should be moved - // instead of copied. - // Thread-safe. - template bool enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) - { - return inner_enqueue_bulk(token, itemFirst, count); - } - - // Enqueues a single item (by copying it). - // Does not allocate memory. Fails if not enough room to enqueue (or implicit - // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - // is 0). - // Thread-safe. - inline bool try_enqueue(T const& item) - { - MOODYCAMEL_CONSTEXPR_IF(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue(item); - } - - // Enqueues a single item (by moving it, if possible). - // Does not allocate memory (except for one-time implicit producer). - // Fails if not enough room to enqueue (or implicit production is - // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). - // Thread-safe. - inline bool try_enqueue(T&& item) - { - MOODYCAMEL_CONSTEXPR_IF(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue(std::move(item)); - } - - // Enqueues a single item (by copying it) using an explicit producer token. - // Does not allocate memory. Fails if not enough room to enqueue. - // Thread-safe. - inline bool try_enqueue(producer_token_t const& token, T const& item) - { - return inner_enqueue(token, item); - } - - // Enqueues a single item (by moving it, if possible) using an explicit producer token. - // Does not allocate memory. Fails if not enough room to enqueue. - // Thread-safe. - inline bool try_enqueue(producer_token_t const& token, T&& item) - { - return inner_enqueue(token, std::move(item)); - } - - // Enqueues several items. - // Does not allocate memory (except for one-time implicit producer). - // Fails if not enough room to enqueue (or implicit production is - // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). - // Note: Use std::make_move_iterator if the elements should be moved - // instead of copied. - // Thread-safe. - template bool try_enqueue_bulk(It itemFirst, size_t count) - { - MOODYCAMEL_CONSTEXPR_IF(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; - else return inner_enqueue_bulk(itemFirst, count); - } - - // Enqueues several items using an explicit producer token. - // Does not allocate memory. Fails if not enough room to enqueue. - // Note: Use std::make_move_iterator if the elements should be moved - // instead of copied. - // Thread-safe. - template bool try_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) - { - return inner_enqueue_bulk(token, itemFirst, count); - } - - // Attempts to dequeue from the queue. - // Returns false if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template bool try_dequeue(U& item) - { - // Instead of simply trying each producer in turn (which could cause needless contention on the first - // producer), we score them heuristically. - size_t nonEmptyCount = 0; - ProducerBase* best = nullptr; - size_t bestSize = 0; - for (auto ptr = producerListTail.load(std::memory_order_acquire); nonEmptyCount < 3 && ptr != nullptr; - ptr = ptr->next_prod()) { - auto size = ptr->size_approx(); - if (size > 0) { - if (size > bestSize) { - bestSize = size; - best = ptr; - } - ++nonEmptyCount; - } - } - - // If there was at least one non-empty queue but it appears empty at the time - // we try to dequeue from it, we need to make sure every queue's been tried - if (nonEmptyCount > 0) { - if ((details::likely)(best->dequeue(item))) { - return true; - } - for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - if (ptr != best && ptr->dequeue(item)) { - return true; - } - } - } - return false; - } - - // Attempts to dequeue from the queue. - // Returns false if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // This differs from the try_dequeue(item) method in that this one does - // not attempt to reduce contention by interleaving the order that producer - // streams are dequeued from. So, using this method can reduce overall throughput - // under contention, but will give more predictable results in single-threaded - // consumer scenarios. This is mostly only useful for internal unit tests. - // Never allocates. Thread-safe. - template bool try_dequeue_non_interleaved(U& item) - { - for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - if (ptr->dequeue(item)) { - return true; - } - } - return false; - } - - // Attempts to dequeue from the queue using an explicit consumer token. - // Returns false if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template bool try_dequeue(consumer_token_t& token, U& item) - { - // The idea is roughly as follows: - // Every 256 items from one producer, make everyone rotate (increase the global offset) -> this means the - // highest efficiency consumer dictates the rotation speed of everyone else, more or less If you see that the - // global offset has changed, you must reset your consumption counter and move to your designated place If - // there's no items where you're supposed to be, keep moving until you find a producer with some items If the - // global offset has not changed but you've run out of items to consume, move over from your current position - // until you find an producer with something in it - - if (token.desiredProducer == nullptr || - token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) { - if (!update_current_producer_after_rotation(token)) { - return false; - } - } - - // If there was at least one non-empty queue but it appears empty at the time - // we try to dequeue from it, we need to make sure every queue's been tried - if (static_cast(token.currentProducer)->dequeue(item)) { - if (++token.itemsConsumedFromCurrent == EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) { - globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed); - } - return true; - } - - auto tail = producerListTail.load(std::memory_order_acquire); - auto ptr = static_cast(token.currentProducer)->next_prod(); - if (ptr == nullptr) { - ptr = tail; - } - while (ptr != static_cast(token.currentProducer)) { - if (ptr->dequeue(item)) { - token.currentProducer = ptr; - token.itemsConsumedFromCurrent = 1; - return true; - } - ptr = ptr->next_prod(); - if (ptr == nullptr) { - ptr = tail; - } - } - return false; - } - - // Attempts to dequeue several elements from the queue. - // Returns the number of items actually dequeued. - // Returns 0 if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template size_t try_dequeue_bulk(It itemFirst, size_t max) - { - size_t count = 0; - for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - count += ptr->dequeue_bulk(itemFirst, max - count); - if (count == max) { - break; - } - } - return count; - } - - // Attempts to dequeue several elements from the queue using an explicit consumer token. - // Returns the number of items actually dequeued. - // Returns 0 if all producer streams appeared empty at the time they - // were checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template size_t try_dequeue_bulk(consumer_token_t& token, It itemFirst, size_t max) - { - if (token.desiredProducer == nullptr || - token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) { - if (!update_current_producer_after_rotation(token)) { - return 0; - } - } - - size_t count = static_cast(token.currentProducer)->dequeue_bulk(itemFirst, max); - if (count == max) { - if ((token.itemsConsumedFromCurrent += static_cast(max)) >= - EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) { - globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed); - } - return max; - } - token.itemsConsumedFromCurrent += static_cast(count); - max -= count; - - auto tail = producerListTail.load(std::memory_order_acquire); - auto ptr = static_cast(token.currentProducer)->next_prod(); - if (ptr == nullptr) { - ptr = tail; - } - while (ptr != static_cast(token.currentProducer)) { - auto dequeued = ptr->dequeue_bulk(itemFirst, max); - count += dequeued; - if (dequeued != 0) { - token.currentProducer = ptr; - token.itemsConsumedFromCurrent = static_cast(dequeued); - } - if (dequeued == max) { - break; - } - max -= dequeued; - ptr = ptr->next_prod(); - if (ptr == nullptr) { - ptr = tail; - } - } - return count; - } - - // Attempts to dequeue from a specific producer's inner queue. - // If you happen to know which producer you want to dequeue from, this - // is significantly faster than using the general-case try_dequeue methods. - // Returns false if the producer's queue appeared empty at the time it - // was checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template inline bool try_dequeue_from_producer(producer_token_t const& producer, U& item) - { - return static_cast(producer.producer)->dequeue(item); - } - - // Attempts to dequeue several elements from a specific producer's inner queue. - // Returns the number of items actually dequeued. - // If you happen to know which producer you want to dequeue from, this - // is significantly faster than using the general-case try_dequeue methods. - // Returns 0 if the producer's queue appeared empty at the time it - // was checked (so, the queue is likely but not guaranteed to be empty). - // Never allocates. Thread-safe. - template - inline size_t try_dequeue_bulk_from_producer(producer_token_t const& producer, It itemFirst, size_t max) - { - return static_cast(producer.producer)->dequeue_bulk(itemFirst, max); - } - - // Returns an estimate of the total number of elements currently in the queue. This - // estimate is only accurate if the queue has completely stabilized before it is called - // (i.e. all enqueue and dequeue operations have completed and their memory effects are - // visible on the calling thread, and no further operations start while this method is - // being called). - // Thread-safe. - size_t size_approx() const - { - size_t size = 0; - for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - size += ptr->size_approx(); - } - return size; - } - - // Returns true if the underlying atomic variables used by - // the queue are lock-free (they should be on most platforms). - // Thread-safe. - static constexpr bool is_lock_free() - { - return details::static_is_lock_free::value == 2 && details::static_is_lock_free::value == 2 && - details::static_is_lock_free::value == 2 && - details::static_is_lock_free::value == 2 && details::static_is_lock_free::value == 2 && - details::static_is_lock_free< - typename details::thread_id_converter::thread_id_numeric_size_t>::value == 2; - } - - private: - friend struct ProducerToken; - friend struct ConsumerToken; - struct ExplicitProducer; - friend struct ExplicitProducer; - struct ImplicitProducer; - friend struct ImplicitProducer; - friend class ConcurrentQueueTests; - - enum AllocationMode { CanAlloc, CannotAlloc }; - - /////////////////////////////// - // Queue methods - /////////////////////////////// - - template inline bool inner_enqueue(producer_token_t const& token, U&& element) - { - return static_cast(token.producer) - ->ConcurrentQueue::ExplicitProducer::template enqueue(std::forward(element)); - } - - template inline bool inner_enqueue(U&& element) - { - auto producer = get_or_add_implicit_producer(); - return producer == nullptr - ? false - : producer->ConcurrentQueue::ImplicitProducer::template enqueue(std::forward(element)); - } - - template - inline bool inner_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) - { - return static_cast(token.producer) - ->ConcurrentQueue::ExplicitProducer::template enqueue_bulk(itemFirst, count); - } - - template inline bool inner_enqueue_bulk(It itemFirst, size_t count) - { - auto producer = get_or_add_implicit_producer(); - return producer == nullptr - ? false - : producer->ConcurrentQueue::ImplicitProducer::template enqueue_bulk(itemFirst, count); - } - - inline bool update_current_producer_after_rotation(consumer_token_t& token) - { - // Ah, there's been a rotation, figure out where we should be! - auto tail = producerListTail.load(std::memory_order_acquire); - if (token.desiredProducer == nullptr && tail == nullptr) { - return false; - } - auto prodCount = producerCount.load(std::memory_order_relaxed); - auto globalOffset = globalExplicitConsumerOffset.load(std::memory_order_relaxed); - if ((details::unlikely)(token.desiredProducer == nullptr)) { - // Aha, first time we're dequeueing anything. - // Figure out our local position - // Note: offset is from start, not end, but we're traversing from end -- subtract from count first - std::uint32_t offset = prodCount - 1 - (token.initialOffset % prodCount); - token.desiredProducer = tail; - for (std::uint32_t i = 0; i != offset; ++i) { - token.desiredProducer = static_cast(token.desiredProducer)->next_prod(); - if (token.desiredProducer == nullptr) { - token.desiredProducer = tail; - } - } - } - - std::uint32_t delta = globalOffset - token.lastKnownGlobalOffset; - if (delta >= prodCount) { - delta = delta % prodCount; - } - for (std::uint32_t i = 0; i != delta; ++i) { - token.desiredProducer = static_cast(token.desiredProducer)->next_prod(); - if (token.desiredProducer == nullptr) { - token.desiredProducer = tail; - } - } - - token.lastKnownGlobalOffset = globalOffset; - token.currentProducer = token.desiredProducer; - token.itemsConsumedFromCurrent = 0; - return true; - } - - /////////////////////////// - // Free list - /////////////////////////// - - template struct FreeListNode { - FreeListNode() - : freeListRefs(0) - , freeListNext(nullptr) - {} - - std::atomic freeListRefs; - std::atomic freeListNext; - }; - - // A simple CAS-based lock-free free list. Not the fastest thing in the world under heavy contention, but - // simple and correct (assuming nodes are never freed until after the free list is destroyed), and fairly - // speedy under low contention. - template // N must inherit FreeListNode or have the same fields (and initialization of them) - struct FreeList { - FreeList() - : freeListHead(nullptr) - {} - FreeList(FreeList&& other) - : freeListHead(other.freeListHead.load(std::memory_order_relaxed)) - { - other.freeListHead.store(nullptr, std::memory_order_relaxed); - } - void swap(FreeList& other) { details::swap_relaxed(freeListHead, other.freeListHead); } - - FreeList(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; - FreeList& operator=(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; - - inline void add(N* node) - { -#ifdef MCDBGQ_NOLOCKFREE_FREELIST - debug::DebugLock lock(mutex); -#endif - // We know that the should-be-on-freelist bit is 0 at this point, so it's safe to - // set it using a fetch_add - if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST, std::memory_order_acq_rel) == 0) { - // Oh look! We were the last ones referencing this node, and we know - // we want to add it to the free list, so let's do it! - add_knowing_refcount_is_zero(node); - } - } - - inline N* try_get() - { -#ifdef MCDBGQ_NOLOCKFREE_FREELIST - debug::DebugLock lock(mutex); -#endif - auto head = freeListHead.load(std::memory_order_acquire); - while (head != nullptr) { - auto prevHead = head; - auto refs = head->freeListRefs.load(std::memory_order_relaxed); - if ((refs & REFS_MASK) == 0 || - !head->freeListRefs.compare_exchange_strong( - refs, refs + 1, std::memory_order_acquire, std::memory_order_relaxed)) { - head = freeListHead.load(std::memory_order_acquire); - continue; - } - - // Good, reference count has been incremented (it wasn't at zero), which means we can read the - // next and not worry about it changing between now and the time we do the CAS - auto next = head->freeListNext.load(std::memory_order_relaxed); - if (freeListHead.compare_exchange_strong( - head, next, std::memory_order_acquire, std::memory_order_relaxed)) { - // Yay, got the node. This means it was on the list, which means shouldBeOnFreeList must be false no - // matter the refcount (because nobody else knows it's been taken off yet, it can't have been put - // back on). - assert((head->freeListRefs.load(std::memory_order_relaxed) & SHOULD_BE_ON_FREELIST) == 0); - - // Decrease refcount twice, once for our ref, and once for the list's ref - head->freeListRefs.fetch_sub(2, std::memory_order_release); - return head; - } - - // OK, the head must have changed on us, but we still need to decrease the refcount we increased. - // Note that we don't need to release any memory effects, but we do need to ensure that the reference - // count decrement happens-after the CAS on the head. - refs = prevHead->freeListRefs.fetch_sub(1, std::memory_order_acq_rel); - if (refs == SHOULD_BE_ON_FREELIST + 1) { - add_knowing_refcount_is_zero(prevHead); - } - } - - return nullptr; - } - - // Useful for traversing the list when there's no contention (e.g. to destroy remaining nodes) - N* head_unsafe() const { return freeListHead.load(std::memory_order_relaxed); } - - private: - inline void add_knowing_refcount_is_zero(N* node) - { - // Since the refcount is zero, and nobody can increase it once it's zero (except us, and we run - // only one copy of this method per node at a time, i.e. the single thread case), then we know - // we can safely change the next pointer of the node; however, once the refcount is back above - // zero, then other threads could increase it (happens under heavy contention, when the refcount - // goes to zero in between a load and a refcount increment of a node in try_get, then back up to - // something non-zero, then the refcount increment is done by the other thread) -- so, if the CAS - // to add the node to the actual list fails, decrease the refcount and leave the add operation to - // the next thread who puts the refcount back at zero (which could be us, hence the loop). - auto head = freeListHead.load(std::memory_order_relaxed); - while (true) { - node->freeListNext.store(head, std::memory_order_relaxed); - node->freeListRefs.store(1, std::memory_order_release); - if (!freeListHead.compare_exchange_strong( - head, node, std::memory_order_release, std::memory_order_relaxed)) { - // Hmm, the add failed, but we can only try again when the refcount goes back to zero - if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST - 1, std::memory_order_release) == 1) { - continue; - } - } - return; - } - } - - private: - // Implemented like a stack, but where node order doesn't matter (nodes are inserted out of order under - // contention) - std::atomic freeListHead; - - static const std::uint32_t REFS_MASK = 0x7FFFFFFF; - static const std::uint32_t SHOULD_BE_ON_FREELIST = 0x80000000; - -#ifdef MCDBGQ_NOLOCKFREE_FREELIST - debug::DebugMutex mutex; -#endif - }; - - /////////////////////////// - // Block - /////////////////////////// - - enum InnerQueueContext { implicit_context = 0, explicit_context = 1 }; - - struct Block { - Block() - : next(nullptr) - , elementsCompletelyDequeued(0) - , freeListRefs(0) - , freeListNext(nullptr) - , dynamicallyAllocated(true) - { -#ifdef MCDBGQ_TRACKMEM - owner = nullptr; -#endif - } - - template inline bool is_empty() const - { - MOODYCAMEL_CONSTEXPR_IF(context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) - { - // Check flags - for (size_t i = 0; i < BLOCK_SIZE; ++i) { - if (!emptyFlags[i].load(std::memory_order_relaxed)) { - return false; - } - } - - // Aha, empty; make sure we have all other memory effects that happened before the empty flags were set - std::atomic_thread_fence(std::memory_order_acquire); - return true; - } - else - { - // Check counter - if (elementsCompletelyDequeued.load(std::memory_order_relaxed) == BLOCK_SIZE) { - std::atomic_thread_fence(std::memory_order_acquire); - return true; - } - assert(elementsCompletelyDequeued.load(std::memory_order_relaxed) <= BLOCK_SIZE); - return false; - } - } - - // Returns true if the block is now empty (does not apply in explicit context) - template inline bool set_empty(MOODYCAMEL_MAYBE_UNUSED index_t i) - { - MOODYCAMEL_CONSTEXPR_IF(context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) - { - // Set flag - assert(!emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].load( - std::memory_order_relaxed)); - emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].store( - true, std::memory_order_release); - return false; - } - else - { - // Increment counter - auto prevVal = elementsCompletelyDequeued.fetch_add(1, std::memory_order_release); - assert(prevVal < BLOCK_SIZE); - return prevVal == BLOCK_SIZE - 1; - } - } - - // Sets multiple contiguous item statuses to 'empty' (assumes no wrapping and count > 0). - // Returns true if the block is now empty (does not apply in explicit context). - template inline bool set_many_empty(MOODYCAMEL_MAYBE_UNUSED index_t i, size_t count) - { - MOODYCAMEL_CONSTEXPR_IF(context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) - { - // Set flags - std::atomic_thread_fence(std::memory_order_release); - i = BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1)) - count + 1; - for (size_t j = 0; j != count; ++j) { - assert(!emptyFlags[i + j].load(std::memory_order_relaxed)); - emptyFlags[i + j].store(true, std::memory_order_relaxed); - } - return false; - } - else - { - // Increment counter - auto prevVal = elementsCompletelyDequeued.fetch_add(count, std::memory_order_release); - assert(prevVal + count <= BLOCK_SIZE); - return prevVal + count == BLOCK_SIZE; - } - } - - template inline void set_all_empty() - { - MOODYCAMEL_CONSTEXPR_IF(context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) - { - // Set all flags - for (size_t i = 0; i != BLOCK_SIZE; ++i) { - emptyFlags[i].store(true, std::memory_order_relaxed); - } - } - else - { - // Reset counter - elementsCompletelyDequeued.store(BLOCK_SIZE, std::memory_order_relaxed); - } - } - - template inline void reset_empty() - { - MOODYCAMEL_CONSTEXPR_IF(context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) - { - // Reset flags - for (size_t i = 0; i != BLOCK_SIZE; ++i) { - emptyFlags[i].store(false, std::memory_order_relaxed); - } - } - else - { - // Reset counter - elementsCompletelyDequeued.store(0, std::memory_order_relaxed); - } - } - - inline T* operator[](index_t idx) MOODYCAMEL_NOEXCEPT - { - return static_cast(static_cast(elements)) + - static_cast(idx & static_cast(BLOCK_SIZE - 1)); - } - inline T const* operator[](index_t idx) const MOODYCAMEL_NOEXCEPT - { - return static_cast(static_cast(elements)) + - static_cast(idx & static_cast(BLOCK_SIZE - 1)); - } - - private: - static_assert(std::alignment_of::value <= sizeof(T), - "The queue does not support types with an alignment greater than their size at this time"); - MOODYCAMEL_ALIGNED_TYPE_LIKE(char[sizeof(T) * BLOCK_SIZE], T) elements; - - public: - Block* next; - std::atomic elementsCompletelyDequeued; - std::atomic emptyFlags[BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD ? BLOCK_SIZE : 1]; - - public: - std::atomic freeListRefs; - std::atomic freeListNext; - bool dynamicallyAllocated; // Perhaps a better name for this would be 'isNotPartOfInitialBlockPool' - -#ifdef MCDBGQ_TRACKMEM - void* owner; -#endif - }; - static_assert(std::alignment_of::value >= std::alignment_of::value, - "Internal error: Blocks must be at least as aligned as the type they are wrapping"); - -#ifdef MCDBGQ_TRACKMEM - public: - struct MemStats; - - private: -#endif - - /////////////////////////// - // Producer base - /////////////////////////// - - struct ProducerBase : public details::ConcurrentQueueProducerTypelessBase { - ProducerBase(ConcurrentQueue* parent_, bool isExplicit_) - : tailIndex(0) - , headIndex(0) - , dequeueOptimisticCount(0) - , dequeueOvercommit(0) - , tailBlock(nullptr) - , isExplicit(isExplicit_) - , parent(parent_) - {} - - virtual ~ProducerBase() {} - - template inline bool dequeue(U& element) - { - if (isExplicit) { - return static_cast(this)->dequeue(element); - } else { - return static_cast(this)->dequeue(element); - } - } - - template inline size_t dequeue_bulk(It& itemFirst, size_t max) - { - if (isExplicit) { - return static_cast(this)->dequeue_bulk(itemFirst, max); - } else { - return static_cast(this)->dequeue_bulk(itemFirst, max); - } - } - - inline ProducerBase* next_prod() const { return static_cast(next); } - - inline size_t size_approx() const - { - auto tail = tailIndex.load(std::memory_order_relaxed); - auto head = headIndex.load(std::memory_order_relaxed); - return details::circular_less_than(head, tail) ? static_cast(tail - head) : 0; - } - - inline index_t getTail() const { return tailIndex.load(std::memory_order_relaxed); } - - protected: - std::atomic tailIndex; // Where to enqueue to next - std::atomic headIndex; // Where to dequeue from next - - std::atomic dequeueOptimisticCount; - std::atomic dequeueOvercommit; - - Block* tailBlock; - - public: - bool isExplicit; - ConcurrentQueue* parent; - - protected: -#ifdef MCDBGQ_TRACKMEM - friend struct MemStats; -#endif - }; - - /////////////////////////// - // Explicit queue - /////////////////////////// - - struct ExplicitProducer : public ProducerBase { - explicit ExplicitProducer(ConcurrentQueue* parent_) - : ProducerBase(parent_, true) - , blockIndex(nullptr) - , pr_blockIndexSlotsUsed(0) - , pr_blockIndexSize(EXPLICIT_INITIAL_INDEX_SIZE >> 1) - , pr_blockIndexFront(0) - , pr_blockIndexEntries(nullptr) - , pr_blockIndexRaw(nullptr) - { - size_t poolBasedIndexSize = details::ceil_to_pow_2(parent_->initialBlockPoolSize) >> 1; - if (poolBasedIndexSize > pr_blockIndexSize) { - pr_blockIndexSize = poolBasedIndexSize; - } - - new_block_index( - 0); // This creates an index with double the number of current entries, i.e. EXPLICIT_INITIAL_INDEX_SIZE - } - - ~ExplicitProducer() - { - // Destruct any elements not yet dequeued. - // Since we're in the destructor, we can assume all elements - // are either completely dequeued or completely not (no halfways). - if (this->tailBlock != nullptr) { // Note this means there must be a block index too - // First find the block that's partially dequeued, if any - Block* halfDequeuedBlock = nullptr; - if ((this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) != 0) { - // The head's not on a block boundary, meaning a block somewhere is partially dequeued - // (or the head block is the tail block and was fully dequeued, but the head/tail are still not on a - // boundary) - size_t i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & (pr_blockIndexSize - 1); - while (details::circular_less_than(pr_blockIndexEntries[i].base + BLOCK_SIZE, - this->headIndex.load(std::memory_order_relaxed))) { - i = (i + 1) & (pr_blockIndexSize - 1); - } - assert(details::circular_less_than(pr_blockIndexEntries[i].base, - this->headIndex.load(std::memory_order_relaxed))); - halfDequeuedBlock = pr_blockIndexEntries[i].block; - } - - // Start at the head block (note the first line in the loop gives us the head from the tail on the first - // iteration) - auto block = this->tailBlock; - do { - block = block->next; - if (block->ConcurrentQueue::Block::template is_empty()) { - continue; - } - - size_t i = 0; // Offset into block - if (block == halfDequeuedBlock) { - i = static_cast(this->headIndex.load(std::memory_order_relaxed) & - static_cast(BLOCK_SIZE - 1)); - } - - // Walk through all the items in the block; if this is the tail block, we need to stop when we reach - // the tail index - auto lastValidIndex = - (this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) == 0 - ? BLOCK_SIZE - : static_cast(this->tailIndex.load(std::memory_order_relaxed) & - static_cast(BLOCK_SIZE - 1)); - while (i != BLOCK_SIZE && (block != this->tailBlock || i != lastValidIndex)) { - (*block)[i++]->~T(); - } - } while (block != this->tailBlock); - } - - // Destroy all blocks that we own - if (this->tailBlock != nullptr) { - auto block = this->tailBlock; - do { - auto nextBlock = block->next; - this->parent->add_block_to_free_list(block); - block = nextBlock; - } while (block != this->tailBlock); - } - - // Destroy the block indices - auto header = static_cast(pr_blockIndexRaw); - while (header != nullptr) { - auto prev = static_cast(header->prev); - header->~BlockIndexHeader(); - (Traits::free)(header); - header = prev; - } - } - - template inline bool enqueue(U&& element) - { - index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); - index_t newTailIndex = 1 + currentTailIndex; - if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { - // We reached the end of a block, start a new one - auto startBlock = this->tailBlock; - auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; - if (this->tailBlock != nullptr && - this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { - // We can re-use the block ahead of us, it's empty! - this->tailBlock = this->tailBlock->next; - this->tailBlock->ConcurrentQueue::Block::template reset_empty(); - - // We'll put the block on the block index (guaranteed to be room since we're conceptually removing - // the last block from it first -- except instead of removing then adding, we can just overwrite). - // Note that there must be a valid block index here, since even if allocation failed in the ctor, - // it would have been re-attempted when adding the first block to the queue; since there is such - // a block, a block index must have been successfully allocated. - } else { - // Whatever head value we see here is >= the last value we saw here (relatively), - // and <= its current value. Since we have the most recent tail, the head must be - // <= to it. - auto head = this->headIndex.load(std::memory_order_relaxed); - assert(!details::circular_less_than(currentTailIndex, head)); - if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || - (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && - (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { - // We can't enqueue in another block because there's not enough leeway -- the - // tail could surpass the head by the time the block fills up! (Or we'll exceed - // the size limit, if the second part of the condition was true.) - return false; - } - // We're going to need a new block; check that the block index has room - if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize) { - // Hmm, the circular block index is already full -- we'll need - // to allocate a new index. Note pr_blockIndexRaw can only be nullptr if - // the initial allocation failed in the constructor. - - MOODYCAMEL_CONSTEXPR_IF(allocMode == CannotAlloc) - { - return false; - } - else if (!new_block_index(pr_blockIndexSlotsUsed)) - { - return false; - } - } - - // Insert a new block in the circular linked list - auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); - if (newBlock == nullptr) { - return false; - } -#ifdef MCDBGQ_TRACKMEM - newBlock->owner = this; -#endif - newBlock->ConcurrentQueue::Block::template reset_empty(); - if (this->tailBlock == nullptr) { - newBlock->next = newBlock; - } else { - newBlock->next = this->tailBlock->next; - this->tailBlock->next = newBlock; - } - this->tailBlock = newBlock; - ++pr_blockIndexSlotsUsed; - } - - MOODYCAMEL_CONSTEXPR_IF( - !MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) - { - // The constructor may throw. We want the element not to appear in the queue in - // that case (without corrupting the queue): - MOODYCAMEL_TRY - { - new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); - } - MOODYCAMEL_CATCH(...) - { - // Revert change to the current block, but leave the new block available - // for next time - pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; - this->tailBlock = startBlock == nullptr ? this->tailBlock : startBlock; - MOODYCAMEL_RETHROW; - } - } - else - { - (void)startBlock; - (void)originalBlockIndexSlotsUsed; - } - - // Add block to block index - auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; - entry.base = currentTailIndex; - entry.block = this->tailBlock; - blockIndex.load(std::memory_order_relaxed)->front.store(pr_blockIndexFront, std::memory_order_release); - pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); - - MOODYCAMEL_CONSTEXPR_IF( - !MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) - { - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } - } - - // Enqueue - new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); - - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } - - template bool dequeue(U& element) - { - auto tail = this->tailIndex.load(std::memory_order_relaxed); - auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); - if (details::circular_less_than( - this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) { - // Might be something to dequeue, let's give it a try - - // Note that this if is purely for performance purposes in the common case when the queue is - // empty and the values are eventually consistent -- we may enter here spuriously. - - // Note that whatever the values of overcommit and tail are, they are not going to change (unless we - // change them) and must be the same value at this point (inside the if) as when the if condition was - // evaluated. - - // We insert an acquire fence here to synchronize-with the release upon incrementing dequeueOvercommit - // below. This ensures that whatever the value we got loaded into overcommit, the load of - // dequeueOptisticCount in the fetch_add below will result in a value at least as recent as that (and - // therefore at least as large). Note that I believe a compiler (signal) fence here would be sufficient - // due to the nature of fetch_add (all read-modify-write operations are guaranteed to work on the latest - // value in the modification order), but unfortunately that can't be shown to be correct using only the - // C++11 standard. See - // http://stackoverflow.com/questions/18223161/what-are-the-c11-memory-ordering-guarantees-in-this-corner-case - std::atomic_thread_fence(std::memory_order_acquire); - - // Increment optimistic counter, then check if it went over the boundary - auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed); - - // Note that since dequeueOvercommit must be <= dequeueOptimisticCount (because dequeueOvercommit is - // only ever incremented after dequeueOptimisticCount -- this is enforced in the `else` block below), - // and since we now have a version of dequeueOptimisticCount that is at least as recent as overcommit - // (due to the release upon incrementing dequeueOvercommit and the acquire above that synchronizes with - // it), overcommit <= myDequeueCount. However, we can't assert this since both dequeueOptimisticCount - // and dequeueOvercommit may (independently) overflow; in such a case, though, the logic still holds - // since the difference between the two is maintained. - - // Note that we reload tail here in case it changed; it will be the same value as before or greater, - // since this load is sequenced after (happens after) the earlier load above. This is supported by - // read-read coherency (as defined in the standard), explained here: - // http://en.cppreference.com/w/cpp/atomic/memory_order - tail = this->tailIndex.load(std::memory_order_acquire); - if ((details::likely)(details::circular_less_than(myDequeueCount - overcommit, tail))) { - // Guaranteed to be at least one element to dequeue! - - // Get the index. Note that since there's guaranteed to be at least one element, this - // will never exceed tail. We need to do an acquire-release fence here since it's possible - // that whatever condition got us to this point was for an earlier enqueued element (that - // we already see the memory effects for), but that by the time we increment somebody else - // has incremented it, and we need to see the memory effects for *that* element, which is - // in such a case is necessarily visible on the thread that incremented it in the first - // place with the more current condition (they must have acquired a tail that is at least - // as recent). - auto index = this->headIndex.fetch_add(1, std::memory_order_acq_rel); - - // Determine which block the element is in - - auto localBlockIndex = blockIndex.load(std::memory_order_acquire); - auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); - - // We need to be careful here about subtracting and dividing because of index wrap-around. - // When an index wraps, we need to preserve the sign of the offset when dividing it by the - // block size (in order to get a correct signed block count offset in all cases): - auto headBase = localBlockIndex->entries[localBlockIndexHead].base; - auto blockBaseIndex = index & ~static_cast(BLOCK_SIZE - 1); - auto offset = static_cast( - static_cast::type>(blockBaseIndex - headBase) / - static_cast::type>(BLOCK_SIZE)); - auto block = - localBlockIndex->entries[(localBlockIndexHead + offset) & (localBlockIndex->size - 1)].block; - - // Dequeue - auto& el = *((*block)[index]); - if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, element = std::move(el))) { - // Make sure the element is still fully dequeued and destroyed even if the assignment - // throws - struct Guard { - Block* block; - index_t index; - - ~Guard() - { - (*block)[index]->~T(); - block->ConcurrentQueue::Block::template set_empty(index); - } - } guard = { block, index }; - - element = std::move(el); // NOLINT - } else { - element = std::move(el); // NOLINT - el.~T(); // NOLINT - block->ConcurrentQueue::Block::template set_empty(index); - } - - return true; - } else { - // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent - this->dequeueOvercommit.fetch_add( - 1, std::memory_order_release); // Release so that the fetch_add on dequeueOptimisticCount is - // guaranteed to happen before this write - } - } - - return false; - } - - template - bool MOODYCAMEL_NO_TSAN enqueue_bulk(It itemFirst, size_t count) - { - // First, we need to make sure we have enough room to enqueue all of the elements; - // this means pre-allocating blocks and putting them in the block index (but only if - // all the allocations succeeded). - index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed); - auto startBlock = this->tailBlock; - auto originalBlockIndexFront = pr_blockIndexFront; - auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; - - Block* firstAllocatedBlock = nullptr; - - // Figure out how many blocks we'll need to allocate, and do so - size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); - index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); - if (blockBaseDiff > 0) { - // Allocate as many blocks as possible from ahead - while (blockBaseDiff > 0 && this->tailBlock != nullptr && - this->tailBlock->next != firstAllocatedBlock && - this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { - blockBaseDiff -= static_cast(BLOCK_SIZE); - currentTailIndex += static_cast(BLOCK_SIZE); - - this->tailBlock = this->tailBlock->next; - firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; - - auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; - entry.base = currentTailIndex; - entry.block = this->tailBlock; - pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); - } - - // Now allocate as many blocks as necessary from the block pool - while (blockBaseDiff > 0) { - blockBaseDiff -= static_cast(BLOCK_SIZE); - currentTailIndex += static_cast(BLOCK_SIZE); - - auto head = this->headIndex.load(std::memory_order_relaxed); - assert(!details::circular_less_than(currentTailIndex, head)); - bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || - (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && - (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); - if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize || full) { - MOODYCAMEL_CONSTEXPR_IF(allocMode == CannotAlloc) - { - // Failed to allocate, undo changes (but keep injected blocks) - pr_blockIndexFront = originalBlockIndexFront; - pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; - this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; - return false; - } - else if (full || !new_block_index(originalBlockIndexSlotsUsed)) - { - // Failed to allocate, undo changes (but keep injected blocks) - pr_blockIndexFront = originalBlockIndexFront; - pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; - this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; - return false; - } - - // pr_blockIndexFront is updated inside new_block_index, so we need to - // update our fallback value too (since we keep the new index even if we - // later fail) - originalBlockIndexFront = originalBlockIndexSlotsUsed; - } - - // Insert a new block in the circular linked list - auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); - if (newBlock == nullptr) { - pr_blockIndexFront = originalBlockIndexFront; - pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; - this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; - return false; - } - -#ifdef MCDBGQ_TRACKMEM - newBlock->owner = this; -#endif - newBlock->ConcurrentQueue::Block::template set_all_empty(); - if (this->tailBlock == nullptr) { - newBlock->next = newBlock; - } else { - newBlock->next = this->tailBlock->next; - this->tailBlock->next = newBlock; - } - this->tailBlock = newBlock; - firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; - - ++pr_blockIndexSlotsUsed; - - auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; - entry.base = currentTailIndex; - entry.block = this->tailBlock; - pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); - } - - // Excellent, all allocations succeeded. Reset each block's emptiness before we fill them up, and - // publish the new block index front - auto block = firstAllocatedBlock; - while (true) { - block->ConcurrentQueue::Block::template reset_empty(); - if (block == this->tailBlock) { - break; - } - block = block->next; - } - - MOODYCAMEL_CONSTEXPR_IF(MOODYCAMEL_NOEXCEPT_CTOR( - T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) - { - blockIndex.load(std::memory_order_relaxed) - ->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release); - } - } - - // Enqueue, one block at a time - index_t newTailIndex = startTailIndex + static_cast(count); - currentTailIndex = startTailIndex; - auto endBlock = this->tailBlock; - this->tailBlock = startBlock; - assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || - count == 0); - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { - this->tailBlock = firstAllocatedBlock; - } - while (true) { - index_t stopIndex = - (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - if (details::circular_less_than(newTailIndex, stopIndex)) { - stopIndex = newTailIndex; - } - MOODYCAMEL_CONSTEXPR_IF(MOODYCAMEL_NOEXCEPT_CTOR( - T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) - { - while (currentTailIndex != stopIndex) { - new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++); - } - } - else - { - MOODYCAMEL_TRY - { - while (currentTailIndex != stopIndex) { - // Must use copy constructor even if move constructor is available - // because we may have to revert if there's an exception. - // Sorry about the horrible templated next line, but it was the only way - // to disable moving *at compile time*, which is important because a type - // may only define a (noexcept) move constructor, and so calls to the - // cctor will not compile, even if they are in an if branch that will never - // be executed - new ((*this->tailBlock)[currentTailIndex]) - T(details::nomove_if(nullptr)) - T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst)); - ++currentTailIndex; - ++itemFirst; - } - } - MOODYCAMEL_CATCH(...) - { - // Oh dear, an exception's been thrown -- destroy the elements that - // were enqueued so far and revert the entire bulk operation (we'll keep - // any allocated blocks in our linked list for later, though). - auto constructedStopIndex = currentTailIndex; - auto lastBlockEnqueued = this->tailBlock; - - pr_blockIndexFront = originalBlockIndexFront; - pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; - this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; - - if (!details::is_trivially_destructible::value) { - auto block = startBlock; - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { - block = firstAllocatedBlock; - } - currentTailIndex = startTailIndex; - while (true) { - stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + - static_cast(BLOCK_SIZE); - if (details::circular_less_than(constructedStopIndex, stopIndex)) { - stopIndex = constructedStopIndex; - } - while (currentTailIndex != stopIndex) { - (*block)[currentTailIndex++]->~T(); - } - if (block == lastBlockEnqueued) { - break; - } - block = block->next; - } - } - MOODYCAMEL_RETHROW; - } - } - - if (this->tailBlock == endBlock) { - assert(currentTailIndex == newTailIndex); - break; - } - this->tailBlock = this->tailBlock->next; - } - - MOODYCAMEL_CONSTEXPR_IF(!MOODYCAMEL_NOEXCEPT_CTOR( - T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) - { - if (firstAllocatedBlock != nullptr) - blockIndex.load(std::memory_order_relaxed) - ->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release); - } - - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } - - template size_t dequeue_bulk(It& itemFirst, size_t max) - { - auto tail = this->tailIndex.load(std::memory_order_relaxed); - auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); - auto desiredCount = - static_cast(tail - (this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit)); - if (details::circular_less_than(0, desiredCount)) { - desiredCount = desiredCount < max ? desiredCount : max; - std::atomic_thread_fence(std::memory_order_acquire); - - auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed); - - tail = this->tailIndex.load(std::memory_order_acquire); - auto actualCount = static_cast(tail - (myDequeueCount - overcommit)); - if (details::circular_less_than(0, actualCount)) { - actualCount = desiredCount < actualCount ? desiredCount : actualCount; - if (actualCount < desiredCount) { - this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release); - } - - // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this - // will never exceed tail. - auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel); - - // Determine which block the first element is in - auto localBlockIndex = blockIndex.load(std::memory_order_acquire); - auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); - - auto headBase = localBlockIndex->entries[localBlockIndexHead].base; - auto firstBlockBaseIndex = firstIndex & ~static_cast(BLOCK_SIZE - 1); - auto offset = static_cast( - static_cast::type>(firstBlockBaseIndex - headBase) / - static_cast::type>(BLOCK_SIZE)); - auto indexIndex = (localBlockIndexHead + offset) & (localBlockIndex->size - 1); - - // Iterate the blocks and dequeue - auto index = firstIndex; - do { - auto firstIndexInBlock = index; - index_t endIndex = - (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), - endIndex) - ? firstIndex + static_cast(actualCount) - : endIndex; - auto block = localBlockIndex->entries[indexIndex].block; - if (MOODYCAMEL_NOEXCEPT_ASSIGN( - T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { - while (index != endIndex) { - auto& el = *((*block)[index]); - *itemFirst++ = std::move(el); - el.~T(); - ++index; - } - } else { - MOODYCAMEL_TRY - { - while (index != endIndex) { - auto& el = *((*block)[index]); - *itemFirst = std::move(el); - ++itemFirst; - el.~T(); - ++index; - } - } - MOODYCAMEL_CATCH(...) - { - // It's too late to revert the dequeue, but we can make sure that all - // the dequeued objects are properly destroyed and the block index - // (and empty count) are properly updated before we propagate the exception - do { - block = localBlockIndex->entries[indexIndex].block; - while (index != endIndex) { - (*block)[index++]->~T(); - } - block->ConcurrentQueue::Block::template set_many_empty( - firstIndexInBlock, static_cast(endIndex - firstIndexInBlock)); - indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); - - firstIndexInBlock = index; - endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + - static_cast(BLOCK_SIZE); - endIndex = details::circular_less_than( - firstIndex + static_cast(actualCount), endIndex) - ? firstIndex + static_cast(actualCount) - : endIndex; - } while (index != firstIndex + actualCount); - - MOODYCAMEL_RETHROW; - } - } - block->ConcurrentQueue::Block::template set_many_empty( - firstIndexInBlock, static_cast(endIndex - firstIndexInBlock)); - indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); - } while (index != firstIndex + actualCount); - - return actualCount; - } else { - // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent - this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release); - } - } - - return 0; - } - - private: - struct BlockIndexEntry { - index_t base; - Block* block; - }; - - struct BlockIndexHeader { - size_t size; - std::atomic front; // Current slot (not next, like pr_blockIndexFront) - BlockIndexEntry* entries; - void* prev; - }; - - bool new_block_index(size_t numberOfFilledSlotsToExpose) - { - auto prevBlockSizeMask = pr_blockIndexSize - 1; - - // Create the new block - pr_blockIndexSize <<= 1; - auto newRawPtr = static_cast((Traits::malloc)(sizeof(BlockIndexHeader) + - std::alignment_of::value - 1 + - sizeof(BlockIndexEntry) * pr_blockIndexSize)); - if (newRawPtr == nullptr) { - pr_blockIndexSize >>= 1; // Reset to allow graceful retry - return false; - } - - auto newBlockIndexEntries = reinterpret_cast( - details::align_for(newRawPtr + sizeof(BlockIndexHeader))); - - // Copy in all the old indices, if any - size_t j = 0; - if (pr_blockIndexSlotsUsed != 0) { - auto i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & prevBlockSizeMask; - do { - newBlockIndexEntries[j++] = pr_blockIndexEntries[i]; - i = (i + 1) & prevBlockSizeMask; - } while (i != pr_blockIndexFront); - } - - // Update everything - auto header = new (newRawPtr) BlockIndexHeader; - header->size = pr_blockIndexSize; - header->front.store(numberOfFilledSlotsToExpose - 1, std::memory_order_relaxed); - header->entries = newBlockIndexEntries; - header->prev = pr_blockIndexRaw; // we link the new block to the old one so we can free it later - - pr_blockIndexFront = j; - pr_blockIndexEntries = newBlockIndexEntries; - pr_blockIndexRaw = newRawPtr; - blockIndex.store(header, std::memory_order_release); - - return true; - } - - private: - std::atomic blockIndex; - - // To be used by producer only -- consumer must use the ones in referenced by blockIndex - size_t pr_blockIndexSlotsUsed; - size_t pr_blockIndexSize; - size_t pr_blockIndexFront; // Next slot (not current) - BlockIndexEntry* pr_blockIndexEntries; - void* pr_blockIndexRaw; - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - public: - ExplicitProducer* nextExplicitProducer; - - private: -#endif - -#ifdef MCDBGQ_TRACKMEM - friend struct MemStats; -#endif - }; - - ////////////////////////////////// - // Implicit queue - ////////////////////////////////// - - struct ImplicitProducer : public ProducerBase { - ImplicitProducer(ConcurrentQueue* parent_) - : ProducerBase(parent_, false) - , nextBlockIndexCapacity(IMPLICIT_INITIAL_INDEX_SIZE) - , blockIndex(nullptr) - { - new_block_index(); - } - - ~ImplicitProducer() - { - // Note that since we're in the destructor we can assume that all enqueue/dequeue operations - // completed already; this means that all undequeued elements are placed contiguously across - // contiguous blocks, and that only the first and last remaining blocks can be only partially - // empty (all other remaining blocks must be completely full). - -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - // Unregister ourselves for thread termination notification - if (!this->inactive.load(std::memory_order_relaxed)) { - details::ThreadExitNotifier::unsubscribe(&threadExitListener); - } -#endif - - // Destroy all remaining elements! - auto tail = this->tailIndex.load(std::memory_order_relaxed); - auto index = this->headIndex.load(std::memory_order_relaxed); - Block* block = nullptr; - assert(index == tail || details::circular_less_than(index, tail)); - bool forceFreeLastBlock = - index != tail; // If we enter the loop, then the last (tail) block will not be freed - while (index != tail) { - if ((index & static_cast(BLOCK_SIZE - 1)) == 0 || block == nullptr) { - if (block != nullptr) { - // Free the old block - this->parent->add_block_to_free_list(block); - } - - block = get_block_index_entry_for_index(index)->value.load(std::memory_order_relaxed); - } - - ((*block)[index])->~T(); - ++index; - } - // Even if the queue is empty, there's still one block that's not on the free list - // (unless the head index reached the end of it, in which case the tail will be poised - // to create a new block). - if (this->tailBlock != nullptr && - (forceFreeLastBlock || (tail & static_cast(BLOCK_SIZE - 1)) != 0)) { - this->parent->add_block_to_free_list(this->tailBlock); - } - - // Destroy block index - auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); - if (localBlockIndex != nullptr) { - for (size_t i = 0; i != localBlockIndex->capacity; ++i) { - localBlockIndex->index[i]->~BlockIndexEntry(); - } - do { - auto prev = localBlockIndex->prev; - localBlockIndex->~BlockIndexHeader(); - (Traits::free)(localBlockIndex); - localBlockIndex = prev; - } while (localBlockIndex != nullptr); - } - } - - template inline bool enqueue(U&& element) - { - index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); - index_t newTailIndex = 1 + currentTailIndex; - if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { - // We reached the end of a block, start a new one - auto head = this->headIndex.load(std::memory_order_relaxed); - assert(!details::circular_less_than(currentTailIndex, head)); - if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || - (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && - (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { - return false; - } -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - // Find out where we'll be inserting this block in the block index - BlockIndexEntry* idxEntry; - if (!insert_block_index_entry(idxEntry, currentTailIndex)) { - return false; - } - - // Get ahold of a new block - auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); - if (newBlock == nullptr) { - rewind_block_index_tail(); - idxEntry->value.store(nullptr, std::memory_order_relaxed); - return false; - } -#ifdef MCDBGQ_TRACKMEM - newBlock->owner = this; -#endif - newBlock->ConcurrentQueue::Block::template reset_empty(); - - MOODYCAMEL_CONSTEXPR_IF( - !MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) - { - // May throw, try to insert now before we publish the fact that we have this new block - MOODYCAMEL_TRY - { - new ((*newBlock)[currentTailIndex]) T(std::forward(element)); - } - MOODYCAMEL_CATCH(...) - { - rewind_block_index_tail(); - idxEntry->value.store(nullptr, std::memory_order_relaxed); - this->parent->add_block_to_free_list(newBlock); - MOODYCAMEL_RETHROW; - } - } - - // Insert the new block into the index - idxEntry->value.store(newBlock, std::memory_order_relaxed); - - this->tailBlock = newBlock; - - MOODYCAMEL_CONSTEXPR_IF( - !MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) - { - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } - } - - // Enqueue - new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); - - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } - - template bool dequeue(U& element) - { - // See ExplicitProducer::dequeue for rationale and explanation - index_t tail = this->tailIndex.load(std::memory_order_relaxed); - index_t overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); - if (details::circular_less_than( - this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) { - std::atomic_thread_fence(std::memory_order_acquire); - - index_t myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed); - tail = this->tailIndex.load(std::memory_order_acquire); - if ((details::likely)(details::circular_less_than(myDequeueCount - overcommit, tail))) { - index_t index = this->headIndex.fetch_add(1, std::memory_order_acq_rel); - - // Determine which block the element is in - auto entry = get_block_index_entry_for_index(index); - - // Dequeue - auto block = entry->value.load(std::memory_order_relaxed); - auto& el = *((*block)[index]); - - if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, element = std::move(el))) { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - // Note: Acquiring the mutex with every dequeue instead of only when a block - // is released is very sub-optimal, but it is, after all, purely debug code. - debug::DebugLock lock(producer->mutex); -#endif - struct Guard { - Block* block; - index_t index; - BlockIndexEntry* entry; - ConcurrentQueue* parent; - - ~Guard() - { - (*block)[index]->~T(); - if (block->ConcurrentQueue::Block::template set_empty(index)) { - entry->value.store(nullptr, std::memory_order_relaxed); - parent->add_block_to_free_list(block); - } - } - } guard = { block, index, entry, this->parent }; - - element = std::move(el); // NOLINT - } else { - element = std::move(el); // NOLINT - el.~T(); // NOLINT - - if (block->ConcurrentQueue::Block::template set_empty(index)) { - { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - // Add the block back into the global free pool (and remove from block index) - entry->value.store(nullptr, std::memory_order_relaxed); - } - this->parent->add_block_to_free_list(block); // releases the above store - } - } - - return true; - } else { - this->dequeueOvercommit.fetch_add(1, std::memory_order_release); - } - } - - return false; - } - -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4706) // assignment within conditional expression -#endif - template bool enqueue_bulk(It itemFirst, size_t count) - { - // First, we need to make sure we have enough room to enqueue all of the elements; - // this means pre-allocating blocks and putting them in the block index (but only if - // all the allocations succeeded). - - // Note that the tailBlock we start off with may not be owned by us any more; - // this happens if it was filled up exactly to the top (setting tailIndex to - // the first index of the next block which is not yet allocated), then dequeued - // completely (putting it on the free list) before we enqueue again. - - index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed); - auto startBlock = this->tailBlock; - Block* firstAllocatedBlock = nullptr; - auto endBlock = this->tailBlock; - - // Figure out how many blocks we'll need to allocate, and do so - size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); - index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); - if (blockBaseDiff > 0) { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - do { - blockBaseDiff -= static_cast(BLOCK_SIZE); - currentTailIndex += static_cast(BLOCK_SIZE); - - // Find out where we'll be inserting this block in the block index - BlockIndexEntry* idxEntry = - nullptr; // initialization here unnecessary but compiler can't always tell - Block* newBlock; - bool indexInserted = false; - auto head = this->headIndex.load(std::memory_order_relaxed); - assert(!details::circular_less_than(currentTailIndex, head)); - bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || - (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && - (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); - - if (full || !(indexInserted = insert_block_index_entry(idxEntry, currentTailIndex)) || - (newBlock = this->parent->ConcurrentQueue::template requisition_block()) == - nullptr) { - // Index allocation or block allocation failed; revert any other allocations - // and index insertions done so far for this operation - if (indexInserted) { - rewind_block_index_tail(); - idxEntry->value.store(nullptr, std::memory_order_relaxed); - } - currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); - for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { - currentTailIndex += static_cast(BLOCK_SIZE); - idxEntry = get_block_index_entry_for_index(currentTailIndex); - idxEntry->value.store(nullptr, std::memory_order_relaxed); - rewind_block_index_tail(); - } - this->parent->add_blocks_to_free_list(firstAllocatedBlock); - this->tailBlock = startBlock; - - return false; - } - -#ifdef MCDBGQ_TRACKMEM - newBlock->owner = this; -#endif - newBlock->ConcurrentQueue::Block::template reset_empty(); - newBlock->next = nullptr; - - // Insert the new block into the index - idxEntry->value.store(newBlock, std::memory_order_relaxed); - - // Store the chain of blocks so that we can undo if later allocations fail, - // and so that we can find the blocks when we do the actual enqueueing - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || - firstAllocatedBlock != nullptr) { - assert(this->tailBlock != nullptr); - this->tailBlock->next = newBlock; - } - this->tailBlock = newBlock; - endBlock = newBlock; - firstAllocatedBlock = firstAllocatedBlock == nullptr ? newBlock : firstAllocatedBlock; - } while (blockBaseDiff > 0); - } - - // Enqueue, one block at a time - index_t newTailIndex = startTailIndex + static_cast(count); - currentTailIndex = startTailIndex; - this->tailBlock = startBlock; - assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || - count == 0); - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { - this->tailBlock = firstAllocatedBlock; - } - while (true) { - index_t stopIndex = - (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - if (details::circular_less_than(newTailIndex, stopIndex)) { - stopIndex = newTailIndex; - } - MOODYCAMEL_CONSTEXPR_IF(MOODYCAMEL_NOEXCEPT_CTOR( - T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) - { - while (currentTailIndex != stopIndex) { - new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++); - } - } - else - { - MOODYCAMEL_TRY - { - while (currentTailIndex != stopIndex) { - new ((*this->tailBlock)[currentTailIndex]) - T(details::nomove_if(nullptr)) - T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst)); - ++currentTailIndex; - ++itemFirst; - } - } - MOODYCAMEL_CATCH(...) - { - auto constructedStopIndex = currentTailIndex; - auto lastBlockEnqueued = this->tailBlock; - - if (!details::is_trivially_destructible::value) { - auto block = startBlock; - if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { - block = firstAllocatedBlock; - } - currentTailIndex = startTailIndex; - while (true) { - stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + - static_cast(BLOCK_SIZE); - if (details::circular_less_than(constructedStopIndex, stopIndex)) { - stopIndex = constructedStopIndex; - } - while (currentTailIndex != stopIndex) { - (*block)[currentTailIndex++]->~T(); - } - if (block == lastBlockEnqueued) { - break; - } - block = block->next; - } - } - - currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); - for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { - currentTailIndex += static_cast(BLOCK_SIZE); - auto idxEntry = get_block_index_entry_for_index(currentTailIndex); - idxEntry->value.store(nullptr, std::memory_order_relaxed); - rewind_block_index_tail(); - } - this->parent->add_blocks_to_free_list(firstAllocatedBlock); - this->tailBlock = startBlock; - MOODYCAMEL_RETHROW; - } - } - - if (this->tailBlock == endBlock) { - assert(currentTailIndex == newTailIndex); - break; - } - this->tailBlock = this->tailBlock->next; - } - this->tailIndex.store(newTailIndex, std::memory_order_release); - return true; - } -#ifdef _MSC_VER -#pragma warning(pop) -#endif - - template size_t dequeue_bulk(It& itemFirst, size_t max) - { - auto tail = this->tailIndex.load(std::memory_order_relaxed); - auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); - auto desiredCount = - static_cast(tail - (this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit)); - if (details::circular_less_than(0, desiredCount)) { - desiredCount = desiredCount < max ? desiredCount : max; - std::atomic_thread_fence(std::memory_order_acquire); - - auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed); - - tail = this->tailIndex.load(std::memory_order_acquire); - auto actualCount = static_cast(tail - (myDequeueCount - overcommit)); - if (details::circular_less_than(0, actualCount)) { - actualCount = desiredCount < actualCount ? desiredCount : actualCount; - if (actualCount < desiredCount) { - this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release); - } - - // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this - // will never exceed tail. - auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel); - - // Iterate the blocks and dequeue - auto index = firstIndex; - BlockIndexHeader* localBlockIndex; - auto indexIndex = get_block_index_index_for_index(index, localBlockIndex); - do { - auto blockStartIndex = index; - index_t endIndex = - (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); - endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), - endIndex) - ? firstIndex + static_cast(actualCount) - : endIndex; - - auto entry = localBlockIndex->index[indexIndex]; - auto block = entry->value.load(std::memory_order_relaxed); - if (MOODYCAMEL_NOEXCEPT_ASSIGN( - T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { - while (index != endIndex) { - auto& el = *((*block)[index]); - *itemFirst++ = std::move(el); - el.~T(); - ++index; - } - } else { - MOODYCAMEL_TRY - { - while (index != endIndex) { - auto& el = *((*block)[index]); - *itemFirst = std::move(el); - ++itemFirst; - el.~T(); - ++index; - } - } - MOODYCAMEL_CATCH(...) - { - do { - entry = localBlockIndex->index[indexIndex]; - block = entry->value.load(std::memory_order_relaxed); - while (index != endIndex) { - (*block)[index++]->~T(); - } - - if (block->ConcurrentQueue::Block::template set_many_empty( - blockStartIndex, static_cast(endIndex - blockStartIndex))) { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - entry->value.store(nullptr, std::memory_order_relaxed); - this->parent->add_block_to_free_list(block); - } - indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); - - blockStartIndex = index; - endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + - static_cast(BLOCK_SIZE); - endIndex = details::circular_less_than( - firstIndex + static_cast(actualCount), endIndex) - ? firstIndex + static_cast(actualCount) - : endIndex; - } while (index != firstIndex + actualCount); - - MOODYCAMEL_RETHROW; - } - } - if (block->ConcurrentQueue::Block::template set_many_empty( - blockStartIndex, static_cast(endIndex - blockStartIndex))) { - { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - // Note that the set_many_empty above did a release, meaning that anybody who acquires - // the block we're about to free can use it safely since our writes (and reads!) will - // have happened-before then. - entry->value.store(nullptr, std::memory_order_relaxed); - } - this->parent->add_block_to_free_list(block); // releases the above store - } - indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); - } while (index != firstIndex + actualCount); - - return actualCount; - } else { - this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release); - } - } - - return 0; - } - - private: - // The block size must be > 1, so any number with the low bit set is an invalid block base index - static const index_t INVALID_BLOCK_BASE = 1; - - struct BlockIndexEntry { - std::atomic key; - std::atomic value; - }; - - struct BlockIndexHeader { - size_t capacity; - std::atomic tail; - BlockIndexEntry* entries; - BlockIndexEntry** index; - BlockIndexHeader* prev; - }; - - template - inline bool insert_block_index_entry(BlockIndexEntry*& idxEntry, index_t blockStartIndex) - { - auto localBlockIndex = - blockIndex.load(std::memory_order_relaxed); // We're the only writer thread, relaxed is OK - if (localBlockIndex == nullptr) { - return false; // this can happen if new_block_index failed in the constructor - } - size_t newTail = - (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1); - idxEntry = localBlockIndex->index[newTail]; - if (idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE || - idxEntry->value.load(std::memory_order_relaxed) == nullptr) { - - idxEntry->key.store(blockStartIndex, std::memory_order_relaxed); - localBlockIndex->tail.store(newTail, std::memory_order_release); - return true; - } - - // No room in the old block index, try to allocate another one! - MOODYCAMEL_CONSTEXPR_IF(allocMode == CannotAlloc) - { - return false; - } - else if (!new_block_index()) - { - return false; - } - else - { - localBlockIndex = blockIndex.load(std::memory_order_relaxed); - newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1); - idxEntry = localBlockIndex->index[newTail]; - assert(idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE); - idxEntry->key.store(blockStartIndex, std::memory_order_relaxed); - localBlockIndex->tail.store(newTail, std::memory_order_release); - return true; - } - } - - inline void rewind_block_index_tail() - { - auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); - localBlockIndex->tail.store((localBlockIndex->tail.load(std::memory_order_relaxed) - 1) & - (localBlockIndex->capacity - 1), - std::memory_order_relaxed); - } - - inline BlockIndexEntry* get_block_index_entry_for_index(index_t index) const - { - BlockIndexHeader* localBlockIndex; - auto idx = get_block_index_index_for_index(index, localBlockIndex); - return localBlockIndex->index[idx]; - } - - inline size_t get_block_index_index_for_index(index_t index, BlockIndexHeader*& localBlockIndex) const - { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - debug::DebugLock lock(mutex); -#endif - index &= ~static_cast(BLOCK_SIZE - 1); - localBlockIndex = blockIndex.load(std::memory_order_acquire); - auto tail = localBlockIndex->tail.load(std::memory_order_acquire); - auto tailBase = localBlockIndex->index[tail]->key.load(std::memory_order_relaxed); - assert(tailBase != INVALID_BLOCK_BASE); - // Note: Must use division instead of shift because the index may wrap around, causing a negative - // offset, whose negativity we want to preserve - auto offset = static_cast(static_cast::type>(index - tailBase) / - static_cast::type>(BLOCK_SIZE)); - size_t idx = (tail + offset) & (localBlockIndex->capacity - 1); - assert(localBlockIndex->index[idx]->key.load(std::memory_order_relaxed) == index && - localBlockIndex->index[idx]->value.load(std::memory_order_relaxed) != nullptr); - return idx; - } - - bool new_block_index() - { - auto prev = blockIndex.load(std::memory_order_relaxed); - size_t prevCapacity = prev == nullptr ? 0 : prev->capacity; - auto entryCount = prev == nullptr ? nextBlockIndexCapacity : prevCapacity; - auto raw = static_cast( - (Traits::malloc)(sizeof(BlockIndexHeader) + std::alignment_of::value - 1 + - sizeof(BlockIndexEntry) * entryCount + std::alignment_of::value - 1 + - sizeof(BlockIndexEntry*) * nextBlockIndexCapacity)); - if (raw == nullptr) { - return false; - } - - auto header = new (raw) BlockIndexHeader; - auto entries = - reinterpret_cast(details::align_for(raw + sizeof(BlockIndexHeader))); - auto index = reinterpret_cast(details::align_for( - reinterpret_cast(entries) + sizeof(BlockIndexEntry) * entryCount)); - if (prev != nullptr) { - auto prevTail = prev->tail.load(std::memory_order_relaxed); - auto prevPos = prevTail; - size_t i = 0; - do { - prevPos = (prevPos + 1) & (prev->capacity - 1); - index[i++] = prev->index[prevPos]; - } while (prevPos != prevTail); - assert(i == prevCapacity); - } - for (size_t i = 0; i != entryCount; ++i) { - new (entries + i) BlockIndexEntry; - entries[i].key.store(INVALID_BLOCK_BASE, std::memory_order_relaxed); - index[prevCapacity + i] = entries + i; - } - header->prev = prev; - header->entries = entries; - header->index = index; - header->capacity = nextBlockIndexCapacity; - header->tail.store((prevCapacity - 1) & (nextBlockIndexCapacity - 1), std::memory_order_relaxed); - - blockIndex.store(header, std::memory_order_release); - - nextBlockIndexCapacity <<= 1; - - return true; - } - - private: - size_t nextBlockIndexCapacity; - std::atomic blockIndex; - -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - public: - details::ThreadExitListener threadExitListener; - - private: -#endif - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - public: - ImplicitProducer* nextImplicitProducer; - - private: -#endif - -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX - mutable debug::DebugMutex mutex; -#endif -#ifdef MCDBGQ_TRACKMEM - friend struct MemStats; -#endif - }; - - ////////////////////////////////// - // Block pool manipulation - ////////////////////////////////// - - void populate_initial_block_list(size_t blockCount) - { - initialBlockPoolSize = blockCount; - if (initialBlockPoolSize == 0) { - initialBlockPool = nullptr; - return; - } - - initialBlockPool = create_array(blockCount); - if (initialBlockPool == nullptr) { - initialBlockPoolSize = 0; - } - for (size_t i = 0; i < initialBlockPoolSize; ++i) { - initialBlockPool[i].dynamicallyAllocated = false; - } - } - - inline Block* try_get_block_from_initial_pool() - { - if (initialBlockPoolIndex.load(std::memory_order_relaxed) >= initialBlockPoolSize) { - return nullptr; - } - - auto index = initialBlockPoolIndex.fetch_add(1, std::memory_order_relaxed); - - return index < initialBlockPoolSize ? (initialBlockPool + index) : nullptr; - } - - inline void add_block_to_free_list(Block* block) - { -#ifdef MCDBGQ_TRACKMEM - block->owner = nullptr; -#endif - if (!Traits::RECYCLE_ALLOCATED_BLOCKS && block->dynamicallyAllocated) { - destroy(block); - } else { - freeList.add(block); - } - } - - inline void add_blocks_to_free_list(Block* block) - { - while (block != nullptr) { - auto next = block->next; - add_block_to_free_list(block); - block = next; - } - } - - inline Block* try_get_block_from_free_list() { return freeList.try_get(); } - - // Gets a free block from one of the memory pools, or allocates a new one (if applicable) - template Block* requisition_block() - { - auto block = try_get_block_from_initial_pool(); - if (block != nullptr) { - return block; - } - - block = try_get_block_from_free_list(); - if (block != nullptr) { - return block; - } - - MOODYCAMEL_CONSTEXPR_IF(canAlloc == CanAlloc) - { - return create(); - } - else - { - return nullptr; - } - } - -#ifdef MCDBGQ_TRACKMEM - public: - struct MemStats { - size_t allocatedBlocks; - size_t usedBlocks; - size_t freeBlocks; - size_t ownedBlocksExplicit; - size_t ownedBlocksImplicit; - size_t implicitProducers; - size_t explicitProducers; - size_t elementsEnqueued; - size_t blockClassBytes; - size_t queueClassBytes; - size_t implicitBlockIndexBytes; - size_t explicitBlockIndexBytes; - - friend class ConcurrentQueue; - - private: - static MemStats getFor(ConcurrentQueue* q) - { - MemStats stats = { 0 }; - - stats.elementsEnqueued = q->size_approx(); - - auto block = q->freeList.head_unsafe(); - while (block != nullptr) { - ++stats.allocatedBlocks; - ++stats.freeBlocks; - block = block->freeListNext.load(std::memory_order_relaxed); - } - - for (auto ptr = q->producerListTail.load(std::memory_order_acquire); ptr != nullptr; - ptr = ptr->next_prod()) { - bool implicit = dynamic_cast(ptr) != nullptr; - stats.implicitProducers += implicit ? 1 : 0; - stats.explicitProducers += implicit ? 0 : 1; - - if (implicit) { - auto prod = static_cast(ptr); - stats.queueClassBytes += sizeof(ImplicitProducer); - auto head = prod->headIndex.load(std::memory_order_relaxed); - auto tail = prod->tailIndex.load(std::memory_order_relaxed); - auto hash = prod->blockIndex.load(std::memory_order_relaxed); - if (hash != nullptr) { - for (size_t i = 0; i != hash->capacity; ++i) { - if (hash->index[i]->key.load(std::memory_order_relaxed) != - ImplicitProducer::INVALID_BLOCK_BASE && - hash->index[i]->value.load(std::memory_order_relaxed) != nullptr) { - ++stats.allocatedBlocks; - ++stats.ownedBlocksImplicit; - } - } - stats.implicitBlockIndexBytes += - hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry); - for (; hash != nullptr; hash = hash->prev) { - stats.implicitBlockIndexBytes += - sizeof(typename ImplicitProducer::BlockIndexHeader) + - hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry*); - } - } - for (; details::circular_less_than(head, tail); head += BLOCK_SIZE) { - // auto block = prod->get_block_index_entry_for_index(head); - ++stats.usedBlocks; - } - } else { - auto prod = static_cast(ptr); - stats.queueClassBytes += sizeof(ExplicitProducer); - auto tailBlock = prod->tailBlock; - bool wasNonEmpty = false; - if (tailBlock != nullptr) { - auto block = tailBlock; - do { - ++stats.allocatedBlocks; - if (!block->ConcurrentQueue::Block::template is_empty() || wasNonEmpty) { - ++stats.usedBlocks; - wasNonEmpty = wasNonEmpty || block != tailBlock; - } - ++stats.ownedBlocksExplicit; - block = block->next; - } while (block != tailBlock); - } - auto index = prod->blockIndex.load(std::memory_order_relaxed); - while (index != nullptr) { - stats.explicitBlockIndexBytes += - sizeof(typename ExplicitProducer::BlockIndexHeader) + - index->size * sizeof(typename ExplicitProducer::BlockIndexEntry); - index = static_cast(index->prev); - } - } - } - - auto freeOnInitialPool = - q->initialBlockPoolIndex.load(std::memory_order_relaxed) >= q->initialBlockPoolSize - ? 0 - : q->initialBlockPoolSize - q->initialBlockPoolIndex.load(std::memory_order_relaxed); - stats.allocatedBlocks += freeOnInitialPool; - stats.freeBlocks += freeOnInitialPool; - - stats.blockClassBytes = sizeof(Block) * stats.allocatedBlocks; - stats.queueClassBytes += sizeof(ConcurrentQueue); - - return stats; - } - }; - - // For debugging only. Not thread-safe. - MemStats getMemStats() { return MemStats::getFor(this); } - - private: - friend struct MemStats; -#endif - - ////////////////////////////////// - // Producer list manipulation - ////////////////////////////////// - - ProducerBase* recycle_or_create_producer(bool isExplicit) - { -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH - debug::DebugLock lock(implicitProdMutex); -#endif - // Try to re-use one first - for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { - if (ptr->inactive.load(std::memory_order_relaxed) && ptr->isExplicit == isExplicit) { - bool expected = true; - if (ptr->inactive.compare_exchange_strong( - expected, /* desired */ false, std::memory_order_acquire, std::memory_order_relaxed)) { - // We caught one! It's been marked as activated, the caller can have it - return ptr; - } - } - } - - return add_producer(isExplicit ? static_cast(create(this)) - : create(this)); - } - - ProducerBase* add_producer(ProducerBase* producer) - { - // Handle failed memory allocation - if (producer == nullptr) { - return nullptr; - } - - producerCount.fetch_add(1, std::memory_order_relaxed); - - // Add it to the lock-free list - auto prevTail = producerListTail.load(std::memory_order_relaxed); - do { - producer->next = prevTail; - } while (!producerListTail.compare_exchange_weak( - prevTail, producer, std::memory_order_release, std::memory_order_relaxed)); - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - if (producer->isExplicit) { - auto prevTailExplicit = explicitProducers.load(std::memory_order_relaxed); - do { - static_cast(producer)->nextExplicitProducer = prevTailExplicit; - } while (!explicitProducers.compare_exchange_weak(prevTailExplicit, - static_cast(producer), - std::memory_order_release, - std::memory_order_relaxed)); - } else { - auto prevTailImplicit = implicitProducers.load(std::memory_order_relaxed); - do { - static_cast(producer)->nextImplicitProducer = prevTailImplicit; - } while (!implicitProducers.compare_exchange_weak(prevTailImplicit, - static_cast(producer), - std::memory_order_release, - std::memory_order_relaxed)); - } -#endif - - return producer; - } - - void reown_producers() - { - // After another instance is moved-into/swapped-with this one, all the - // producers we stole still think their parents are the other queue. - // So fix them up! - for (auto ptr = producerListTail.load(std::memory_order_relaxed); ptr != nullptr; ptr = ptr->next_prod()) { - ptr->parent = this; - } - } - - ////////////////////////////////// - // Implicit producer hash - ////////////////////////////////// - - struct ImplicitProducerKVP { - std::atomic key; - ImplicitProducer* - value; // No need for atomicity since it's only read by the thread that sets it in the first place - - ImplicitProducerKVP() - : value(nullptr) - {} - - ImplicitProducerKVP(ImplicitProducerKVP&& other) MOODYCAMEL_NOEXCEPT - { - key.store(other.key.load(std::memory_order_relaxed), std::memory_order_relaxed); - value = other.value; - } - - inline ImplicitProducerKVP& operator=(ImplicitProducerKVP&& other) MOODYCAMEL_NOEXCEPT - { - swap(other); - return *this; - } - - inline void swap(ImplicitProducerKVP& other) MOODYCAMEL_NOEXCEPT - { - if (this != &other) { - details::swap_relaxed(key, other.key); - std::swap(value, other.value); - } - } - }; - - template - friend void moodycamel::swap(typename ConcurrentQueue::ImplicitProducerKVP&, - typename ConcurrentQueue::ImplicitProducerKVP&) MOODYCAMEL_NOEXCEPT; - - struct ImplicitProducerHash { - size_t capacity; - ImplicitProducerKVP* entries; - ImplicitProducerHash* prev; - }; - - inline void populate_initial_implicit_producer_hash() - { - MOODYCAMEL_CONSTEXPR_IF(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) - { - return; - } - else - { - implicitProducerHashCount.store(0, std::memory_order_relaxed); - auto hash = &initialImplicitProducerHash; - hash->capacity = INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; - hash->entries = &initialImplicitProducerHashEntries[0]; - for (size_t i = 0; i != INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; ++i) { - initialImplicitProducerHashEntries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed); - } - hash->prev = nullptr; - implicitProducerHash.store(hash, std::memory_order_relaxed); - } - } - - void swap_implicit_producer_hashes(ConcurrentQueue& other) - { - MOODYCAMEL_CONSTEXPR_IF(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) - { - return; - } - else - { - // Swap (assumes our implicit producer hash is initialized) - initialImplicitProducerHashEntries.swap(other.initialImplicitProducerHashEntries); - initialImplicitProducerHash.entries = &initialImplicitProducerHashEntries[0]; - other.initialImplicitProducerHash.entries = &other.initialImplicitProducerHashEntries[0]; - - details::swap_relaxed(implicitProducerHashCount, other.implicitProducerHashCount); - - details::swap_relaxed(implicitProducerHash, other.implicitProducerHash); - if (implicitProducerHash.load(std::memory_order_relaxed) == &other.initialImplicitProducerHash) { - implicitProducerHash.store(&initialImplicitProducerHash, std::memory_order_relaxed); - } else { - ImplicitProducerHash* hash; - for (hash = implicitProducerHash.load(std::memory_order_relaxed); - hash->prev != &other.initialImplicitProducerHash; - hash = hash->prev) { - continue; - } - hash->prev = &initialImplicitProducerHash; - } - if (other.implicitProducerHash.load(std::memory_order_relaxed) == &initialImplicitProducerHash) { - other.implicitProducerHash.store(&other.initialImplicitProducerHash, std::memory_order_relaxed); - } else { - ImplicitProducerHash* hash; - for (hash = other.implicitProducerHash.load(std::memory_order_relaxed); - hash->prev != &initialImplicitProducerHash; - hash = hash->prev) { - continue; - } - hash->prev = &other.initialImplicitProducerHash; - } - } - } - - // Only fails (returns nullptr) if memory allocation fails - ImplicitProducer* get_or_add_implicit_producer() - { - // Note that since the data is essentially thread-local (key is thread ID), - // there's a reduced need for fences (memory ordering is already consistent - // for any individual thread), except for the current table itself. - - // Start by looking for the thread ID in the current and all previous hash tables. - // If it's not found, it must not be in there yet, since this same thread would - // have added it previously to one of the tables that we traversed. - - // Code and algorithm adapted from http://preshing.com/20130605/the-worlds-simplest-lock-free-hash-table - -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH - debug::DebugLock lock(implicitProdMutex); -#endif - - auto id = details::thread_id(); - auto hashedId = details::hash_thread_id(id); - - auto mainHash = implicitProducerHash.load(std::memory_order_acquire); - assert(mainHash != nullptr); // silence clang-tidy and MSVC warnings (hash cannot be null) - for (auto hash = mainHash; hash != nullptr; hash = hash->prev) { - // Look for the id in this hash - auto index = hashedId; - while (true) { // Not an infinite loop because at least one slot is free in the hash table - index &= hash->capacity - 1u; - - auto probedKey = hash->entries[index].key.load(std::memory_order_relaxed); - if (probedKey == id) { - // Found it! If we had to search several hashes deep, though, we should lazily add it - // to the current main hash table to avoid the extended search next time. - // Note there's guaranteed to be room in the current hash table since every subsequent - // table implicitly reserves space for all previous tables (there's only one - // implicitProducerHashCount). - auto value = hash->entries[index].value; - if (hash != mainHash) { - index = hashedId; - while (true) { - index &= mainHash->capacity - 1u; - auto empty = details::invalid_thread_id; -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - auto reusable = details::invalid_thread_id2; - if (mainHash->entries[index].key.compare_exchange_strong( - empty, id, std::memory_order_seq_cst, std::memory_order_relaxed) || - mainHash->entries[index].key.compare_exchange_strong( - reusable, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { -#else - if (mainHash->entries[index].key.compare_exchange_strong( - empty, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { -#endif - mainHash->entries[index].value = value; - break; - } - ++index; - } - } - - return value; - } - if (probedKey == details::invalid_thread_id) { - break; // Not in this hash table - } - ++index; - } - } - - // Insert! - auto newCount = 1 + implicitProducerHashCount.fetch_add(1, std::memory_order_relaxed); - while (true) { - // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) - if (newCount >= (mainHash->capacity >> 1) && - !implicitProducerHashResizeInProgress.test_and_set(std::memory_order_acquire)) { - // We've acquired the resize lock, try to allocate a bigger hash table. - // Note the acquire fence synchronizes with the release fence at the end of this block, and hence when - // we reload implicitProducerHash it must be the most recent version (it only gets changed within this - // locked block). - mainHash = implicitProducerHash.load(std::memory_order_acquire); - if (newCount >= (mainHash->capacity >> 1)) { - size_t newCapacity = mainHash->capacity << 1; - while (newCount >= (newCapacity >> 1)) { - newCapacity <<= 1; - } - auto raw = static_cast((Traits::malloc)(sizeof(ImplicitProducerHash) + - std::alignment_of::value - 1 + - sizeof(ImplicitProducerKVP) * newCapacity)); - if (raw == nullptr) { - // Allocation failed - implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); - implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); - return nullptr; - } - - auto newHash = new (raw) ImplicitProducerHash; - newHash->capacity = static_cast(newCapacity); - newHash->entries = reinterpret_cast( - details::align_for(raw + sizeof(ImplicitProducerHash))); - for (size_t i = 0; i != newCapacity; ++i) { - new (newHash->entries + i) ImplicitProducerKVP; - newHash->entries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed); - } - newHash->prev = mainHash; - implicitProducerHash.store(newHash, std::memory_order_release); - implicitProducerHashResizeInProgress.clear(std::memory_order_release); - mainHash = newHash; - } else { - implicitProducerHashResizeInProgress.clear(std::memory_order_release); - } - } - - // If it's < three-quarters full, add to the old one anyway so that we don't have to wait for the next table - // to finish being allocated by another thread (and if we just finished allocating above, the condition will - // always be true) - if (newCount < (mainHash->capacity >> 1) + (mainHash->capacity >> 2)) { - auto producer = static_cast(recycle_or_create_producer(false)); - if (producer == nullptr) { - implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); - return nullptr; - } - -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - producer->threadExitListener.callback = &ConcurrentQueue::implicit_producer_thread_exited_callback; - producer->threadExitListener.userData = producer; - details::ThreadExitNotifier::subscribe(&producer->threadExitListener); -#endif - - auto index = hashedId; - while (true) { - index &= mainHash->capacity - 1u; - auto empty = details::invalid_thread_id; -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - auto reusable = details::invalid_thread_id2; - if (mainHash->entries[index].key.compare_exchange_strong( - reusable, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { - implicitProducerHashCount.fetch_sub( - 1, std::memory_order_relaxed); // already counted as a used slot - mainHash->entries[index].value = producer; - break; - } -#endif - if (mainHash->entries[index].key.compare_exchange_strong( - empty, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { - mainHash->entries[index].value = producer; - break; - } - ++index; - } - return producer; - } - - // Hmm, the old hash is quite full and somebody else is busy allocating a new one. - // We need to wait for the allocating thread to finish (if it succeeds, we add, if not, - // we try to allocate ourselves). - mainHash = implicitProducerHash.load(std::memory_order_acquire); - } - } - -#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED - void implicit_producer_thread_exited(ImplicitProducer* producer) - { - // Remove from hash -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH - debug::DebugLock lock(implicitProdMutex); -#endif - auto hash = implicitProducerHash.load(std::memory_order_acquire); - assert(hash != - nullptr); // The thread exit listener is only registered if we were added to a hash in the first place - auto id = details::thread_id(); - auto hashedId = details::hash_thread_id(id); - details::thread_id_t probedKey; - - // We need to traverse all the hashes just in case other threads aren't on the current one yet and are - // trying to add an entry thinking there's a free slot (because they reused a producer) - for (; hash != nullptr; hash = hash->prev) { - auto index = hashedId; - do { - index &= hash->capacity - 1u; - probedKey = id; - if (hash->entries[index].key.compare_exchange_strong( - probedKey, details::invalid_thread_id2, std::memory_order_seq_cst, std::memory_order_relaxed)) { - break; - } - ++index; - } while (probedKey != - details::invalid_thread_id); // Can happen if the hash has changed but we weren't put back in it - // yet, or if we weren't added to this hash in the first place - } - - // Mark the queue as being recyclable - producer->inactive.store(true, std::memory_order_release); - } - - static void implicit_producer_thread_exited_callback(void* userData) - { - auto producer = static_cast(userData); - auto queue = producer->parent; - queue->implicit_producer_thread_exited(producer); - } -#endif - - ////////////////////////////////// - // Utility functions - ////////////////////////////////// - - template static inline void* aligned_malloc(size_t size) - { - MOODYCAMEL_CONSTEXPR_IF(std::alignment_of::value <= std::alignment_of::value) - return (Traits::malloc)(size); - else - { - size_t alignment = std::alignment_of::value; - void* raw = (Traits::malloc)(size + alignment - 1 + sizeof(void*)); - if (!raw) - return nullptr; - char* ptr = details::align_for(reinterpret_cast(raw) + sizeof(void*)); - *(reinterpret_cast(ptr) - 1) = raw; - return ptr; - } - } - - template static inline void aligned_free(void* ptr) - { - MOODYCAMEL_CONSTEXPR_IF(std::alignment_of::value <= std::alignment_of::value) - return (Traits::free)(ptr); - else(Traits::free)(ptr ? *(reinterpret_cast(ptr) - 1) : nullptr); - } - - template static inline U* create_array(size_t count) - { - assert(count > 0); - U* p = static_cast(aligned_malloc(sizeof(U) * count)); - if (p == nullptr) - return nullptr; - - for (size_t i = 0; i != count; ++i) - new (p + i) U(); - return p; - } - - template static inline void destroy_array(U* p, size_t count) - { - if (p != nullptr) { - assert(count > 0); - for (size_t i = count; i != 0;) - (p + --i)->~U(); - } - aligned_free(p); - } - - template static inline U* create() - { - void* p = aligned_malloc(sizeof(U)); - return p != nullptr ? new (p) U : nullptr; - } - - template static inline U* create(A1&& a1) - { - void* p = aligned_malloc(sizeof(U)); - return p != nullptr ? new (p) U(std::forward(a1)) : nullptr; - } - - template static inline void destroy(U* p) - { - if (p != nullptr) - p->~U(); - aligned_free(p); - } - - private: - std::atomic producerListTail; - std::atomic producerCount; - - std::atomic initialBlockPoolIndex; - Block* initialBlockPool; - size_t initialBlockPoolSize; - -#ifndef MCDBGQ_USEDEBUGFREELIST - FreeList freeList; -#else - debug::DebugFreeList freeList; -#endif - - std::atomic implicitProducerHash; - std::atomic implicitProducerHashCount; // Number of slots logically used - ImplicitProducerHash initialImplicitProducerHash; - std::array initialImplicitProducerHashEntries; - std::atomic_flag implicitProducerHashResizeInProgress; - - std::atomic nextExplicitConsumerId; - std::atomic globalExplicitConsumerOffset; - -#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH - debug::DebugMutex implicitProdMutex; -#endif - -#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG - std::atomic explicitProducers; - std::atomic implicitProducers; -#endif -}; - -template -ProducerToken::ProducerToken(ConcurrentQueue& queue) - : producer(queue.recycle_or_create_producer(true)) -{ - if (producer != nullptr) { - producer->token = this; - } -} - -template -ProducerToken::ProducerToken(BlockingConcurrentQueue& queue) - : producer(reinterpret_cast*>(&queue)->recycle_or_create_producer(true)) -{ - if (producer != nullptr) { - producer->token = this; - } -} - -template -ConsumerToken::ConsumerToken(ConcurrentQueue& queue) - : itemsConsumedFromCurrent(0) - , currentProducer(nullptr) - , desiredProducer(nullptr) -{ - initialOffset = queue.nextExplicitConsumerId.fetch_add(1, std::memory_order_release); - lastKnownGlobalOffset = static_cast(-1); -} - -template -ConsumerToken::ConsumerToken(BlockingConcurrentQueue& queue) - : itemsConsumedFromCurrent(0) - , currentProducer(nullptr) - , desiredProducer(nullptr) -{ - initialOffset = reinterpret_cast*>(&queue)->nextExplicitConsumerId.fetch_add( - 1, std::memory_order_release); - lastKnownGlobalOffset = static_cast(-1); -} - -template -inline void swap(ConcurrentQueue& a, ConcurrentQueue& b) MOODYCAMEL_NOEXCEPT -{ - a.swap(b); -} - -inline void swap(ProducerToken& a, ProducerToken& b) MOODYCAMEL_NOEXCEPT -{ - a.swap(b); -} - -inline void swap(ConsumerToken& a, ConsumerToken& b) MOODYCAMEL_NOEXCEPT -{ - a.swap(b); -} - -template -inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, - typename ConcurrentQueue::ImplicitProducerKVP& b) MOODYCAMEL_NOEXCEPT -{ - a.swap(b); -} - -} // namespace moodycamel - -#if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17) -#pragma warning(pop) -#endif - -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) -#pragma GCC diagnostic pop -#endif \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/common/moody/lightweightsemaphore.h b/barretenberg/cpp/src/barretenberg/common/moody/lightweightsemaphore.h deleted file mode 100644 index d0ee0e45253..00000000000 --- a/barretenberg/cpp/src/barretenberg/common/moody/lightweightsemaphore.h +++ /dev/null @@ -1,396 +0,0 @@ -// Provides an efficient implementation of a semaphore (LightweightSemaphore). -// This is an extension of Jeff Preshing's sempahore implementation (licensed -// under the terms of its separate zlib license) that has been adapted and -// extended by Cameron Desrochers. - -#pragma once - -#include // For std::size_t -#include -#include // For std::make_signed - -#if defined(_WIN32) -// Avoid including windows.h in a header; we only need a handful of -// items, so we'll redeclare them here (this is relatively safe since -// the API generally has to remain stable between Windows versions). -// I know this is an ugly hack but it still beats polluting the global -// namespace with thousands of generic names or adding a .cpp for nothing. -extern "C" { -struct _SECURITY_ATTRIBUTES; -__declspec(dllimport) void* __stdcall CreateSemaphoreW(_SECURITY_ATTRIBUTES* lpSemaphoreAttributes, - long lInitialCount, - long lMaximumCount, - const wchar_t* lpName); -__declspec(dllimport) int __stdcall CloseHandle(void* hObject); -__declspec(dllimport) unsigned long __stdcall WaitForSingleObject(void* hHandle, unsigned long dwMilliseconds); -__declspec(dllimport) int __stdcall ReleaseSemaphore(void* hSemaphore, long lReleaseCount, long* lpPreviousCount); -} -#elif defined(__MACH__) -#include -#elif defined(__unix__) || defined(__wasm__) -#include - -#if defined(__GLIBC_PREREQ) && defined(_GNU_SOURCE) -#if __GLIBC_PREREQ(2, 30) -#define MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC -#endif -#endif -#endif - -namespace moodycamel { -namespace details { - -// Code in the mpmc_sema namespace below is an adaptation of Jeff Preshing's -// portable + lightweight semaphore implementations, originally from -// https://github.com/preshing/cpp11-on-multicore/blob/master/common/sema.h -// LICENSE: -// Copyright (c) 2015 Jeff Preshing -// -// This software is provided 'as-is', without any express or implied -// warranty. In no event will the authors be held liable for any damages -// arising from the use of this software. -// -// Permission is granted to anyone to use this software for any purpose, -// including commercial applications, and to alter it and redistribute it -// freely, subject to the following restrictions: -// -// 1. The origin of this software must not be misrepresented; you must not -// claim that you wrote the original software. If you use this software -// in a product, an acknowledgement in the product documentation would be -// appreciated but is not required. -// 2. Altered source versions must be plainly marked as such, and must not be -// misrepresented as being the original software. -// 3. This notice may not be removed or altered from any source distribution. -#if defined(_WIN32) -class Semaphore { - private: - void* m_hSema; - - Semaphore(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION; - Semaphore& operator=(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION; - - public: - Semaphore(int initialCount = 0) - { - assert(initialCount >= 0); - const long maxLong = 0x7fffffff; - m_hSema = CreateSemaphoreW(nullptr, initialCount, maxLong, nullptr); - assert(m_hSema); - } - - ~Semaphore() { CloseHandle(m_hSema); } - - bool wait() - { - const unsigned long infinite = 0xffffffff; - return WaitForSingleObject(m_hSema, infinite) == 0; - } - - bool try_wait() { return WaitForSingleObject(m_hSema, 0) == 0; } - - bool timed_wait(std::uint64_t usecs) { return WaitForSingleObject(m_hSema, (unsigned long)(usecs / 1000)) == 0; } - - void signal(int count = 1) - { - while (!ReleaseSemaphore(m_hSema, count, nullptr)) - ; - } -}; -#elif defined(__MACH__) -//--------------------------------------------------------- -// Semaphore (Apple iOS and OSX) -// Can't use POSIX semaphores due to http://lists.apple.com/archives/darwin-kernel/2009/Apr/msg00010.html -//--------------------------------------------------------- -class Semaphore { - private: - semaphore_t m_sema; - - Semaphore(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION; - Semaphore& operator=(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION; - - public: - Semaphore(int initialCount = 0) - { - assert(initialCount >= 0); - kern_return_t rc = semaphore_create(mach_task_self(), &m_sema, SYNC_POLICY_FIFO, initialCount); - assert(rc == KERN_SUCCESS); - (void)rc; - } - - ~Semaphore() { semaphore_destroy(mach_task_self(), m_sema); } - - bool wait() { return semaphore_wait(m_sema) == KERN_SUCCESS; } - - bool try_wait() { return timed_wait(0); } - - bool timed_wait(std::uint64_t timeout_usecs) - { - mach_timespec_t ts; - ts.tv_sec = static_cast(timeout_usecs / 1000000); - ts.tv_nsec = static_cast((timeout_usecs % 1000000) * 1000); - - // added in OSX 10.10: - // https://developer.apple.com/library/prerelease/mac/documentation/General/Reference/APIDiffsMacOSX10_10SeedDiff/modules/Darwin.html - kern_return_t rc = semaphore_timedwait(m_sema, ts); - return rc == KERN_SUCCESS; - } - - void signal() - { - while (semaphore_signal(m_sema) != KERN_SUCCESS) - ; - } - - void signal(int count) - { - while (count-- > 0) { - while (semaphore_signal(m_sema) != KERN_SUCCESS) - ; - } - } -}; -#elif defined(__unix__) || defined(__wasm__) -//--------------------------------------------------------- -// Semaphore (POSIX, Linux) -//--------------------------------------------------------- -class Semaphore { - private: - sem_t m_sema; - - Semaphore(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION; - Semaphore& operator=(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION; - - public: - Semaphore(int initialCount = 0) - { - assert(initialCount >= 0); - int rc = sem_init(&m_sema, 0, static_cast(initialCount)); - assert(rc == 0); - (void)rc; - } - - ~Semaphore() { sem_destroy(&m_sema); } - - bool wait() - { - // http://stackoverflow.com/questions/2013181/gdb-causes-sem-wait-to-fail-with-eintr-error - int rc; - do { - rc = sem_wait(&m_sema); - } while (rc == -1 && errno == EINTR); - return rc == 0; - } - - bool try_wait() - { - int rc; - do { - rc = sem_trywait(&m_sema); - } while (rc == -1 && errno == EINTR); - return rc == 0; - } - - bool timed_wait(std::uint64_t usecs) - { - struct timespec ts; - const int usecs_in_1_sec = 1000000; - const int nsecs_in_1_sec = 1000000000; -#ifdef MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC - clock_gettime(CLOCK_MONOTONIC, &ts); -#else - clock_gettime(CLOCK_REALTIME, &ts); -#endif - ts.tv_sec += (time_t)(usecs / usecs_in_1_sec); - ts.tv_nsec += (long)(usecs % usecs_in_1_sec) * 1000; - // sem_timedwait bombs if you have more than 1e9 in tv_nsec - // so we have to clean things up before passing it in - if (ts.tv_nsec >= nsecs_in_1_sec) { - ts.tv_nsec -= nsecs_in_1_sec; - ++ts.tv_sec; - } - - int rc; - do { -#ifdef MOODYCAMEL_LIGHTWEIGHTSEMAPHORE_MONOTONIC - rc = sem_clockwait(&m_sema, CLOCK_MONOTONIC, &ts); -#else - rc = sem_timedwait(&m_sema, &ts); -#endif - } while (rc == -1 && errno == EINTR); - return rc == 0; - } - - void signal() - { - while (sem_post(&m_sema) == -1) - ; - } - - void signal(int count) - { - while (count-- > 0) { - while (sem_post(&m_sema) == -1) - ; - } - } -}; -#else -#error Unsupported platform! (No semaphore wrapper available) -#endif - -} // end namespace details - -//--------------------------------------------------------- -// LightweightSemaphore -//--------------------------------------------------------- -class LightweightSemaphore { - public: - typedef std::make_signed::type ssize_t; - - private: - std::atomic m_count; - details::Semaphore m_sema; - int m_maxSpins; - - bool waitWithPartialSpinning(std::int64_t timeout_usecs = -1) - { - ssize_t oldCount; - int spin = m_maxSpins; - while (--spin >= 0) { - oldCount = m_count.load(std::memory_order_relaxed); - if ((oldCount > 0) && m_count.compare_exchange_strong( - oldCount, oldCount - 1, std::memory_order_acquire, std::memory_order_relaxed)) - return true; - std::atomic_signal_fence(std::memory_order_acquire); // Prevent the compiler from collapsing the loop. - } - oldCount = m_count.fetch_sub(1, std::memory_order_acquire); - if (oldCount > 0) - return true; - if (timeout_usecs < 0) { - if (m_sema.wait()) - return true; - } - if (timeout_usecs > 0 && m_sema.timed_wait((std::uint64_t)timeout_usecs)) - return true; - // At this point, we've timed out waiting for the semaphore, but the - // count is still decremented indicating we may still be waiting on - // it. So we have to re-adjust the count, but only if the semaphore - // wasn't signaled enough times for us too since then. If it was, we - // need to release the semaphore too. - while (true) { - oldCount = m_count.load(std::memory_order_acquire); - if (oldCount >= 0 && m_sema.try_wait()) - return true; - if (oldCount < 0 && m_count.compare_exchange_strong( - oldCount, oldCount + 1, std::memory_order_relaxed, std::memory_order_relaxed)) - return false; - } - } - - ssize_t waitManyWithPartialSpinning(ssize_t max, std::int64_t timeout_usecs = -1) - { - assert(max > 0); - ssize_t oldCount; - int spin = m_maxSpins; - while (--spin >= 0) { - oldCount = m_count.load(std::memory_order_relaxed); - if (oldCount > 0) { - ssize_t newCount = oldCount > max ? oldCount - max : 0; - if (m_count.compare_exchange_strong( - oldCount, newCount, std::memory_order_acquire, std::memory_order_relaxed)) - return oldCount - newCount; - } - std::atomic_signal_fence(std::memory_order_acquire); - } - oldCount = m_count.fetch_sub(1, std::memory_order_acquire); - if (oldCount <= 0) { - if ((timeout_usecs == 0) || (timeout_usecs < 0 && !m_sema.wait()) || - (timeout_usecs > 0 && !m_sema.timed_wait((std::uint64_t)timeout_usecs))) { - while (true) { - oldCount = m_count.load(std::memory_order_acquire); - if (oldCount >= 0 && m_sema.try_wait()) - break; - if (oldCount < 0 && - m_count.compare_exchange_strong( - oldCount, oldCount + 1, std::memory_order_relaxed, std::memory_order_relaxed)) - return 0; - } - } - } - if (max > 1) - return 1 + tryWaitMany(max - 1); - return 1; - } - - public: - LightweightSemaphore(ssize_t initialCount = 0, int maxSpins = 10000) - : m_count(initialCount) - , m_maxSpins(maxSpins) - { - assert(initialCount >= 0); - assert(maxSpins >= 0); - } - - bool tryWait() - { - ssize_t oldCount = m_count.load(std::memory_order_relaxed); - while (oldCount > 0) { - if (m_count.compare_exchange_weak( - oldCount, oldCount - 1, std::memory_order_acquire, std::memory_order_relaxed)) - return true; - } - return false; - } - - bool wait() { return tryWait() || waitWithPartialSpinning(); } - - bool wait(std::int64_t timeout_usecs) { return tryWait() || waitWithPartialSpinning(timeout_usecs); } - - // Acquires between 0 and (greedily) max, inclusive - ssize_t tryWaitMany(ssize_t max) - { - assert(max >= 0); - ssize_t oldCount = m_count.load(std::memory_order_relaxed); - while (oldCount > 0) { - ssize_t newCount = oldCount > max ? oldCount - max : 0; - if (m_count.compare_exchange_weak(oldCount, newCount, std::memory_order_acquire, std::memory_order_relaxed)) - return oldCount - newCount; - } - return 0; - } - - // Acquires at least one, and (greedily) at most max - ssize_t waitMany(ssize_t max, std::int64_t timeout_usecs) - { - assert(max >= 0); - ssize_t result = tryWaitMany(max); - if (result == 0 && max > 0) - result = waitManyWithPartialSpinning(max, timeout_usecs); - return result; - } - - ssize_t waitMany(ssize_t max) - { - ssize_t result = waitMany(max, -1); - assert(result > 0); - return result; - } - - void signal(ssize_t count = 1) - { - assert(count >= 0); - ssize_t oldCount = m_count.fetch_add(count, std::memory_order_release); - ssize_t toRelease = -oldCount < count ? -oldCount : count; - if (toRelease > 0) { - m_sema.signal((int)toRelease); - } - } - - std::size_t availableApprox() const - { - ssize_t count = m_count.load(std::memory_order_relaxed); - return count > 0 ? static_cast(count) : 0; - } -}; - -} // end namespace moodycamel \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/common/parallel_for_atomic_pool.cpp b/barretenberg/cpp/src/barretenberg/common/parallel_for_atomic_pool.cpp index 29c45e34896..f71522f9f02 100644 --- a/barretenberg/cpp/src/barretenberg/common/parallel_for_atomic_pool.cpp +++ b/barretenberg/cpp/src/barretenberg/common/parallel_for_atomic_pool.cpp @@ -1,3 +1,4 @@ +#ifndef NO_MULTITHREADING #include "log.hpp" #include "thread.hpp" #include @@ -113,4 +114,5 @@ void parallel_for_atomic_pool(size_t num_iterations, const std::function -#include -#include -#include -#include -#include -#include - -namespace { -class ThreadPool { - public: - ThreadPool(size_t num_threads) - : tasks(1024) - , complete_queue_(1) - { - workers.reserve(num_threads); - for (size_t i = 0; i < num_threads; ++i) { - workers.emplace_back(&ThreadPool::worker_loop, this, i); - } - } - - ~ThreadPool() - { - stop = true; - for (size_t i = 0; i < workers.size(); ++i) { - tasks.enqueue([]() {}); - } - for (auto& worker : workers) { - worker.join(); - } - } - - ThreadPool(const ThreadPool& other) = delete; - ThreadPool(ThreadPool&& other) = delete; - ThreadPool& operator=(const ThreadPool& other) = delete; - ThreadPool& operator=(ThreadPool&& other) = delete; - - void start_tasks(const std::function& task, size_t num_iterations) - { - std::atomic complete_counter; - // 3rd party library expects c-style array as input. Boo. - // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) - std::function funcs[num_iterations]; - for (size_t i = 0; i < num_iterations; ++i) { - funcs[i] = [&, i]() { - // Timer t; - task(i); - // info("task took: ", t.nanoseconds()); - if (complete_counter.fetch_add(1, std::memory_order_relaxed) == num_iterations - 1) { - // info("iteration ", i, " was the last"); - complete_queue_.enqueue(true); - } - }; - } - tasks.enqueue_bulk(funcs, num_iterations); - - { - std::function task; - while (tasks.try_dequeue(task)) { - task(); - } - } - - bool complete = false; - complete_queue_.wait_dequeue(complete); - // info("all done!"); - } - - private: - std::vector workers; - moodycamel::BlockingConcurrentQueue> tasks; - moodycamel::BlockingConcurrentQueue complete_queue_; - std::atomic stop = false; - - void worker_loop(size_t /*unused*/) - { - // info("worker started"); - while (!stop) { - std::function task; - tasks.wait_dequeue(task); - task(); - } - } -}; -} // namespace - -namespace bb { -/** - * A Thread pooled strategy that uses a popular lock-free multiple-producer multiple-consume queue library by - * "moodycamel" as the underlying mechanism to distribute work and join on completion. - */ -void parallel_for_moody(size_t num_iterations, const std::function& func) -{ - // -1 because main thread works. - static ThreadPool pool(get_num_cpus() - 1); - - pool.start_tasks(func, num_iterations); -} -} // namespace bb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/common/parallel_for_mutex_pool.cpp b/barretenberg/cpp/src/barretenberg/common/parallel_for_mutex_pool.cpp index 24479dec9c1..16b922b99c8 100644 --- a/barretenberg/cpp/src/barretenberg/common/parallel_for_mutex_pool.cpp +++ b/barretenberg/cpp/src/barretenberg/common/parallel_for_mutex_pool.cpp @@ -1,3 +1,4 @@ +#ifndef NO_MULTITHREADING #include "log.hpp" #include "thread.hpp" #include @@ -128,4 +129,5 @@ void parallel_for_mutex_pool(size_t num_iterations, const std::function #include @@ -11,4 +12,5 @@ void parallel_for_omp(size_t num_iterations, const std::function& func(i); } } -} // namespace bb \ No newline at end of file +} // namespace bb +#endif \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/common/parallel_for_queued.cpp b/barretenberg/cpp/src/barretenberg/common/parallel_for_queued.cpp index 33dc7af9c29..acaf920fd03 100644 --- a/barretenberg/cpp/src/barretenberg/common/parallel_for_queued.cpp +++ b/barretenberg/cpp/src/barretenberg/common/parallel_for_queued.cpp @@ -1,3 +1,4 @@ +#ifndef NO_MULTITHREADING #include "log.hpp" #include "thread.hpp" #include @@ -123,4 +124,5 @@ void parallel_for_queued(size_t num_iterations, const std::function namespace bb { /** @@ -41,3 +43,4 @@ void parallel_for_spawning(size_t num_iterations, const std::function inline void read(B& it, std::optional& opt_ } template -concept HasGetAll = requires(T t) { t.get_all(); } && ! -msgpack_concepts::HasMsgPack; +concept HasGetAll = requires(T t) { t.get_all(); } && !msgpack_concepts::HasMsgPack; // Write out a struct that defines get_all() template inline void write(B& buf, T const& value) diff --git a/barretenberg/cpp/src/barretenberg/common/thread.hpp b/barretenberg/cpp/src/barretenberg/common/thread.hpp index 723d2834fa5..77309cd9098 100644 --- a/barretenberg/cpp/src/barretenberg/common/thread.hpp +++ b/barretenberg/cpp/src/barretenberg/common/thread.hpp @@ -4,18 +4,13 @@ #include #include #include -#include #include namespace bb { inline size_t get_num_cpus() { -#ifdef NO_MULTITHREADING - return 1; -#else return env_hardware_concurrency(); -#endif } // For algorithms that need to be divided amongst power of 2 threads. @@ -117,4 +112,4 @@ size_t calculate_num_threads(size_t num_iterations, size_t min_iterations_per_th size_t calculate_num_threads_pow2(size_t num_iterations, size_t min_iterations_per_thread = DEFAULT_MIN_ITERS_PER_THREAD); -} // namespace bb \ No newline at end of file +} // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp b/barretenberg/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp index dabd6374720..d537a18deca 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp @@ -30,6 +30,7 @@ ecdsa_signature ecdsa_construct_signature(const std::string& message, const ecds template typename G1::affine_element ecdsa_recover_public_key(const std::string& message, const ecdsa_signature& sig); +// TODO(https://github.com/AztecProtocol/barretenberg/issues/659) template bool ecdsa_verify_signature(const std::string& message, const typename G1::affine_element& public_key, diff --git a/barretenberg/cpp/src/barretenberg/env/hardware_concurrency.cpp b/barretenberg/cpp/src/barretenberg/env/hardware_concurrency.cpp index 9217e014eb9..5fe706d527f 100644 --- a/barretenberg/cpp/src/barretenberg/env/hardware_concurrency.cpp +++ b/barretenberg/cpp/src/barretenberg/env/hardware_concurrency.cpp @@ -4,10 +4,19 @@ #include #include #include + +#ifndef NO_MULTITHREADING #include +#endif extern "C" { +#ifdef NO_MULTITHREADING +uint32_t env_hardware_concurrency() +{ + return 1; +} +#else uint32_t env_hardware_concurrency() { #ifndef __wasm__ @@ -22,4 +31,5 @@ uint32_t env_hardware_concurrency() } #endif } +#endif } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp b/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp index 18eda8ecd0c..36838486100 100644 --- a/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp @@ -251,18 +251,23 @@ template static constexpr size_t compute * @details The size of the outer tuple is equal to the number of relations. Each relation contributes an inner tuple of * univariates whose size is equal to the number of subrelations of the relation. The length of a univariate in an inner * tuple is determined by the corresponding subrelation length and the number of instances to be folded. + * @tparam optimised Enable optimised version with skipping some of the computation */ -template +template static constexpr auto create_protogalaxy_tuple_of_tuples_of_univariates() { if constexpr (Index >= std::tuple_size::value) { return std::tuple<>{}; // Return empty when reach end of the tuple } else { using UnivariateTuple = - typename std::tuple_element_t::template ProtogalaxyTupleOfUnivariatesOverSubrelations; - return std::tuple_cat(std::tuple{}, - create_protogalaxy_tuple_of_tuples_of_univariates()); + std::conditional_t:: + template OptimisedProtogalaxyTupleOfUnivariatesOverSubrelations, + typename std::tuple_element_t:: + template ProtogalaxyTupleOfUnivariatesOverSubrelations>; + return std::tuple_cat( + std::tuple{}, + create_protogalaxy_tuple_of_tuples_of_univariates()); } } @@ -343,13 +348,16 @@ concept IsUltraFlavor = IsAnyOf; template concept IsGoblinFlavor = IsAnyOf, - GoblinUltraRecursiveFlavor_>; + GoblinUltraRecursiveFlavor_, GoblinUltraRecursiveFlavor_>; template concept IsRecursiveFlavor = IsAnyOf, UltraRecursiveFlavor_, + UltraRecursiveFlavor_, GoblinUltraRecursiveFlavor_, - GoblinUltraRecursiveFlavor_>; + GoblinUltraRecursiveFlavor_ +,GoblinUltraRecursiveFlavor_>; + template concept IsGrumpkinFlavor = IsAnyOf; @@ -357,8 +365,9 @@ template concept IsFoldingFlavor = IsAnyOf, UltraRecursiveFlavor_, + UltraRecursiveFlavor_, GoblinUltraRecursiveFlavor_, - GoblinUltraRecursiveFlavor_>; + GoblinUltraRecursiveFlavor_, GoblinUltraRecursiveFlavor_>; template inline std::string flavor_get_label(Container&& container, const Element& element) { diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/merkle_hash_type.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/merkle_hash_type.hpp index 33adbd44e66..d7d3fe027bd 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/merkle_hash_type.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/merkle_hash_type.hpp @@ -1,6 +1,6 @@ #pragma once namespace bb::merkle { -// TODO(Cody) Get rid of this? +// TODO(https://github.com/AztecProtocol/barretenberg/issues/426) enum HashType { FIXED_BASE_PEDERSEN, LOOKUP_PEDERSEN }; -} // namespace bb::merkle \ No newline at end of file +} // namespace bb::merkle diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/pedersen_commitment_type.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/pedersen_commitment_type.hpp index 0976ebee71f..8e72d71d6bc 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/pedersen_commitment_type.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/pedersen_commitment_type.hpp @@ -1,6 +1,6 @@ #pragma once namespace bb::pedersen { -// TODO(Cody) Get rid of this? +// TODO(https://github.com/AztecProtocol/barretenberg/issues/426) enum CommitmentType { FIXED_BASE_PEDERSEN, LOOKUP_PEDERSEN }; -} // namespace bb::pedersen \ No newline at end of file +} // namespace bb::pedersen diff --git a/barretenberg/cpp/src/barretenberg/polynomials/univariate.hpp b/barretenberg/cpp/src/barretenberg/polynomials/univariate.hpp index aedc1353787..6471ba85b56 100644 --- a/barretenberg/cpp/src/barretenberg/polynomials/univariate.hpp +++ b/barretenberg/cpp/src/barretenberg/polynomials/univariate.hpp @@ -13,17 +13,22 @@ namespace bb { * of the data in those univariates. We do that by taking a view of those elements and then, as needed, using this to * populate new containers. */ -template class UnivariateView; +template class UnivariateView; /** * @brief A univariate polynomial represented by its values on {domain_start, domain_start + 1,..., domain_end - 1}. For * memory efficiency purposes, we store the evaluations in an array starting from 0 and make the mapping to the right * domain under the hood. + * + * @tparam skip_count Skip computing the values of elements [domain_start+1,..,domain_start+skip_count]. Used for + * optimising computation in protogalaxy. The value at [domain_start] is the value from the accumulator instance, while + * the values in [domain_start+1, ... domain_start + skip_count] in the accumulator should be zero if the original + * instances are correct. */ -template class Univariate { +template class Univariate { public: static constexpr size_t LENGTH = domain_end - domain_start; - using View = UnivariateView; + using View = UnivariateView; using value_type = Fr; // used to get the type of the elements consistently with std::array @@ -40,8 +45,27 @@ template class Univariate Univariate(Univariate&& other) noexcept = default; Univariate& operator=(const Univariate& other) = default; Univariate& operator=(Univariate&& other) noexcept = default; - // Construct constant Univariate from scalar which represents the value that all the points in the domain evaluate - // to + + /** + * @brief Convert from a version with skipped evaluations to one without skipping (with zeroes in previously skipped + * locations) + * + * @return Univariate + */ + Univariate convert() const noexcept + { + Univariate result; + result.evaluations[0] = evaluations[0]; + for (size_t i = 1; i < skip_count + 1; i++) { + result.evaluations[i] = Fr::zero(); + } + for (size_t i = skip_count + 1; i < LENGTH; i++) { + result.evaluations[i] = evaluations[i]; + } + return result; + } + // Construct constant Univariate from scalar which represents the value that all the points in the domain + // evaluate to explicit Univariate(Fr value) : evaluations{} { @@ -50,7 +74,7 @@ template class Univariate } } // Construct Univariate from UnivariateView - explicit Univariate(UnivariateView in) + explicit Univariate(UnivariateView in) : evaluations{} { for (size_t i = 0; i < in.evaluations.size(); ++i) { @@ -77,7 +101,7 @@ template class Univariate static Univariate get_random() { - auto output = Univariate(); + auto output = Univariate(); for (size_t i = 0; i != LENGTH; ++i) { output.value_at(i) = Fr::random_element(); } @@ -86,7 +110,7 @@ template class Univariate static Univariate zero() { - auto output = Univariate(); + auto output = Univariate(); for (size_t i = 0; i != LENGTH; ++i) { output.value_at(i) = Fr::zero(); } @@ -100,21 +124,25 @@ template class Univariate Univariate& operator+=(const Univariate& other) { - for (size_t i = 0; i < LENGTH; ++i) { + evaluations[0] += other.evaluations[0]; + for (size_t i = skip_count + 1; i < LENGTH; ++i) { evaluations[i] += other.evaluations[i]; } return *this; } Univariate& operator-=(const Univariate& other) { - for (size_t i = 0; i < LENGTH; ++i) { + evaluations[0] -= other.evaluations[0]; + for (size_t i = skip_count + 1; i < LENGTH; ++i) { + evaluations[i] -= other.evaluations[i]; } return *this; } Univariate& operator*=(const Univariate& other) { - for (size_t i = 0; i < LENGTH; ++i) { + evaluations[0] *= other.evaluations[0]; + for (size_t i = skip_count + 1; i < LENGTH; ++i) { evaluations[i] *= other.evaluations[i]; } return *this; @@ -135,8 +163,12 @@ template class Univariate Univariate operator-() const { Univariate res(*this); + size_t i = 0; for (auto& eval : res.evaluations) { - eval = -eval; + if (i == 0 || i >= (skip_count + 1)) { + eval = -eval; + } + i++; } return res; } @@ -151,23 +183,39 @@ template class Univariate // Operations between Univariate and scalar Univariate& operator+=(const Fr& scalar) { + size_t i = 0; for (auto& eval : evaluations) { - eval += scalar; + if (i == 0 || i >= (skip_count + 1)) { + eval += scalar; + } + i++; } return *this; } Univariate& operator-=(const Fr& scalar) { + size_t i = 0; for (auto& eval : evaluations) { - eval -= scalar; + // If skip count is zero, will be enabled on every line, otherwise don't compute for [domain_start+1,.., + // domain_start + skip_count] + if (i == 0 || i >= (skip_count + 1)) { + eval -= scalar; + } + i++; } return *this; } Univariate& operator*=(const Fr& scalar) { + size_t i = 0; for (auto& eval : evaluations) { - eval *= scalar; + // If skip count is zero, will be enabled on every line, otherwise don't compute for [domain_start+1,.., + // domain_start + skip_count] + if (i == 0 || i >= (skip_count + 1)) { + eval *= scalar; + } + i++; } return *this; } @@ -194,45 +242,48 @@ template class Univariate } // Operations between Univariate and UnivariateView - Univariate& operator+=(const UnivariateView& view) + Univariate& operator+=(const UnivariateView& view) { - for (size_t i = 0; i < LENGTH; ++i) { + evaluations[0] += view.evaluations[0]; + for (size_t i = skip_count + 1; i < LENGTH; ++i) { evaluations[i] += view.evaluations[i]; } return *this; } - Univariate& operator-=(const UnivariateView& view) + Univariate& operator-=(const UnivariateView& view) { - for (size_t i = 0; i < LENGTH; ++i) { + evaluations[0] -= view.evaluations[0]; + for (size_t i = skip_count + 1; i < LENGTH; ++i) { evaluations[i] -= view.evaluations[i]; } return *this; } - Univariate& operator*=(const UnivariateView& view) + Univariate& operator*=(const UnivariateView& view) { - for (size_t i = 0; i < LENGTH; ++i) { + evaluations[0] *= view.evaluations[0]; + for (size_t i = skip_count + 1; i < LENGTH; ++i) { evaluations[i] *= view.evaluations[i]; } return *this; } - Univariate operator+(const UnivariateView& view) const + Univariate operator+(const UnivariateView& view) const { Univariate res(*this); res += view; return res; } - Univariate operator-(const UnivariateView& view) const + Univariate operator-(const UnivariateView& view) const { Univariate res(*this); res -= view; return res; } - Univariate operator*(const UnivariateView& view) const + Univariate operator*(const UnivariateView& view) const { Univariate res(*this); res *= view; @@ -256,39 +307,42 @@ template class Univariate } /** - * @brief Given a univariate f represented by {f(domain_start), ..., f(domain_end - 1)}, compute the evaluations - * {f(domain_end),..., f(extended_domain_end -1)} and return the Univariate represented by {f(domain_start),..., - * f(extended_domain_end -1)} + * @brief Given a univariate f represented by {f(domain_start), ..., f(domain_end - 1)}, compute the + * evaluations {f(domain_end),..., f(extended_domain_end -1)} and return the Univariate represented by + * {f(domain_start),..., f(extended_domain_end -1)} * - * @details Write v_i = f(x_i) on a the domain {x_{domain_start}, ..., x_{domain_end-1}}. To efficiently compute the - * needed values of f, we use the barycentric formula + * @details Write v_i = f(x_i) on a the domain {x_{domain_start}, ..., x_{domain_end-1}}. To efficiently + * compute the needed values of f, we use the barycentric formula * - f(x) = B(x) Σ_{i=domain_start}^{domain_end-1} v_i / (d_i*(x-x_i)) * where * - B(x) = Π_{i=domain_start}^{domain_end-1} (x-x_i) - * - d_i = Π_{j ∈ {domain_start, ..., domain_end-1}, j≠i} (x_i-x_j) for i ∈ {domain_start, ..., domain_end-1} + * - d_i = Π_{j ∈ {domain_start, ..., domain_end-1}, j≠i} (x_i-x_j) for i ∈ {domain_start, ..., + * domain_end-1} * - * When the domain size is two, extending f = v0(1-X) + v1X to a new value involves just one addition and a - * subtraction: setting Δ = v1-v0, the values of f(X) are f(0)=v0, f(1)= v0 + Δ, v2 = f(1) + Δ, v3 = f(2) + Δ... + * When the domain size is two, extending f = v0(1-X) + v1X to a new value involves just one addition + * and a subtraction: setting Δ = v1-v0, the values of f(X) are f(0)=v0, f(1)= v0 + Δ, v2 = f(1) + Δ, v3 + * = f(2) + Δ... * */ - template Univariate extend_to() const + template + Univariate extend_to() const { const size_t EXTENDED_LENGTH = EXTENDED_DOMAIN_END - domain_start; using Data = BarycentricData; static_assert(EXTENDED_LENGTH >= LENGTH); - Univariate result; + Univariate result; std::copy(evaluations.begin(), evaluations.end(), result.evaluations.begin()); static constexpr Fr inverse_two = Fr(2).invert(); + static_assert(NUM_SKIPPED_INDICES < LENGTH); if constexpr (LENGTH == 2) { Fr delta = value_at(1) - value_at(0); static_assert(EXTENDED_LENGTH != 0); for (size_t idx = domain_end - 1; idx < EXTENDED_DOMAIN_END - 1; idx++) { result.value_at(idx + 1) = result.value_at(idx) + delta; } - return result; } else if constexpr (LENGTH == 3) { // Based off https://hackmd.io/@aztec-network/SyR45cmOq?type=view // The technique used here is the same as the length == 3 case below. @@ -304,7 +358,6 @@ template class Univariate result.value_at(idx + 1) = result.value_at(idx) + extra; extra += a2; } - return result; } else if constexpr (LENGTH == 4) { static constexpr Fr inverse_six = Fr(6).invert(); // computed at compile time for efficiency @@ -315,8 +368,8 @@ template class Univariate // a*1 + b*1 + c*1 + d = f(1) // a*2^3 + b*2^2 + c*2 + d = f(2) // a*3^3 + b*3^2 + c*3 + d = f(3) - // These equations can be rewritten as a matrix equation M * [a, b, c, d] = [f(0), f(1), f(2), f(3)], where - // M is: + // These equations can be rewritten as a matrix equation M * [a, b, c, d] = [f(0), f(1), f(2), + // f(3)], where M is: // 0, 0, 0, 1 // 1, 1, 1, 1 // 2^3, 2^2, 2, 1 @@ -326,9 +379,9 @@ template class Univariate // 1, -5/2, 2, -1/2 // -11/6, 3, -3/2, 1/3 // 1, 0, 0, 0 - // To compute these values, we can multiply everything by 6 and multiply by inverse_six at the end for each - // coefficient The resulting computation here does 18 field adds, 6 subtracts, 3 muls to compute a, b, c, - // and d. + // To compute these values, we can multiply everything by 6 and multiply by inverse_six at the + // end for each coefficient The resulting computation here does 18 field adds, 6 subtracts, 3 + // muls to compute a, b, c, and d. Fr zero_times_3 = value_at(0) + value_at(0) + value_at(0); Fr zero_times_6 = zero_times_3 + zero_times_3; Fr zero_times_12 = zero_times_6 + zero_times_6; @@ -368,7 +421,6 @@ template class Univariate linear_term += three_a_plus_two_b; } - return result; } else { for (size_t k = domain_end; k != EXTENDED_DOMAIN_END; ++k) { result.value_at(k) = 0; @@ -381,8 +433,8 @@ template class Univariate // scale the sum by the the value of of B(x) result.value_at(k) *= Data::full_numerator_values[k]; } - return result; } + return result; } /** @@ -399,8 +451,8 @@ template class Univariate full_numerator_value *= u - i; } - // build set of domain size-many denominator inverses 1/(d_i*(x_k - x_j)). will multiply against each of - // these (rather than to divide by something) for each barycentric evaluation + // build set of domain size-many denominator inverses 1/(d_i*(x_k - x_j)). will multiply against + // each of these (rather than to divide by something) for each barycentric evaluation std::array denominator_inverses; for (size_t i = 0; i != LENGTH; ++i) { Fr inv = Data::lagrange_denominators[i]; @@ -443,7 +495,7 @@ inline void write(B& it, Univariate const& univari write(it, univariate.evaluations); } -template class UnivariateView { +template class UnivariateView { public: static constexpr size_t LENGTH = domain_end - domain_start; std::span evaluations; @@ -453,77 +505,84 @@ template class Univariate const Fr& value_at(size_t i) const { return evaluations[i]; }; template - explicit UnivariateView(const Univariate& univariate_in) + explicit UnivariateView(const Univariate& univariate_in) : evaluations(std::span(univariate_in.evaluations.data(), LENGTH)){}; - Univariate operator+(const UnivariateView& other) const + Univariate operator+(const UnivariateView& other) const { - Univariate res(*this); + Univariate res(*this); res += other; return res; } - Univariate operator-(const UnivariateView& other) const + Univariate operator-(const UnivariateView& other) const { - Univariate res(*this); + Univariate res(*this); res -= other; return res; } - Univariate operator-() const + Univariate operator-() const { - Univariate res(*this); + Univariate res(*this); + size_t i = 0; for (auto& eval : res.evaluations) { - eval = -eval; + if (i == 0 || i >= (skip_count + 1)) { + eval = -eval; + } + i++; } return res; } - Univariate operator*(const UnivariateView& other) const + Univariate operator*(const UnivariateView& other) const { - Univariate res(*this); + Univariate res(*this); res *= other; return res; } - Univariate operator*(const Univariate& other) const + Univariate operator*( + const Univariate& other) const { - Univariate res(*this); + Univariate res(*this); res *= other; return res; } - Univariate operator+(const Univariate& other) const + Univariate operator+( + const Univariate& other) const { - Univariate res(*this); + Univariate res(*this); res += other; return res; } - Univariate operator+(const Fr& other) const + Univariate operator+(const Fr& other) const { - Univariate res(*this); + Univariate res(*this); res += other; return res; } - Univariate operator-(const Fr& other) const + Univariate operator-(const Fr& other) const { - Univariate res(*this); + Univariate res(*this); res -= other; return res; } - Univariate operator*(const Fr& other) const + Univariate operator*(const Fr& other) const { - Univariate res(*this); + Univariate res(*this); res *= other; return res; } - Univariate operator-(const Univariate& other) const + Univariate operator-( + const Univariate& other) const { - Univariate res(*this); + Univariate res(*this); res -= other; return res; } @@ -546,8 +605,8 @@ template class Univariate }; /** - * @brief Create a sub-array of `elements` at the indices given in the template pack `Is`, converting them to the new - * type T. + * @brief Create a sub-array of `elements` at the indices given in the template pack `Is`, converting them + * to the new type T. * * @tparam T type to convert to * @tparam U type to convert from @@ -555,8 +614,8 @@ template class Univariate * @tparam Is list of indices we want in the returned array. When the second argument is called with * `std::make_index_sequence`, these will be `0, 1, ..., N-1`. * @param elements array to convert from - * @return std::array result array s.t. result[i] = T(elements[Is[i]]). By default, Is[i] = i when - * called with `std::make_index_sequence`. + * @return std::array result array s.t. result[i] = T(elements[Is[i]]). By default, Is[i] + * = i when called with `std::make_index_sequence`. */ template std::array array_to_array_aux(const std::array& elements, std::index_sequence) @@ -568,11 +627,12 @@ std::array array_to_array_aux(const std::array& elements * @brief Given an std::array, returns an std::array, by calling the (explicit) constructor T(U). * * @details https://stackoverflow.com/a/32175958 - * The main use case is to convert an array of `Univariate` into `UnivariateView`. The main use case would be to let - * Sumcheck decide the required degree of the relation evaluation, rather than hardcoding it inside the relation. The - * `_aux` version could also be used to create an array of only the polynomials required by the relation, and it could - * help us implement the optimization where we extend each edge only up to the maximum degree that is required over all - * relations (for example, `L_LAST` only needs degree 3). + * The main use case is to convert an array of `Univariate` into `UnivariateView`. The main use case would + * be to let Sumcheck decide the required degree of the relation evaluation, rather than hardcoding it + * inside the relation. The + * `_aux` version could also be used to create an array of only the polynomials required by the relation, + * and it could help us implement the optimization where we extend each edge only up to the maximum degree + * that is required over all relations (for example, `L_LAST` only needs degree 3). * * @tparam T Output type * @tparam U Input type (deduced from `elements`) diff --git a/barretenberg/cpp/src/barretenberg/protogalaxy/combiner.test.cpp b/barretenberg/cpp/src/barretenberg/protogalaxy/combiner.test.cpp index 72a6fa53233..4ff7f81cb51 100644 --- a/barretenberg/cpp/src/barretenberg/protogalaxy/combiner.test.cpp +++ b/barretenberg/cpp/src/barretenberg/protogalaxy/combiner.test.cpp @@ -44,6 +44,12 @@ TEST(Protogalaxy, CombinerOn2Instances) auto prover_polynomials = get_sequential_prover_polynomials( /*log_circuit_size=*/1, idx * 128); restrict_to_standard_arithmetic_relation(prover_polynomials); + // This ensures that the combiner accumulator for second instance = 0 + // The value is computed by generating the python script values, computing the resulting accumulator and + // taking the value at index 1 + if (idx == NUM_INSTANCES - 1) { + prover_polynomials.q_c[0] -= 13644570; + } instance->proving_key.polynomials = std::move(prover_polynomials); instance->proving_key.circuit_size = 2; instance_data[idx] = instance; @@ -52,22 +58,22 @@ TEST(Protogalaxy, CombinerOn2Instances) ProverInstances instances{ instance_data }; instances.alphas.fill(bb::Univariate(FF(0))); // focus on the arithmetic relation only auto pow_polynomial = PowPolynomial(std::vector{ 2 }); - auto result = prover.compute_combiner(instances, pow_polynomial); - auto expected_result = Univariate(std::array{ - 87706, - 13644570, - 76451738, - 226257946, - static_cast(500811930), - static_cast(937862426), - static_cast(1575158170), - static_cast(2450447898), - static_cast(3601480346), - static_cast(5066004250), - static_cast(6881768346), - static_cast(9086521370), - }); + auto result = prover.compute_combiner(instances, pow_polynomial); + auto optimised_result = prover.compute_combiner(instances, pow_polynomial); + auto expected_result = Univariate(std::array{ 87706, + 0, + 0x02ee2966, + 0x0b0bd2cc, + 0x00001a98fc32, + 0x000033d5a598, + 0x00005901cefe, + 0x00008c5d7864, + 0x0000d028a1ca, + 0x000126a34b30UL, + 0x0001920d7496UL, + 0x000214a71dfcUL }); EXPECT_EQ(result, expected_result); + EXPECT_EQ(optimised_result, expected_result); } else { std::vector> instance_data(NUM_INSTANCES); ProtoGalaxyProver prover; @@ -130,11 +136,13 @@ TEST(Protogalaxy, CombinerOn2Instances) 0 0 0 0 0 0 0 0 0 6 18 36 60 90 */ auto pow_polynomial = PowPolynomial(std::vector{ 2 }); - auto result = prover.compute_combiner(instances, pow_polynomial); + auto result = prover.compute_combiner(instances, pow_polynomial); + auto optimised_result = prover.compute_combiner(instances, pow_polynomial); auto expected_result = Univariate(std::array{ 0, 0, 12, 36, 72, 120, 180, 252, 336, 432, 540, 660 }); EXPECT_EQ(result, expected_result); + EXPECT_EQ(optimised_result, expected_result); } }; run_test(true); @@ -181,11 +189,13 @@ TEST(Protogalaxy, CombinerOn4Instances) zero_all_selectors(instances[3]->proving_key.polynomials); auto pow_polynomial = PowPolynomial(std::vector{ 2 }); - auto result = prover.compute_combiner(instances, pow_polynomial); + auto result = prover.compute_combiner(instances, pow_polynomial); + auto optimised_result = prover.compute_combiner(instances, pow_polynomial); std::array zeroes; std::fill(zeroes.begin(), zeroes.end(), 0); auto expected_result = Univariate(zeroes); EXPECT_EQ(result, expected_result); + EXPECT_EQ(optimised_result, expected_result); }; run_test(); }; diff --git a/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy.test.cpp b/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy.test.cpp index 0b51c91f57b..3148c54cd40 100644 --- a/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy.test.cpp +++ b/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy.test.cpp @@ -279,6 +279,11 @@ template class ProtoGalaxyTests : public testing::Test { bb::Univariate expected_eta{ { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21 } }; EXPECT_EQ(instances.relation_parameters.eta, expected_eta); + // Optimised relation parameters are the same, we just don't compute any values for non-used indices when + // deriving values from them + for (size_t i = 0; i < 11; i++) { + EXPECT_EQ(instances.optimised_relation_parameters.eta.evaluations[i], expected_eta.evaluations[i]); + } } /** diff --git a/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_prover.hpp b/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_prover.hpp index d9cffcca9c7..c03af2e5333 100644 --- a/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_prover.hpp +++ b/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_prover.hpp @@ -49,15 +49,26 @@ template class ProtoGalaxyProver_ { // The length of ExtendedUnivariate is the largest length (==max_relation_degree + 1) of a univariate polynomial // obtained by composing a relation with folded instance + relation parameters . using ExtendedUnivariate = Univariate; + // Same as ExtendedUnivariate, but uses optimised univariates which skip redundant computation in optimistic cases + // (when we know that the evaluation of all relations is 0 on a particular index, for example) + using OptimisedExtendedUnivariate = + Univariate; // Represents the total length of the combiner univariate, obtained by combining the already folded relations with // the folded relation batching challenge. using ExtendedUnivariateWithRandomization = Univariate; using ExtendedUnivariates = typename Flavor::template ProverUnivariates; + using OptimisedExtendedUnivariates = + typename Flavor::template OptimisedProverUnivariates; using TupleOfTuplesOfUnivariates = typename Flavor::template ProtogalaxyTupleOfTuplesOfUnivariates; + using OptimisedTupleOfTuplesOfUnivariates = + typename Flavor::template OptimisedProtogalaxyTupleOfTuplesOfUnivariates; using RelationEvaluations = typename Flavor::TupleOfArraysOfValues; static constexpr size_t NUM_SUBRELATIONS = ProverInstances::NUM_SUBRELATIONS; @@ -209,14 +220,20 @@ template class ProtoGalaxyProver_ { auto prev_level_width = prev_level_coeffs.size(); // we need degree + 1 terms to represent the intermediate polynomials std::vector> level_coeffs(prev_level_width >> 1, std::vector(degree + 1, 0)); - for (size_t node = 0; node < prev_level_width; node += 2) { - auto parent = node >> 1; - std::copy(prev_level_coeffs[node].begin(), prev_level_coeffs[node].end(), level_coeffs[parent].begin()); - for (size_t d = 0; d < degree; d++) { - level_coeffs[parent][d] += prev_level_coeffs[node + 1][d] * betas[level]; - level_coeffs[parent][d + 1] += prev_level_coeffs[node + 1][d] * deltas[level]; - } - } + run_loop_in_parallel( + prev_level_width >> 1, + [&](size_t start, size_t end) { + for (size_t node = start << 1; node < end << 1; node += 2) { + auto parent = node >> 1; + std::copy( + prev_level_coeffs[node].begin(), prev_level_coeffs[node].end(), level_coeffs[parent].begin()); + for (size_t d = 0; d < degree; d++) { + level_coeffs[parent][d] += prev_level_coeffs[node + 1][d] * betas[level]; + level_coeffs[parent][d + 1] += prev_level_coeffs[node + 1][d] * deltas[level]; + } + } + }, + /*no_multhreading_if_less_or_equal=*/8); return construct_coefficients_tree(betas, deltas, level_coeffs, level + 1); } @@ -236,11 +253,16 @@ template class ProtoGalaxyProver_ { { auto width = full_honk_evaluations.size(); std::vector> first_level_coeffs(width >> 1, std::vector(2, 0)); - for (size_t node = 0; node < width; node += 2) { - auto parent = node >> 1; - first_level_coeffs[parent][0] = full_honk_evaluations[node] + full_honk_evaluations[node + 1] * betas[0]; - first_level_coeffs[parent][1] = full_honk_evaluations[node + 1] * deltas[0]; - } + run_loop_in_parallel(width >> 1, [&](size_t start, size_t end) { + // Run loop in parallel can divide the domain in such way that the indices are odd, which we can't tolerate + // here, so first we divide the width by two, enable parallelism and then reconstruct even start and end + for (size_t node = start << 1; node < end << 1; node += 2) { + auto parent = node >> 1; + first_level_coeffs[parent][0] = + full_honk_evaluations[node] + full_honk_evaluations[node + 1] * betas[0]; + first_level_coeffs[parent][1] = full_honk_evaluations[node + 1] * deltas[0]; + } + }); return construct_coefficients_tree(betas, deltas, first_level_coeffs); } @@ -262,26 +284,38 @@ template class ProtoGalaxyProver_ { return Polynomial(coeffs); } + OptimisedTupleOfTuplesOfUnivariates optimised_univariate_accumulators; TupleOfTuplesOfUnivariates univariate_accumulators; /** * @brief Prepare a univariate polynomial for relation execution in one step of the main loop in folded instance * construction. - * @details For a fixed prover polynomial index, extract that polynomial from each instance in Instances. From each - * polynomial, extract the value at row_idx. Use these values to create a univariate polynomial, and then extend - * (i.e., compute additional evaluations at adjacent domain values) as needed. + * @details For a fixed prover polynomial index, extract that polynomial from each instance in Instances. From + *each polynomial, extract the value at row_idx. Use these values to create a univariate polynomial, and then + *extend (i.e., compute additional evaluations at adjacent domain values) as needed. * @todo TODO(https://github.com/AztecProtocol/barretenberg/issues/751) Optimize memory + * + * */ - void extend_univariates(ExtendedUnivariates& extended_univariates, - const ProverInstances& instances, - const size_t row_idx) + + template + void extend_univariates( + std::conditional_t& extended_univariates, + const ProverInstances& instances, + const size_t row_idx) { - auto base_univariates = instances.row_to_univariates(row_idx); + auto base_univariates = instances.template row_to_univariates(row_idx); for (auto [extended_univariate, base_univariate] : zip_view(extended_univariates.get_all(), base_univariates)) { - extended_univariate = base_univariate.template extend_to(); + extended_univariate = base_univariate.template extend_to(); } } + /** + * @brief Add the value of each relation over univariates to an appropriate accumulator + * + * @tparam Parameters relation parameters type + * @tparam relation_idx The index of the relation + */ template void accumulate_relation_univariates(TupleOfTuplesOfUnivariates& univariate_accumulators, const ExtendedUnivariates& extended_univariates, @@ -294,39 +328,73 @@ template class ProtoGalaxyProver_ { // Repeat for the next relation. if constexpr (relation_idx + 1 < Flavor::NUM_RELATIONS) { - accumulate_relation_univariates( - univariate_accumulators, extended_univariates, relation_parameters, scaling_factor); + accumulate_relation_univariates< + + Parameters, + relation_idx + 1>(univariate_accumulators, extended_univariates, relation_parameters, scaling_factor); } } /** - * @brief Compute the combiner polynomial $G$ in the Protogalaxy paper. + * @brief Add the value of each relation over univariates to an appropriate accumulator with index skipping + * optimisation + * + * @tparam Parameters relation parameters type + * @tparam relation_idx The index of the relation + */ + template + void accumulate_relation_univariates(OptimisedTupleOfTuplesOfUnivariates& univariate_accumulators, + const OptimisedExtendedUnivariates& extended_univariates, + const Parameters& relation_parameters, + const FF& scaling_factor) + { + using Relation = std::tuple_element_t; + Relation::accumulate( + std::get(univariate_accumulators), extended_univariates, relation_parameters, scaling_factor); + + // Repeat for the next relation. + if constexpr (relation_idx + 1 < Flavor::NUM_RELATIONS) { + accumulate_relation_univariates< + + Parameters, + relation_idx + 1>(univariate_accumulators, extended_univariates, relation_parameters, scaling_factor); + } + } + /** + * @brief Compute the combiner polynomial $G$ in the Protogalaxy paper * */ + template = true> ExtendedUnivariateWithRandomization compute_combiner(const ProverInstances& instances, PowPolynomial& pow_betas) { - BB_OP_COUNT_TIME(); size_t common_instance_size = instances[0]->proving_key.circuit_size; pow_betas.compute_values(); // Determine number of threads for multithreading. // Note: Multithreading is "on" for every round but we reduce the number of threads from the max available based - // on a specified minimum number of iterations per thread. This eventually leads to the use of a single thread. - // For now we use a power of 2 number of threads simply to ensure the round size is evenly divided. + // on a specified minimum number of iterations per thread. This eventually leads to the use of a + // single thread. For now we use a power of 2 number of threads simply to ensure the round size is evenly + // divided. size_t max_num_threads = get_num_cpus_pow2(); // number of available threads (power of 2) size_t min_iterations_per_thread = 1 << 6; // min number of iterations for which we'll spin up a unique thread size_t desired_num_threads = common_instance_size / min_iterations_per_thread; size_t num_threads = std::min(desired_num_threads, max_num_threads); // fewer than max if justified num_threads = num_threads > 0 ? num_threads : 1; // ensure num threads is >= 1 size_t iterations_per_thread = common_instance_size / num_threads; // actual iterations per thread + + // Univariates are optimised for usual PG, but we need the unoptimised version for tests (it's a version that + // doesn't skip computation), so we need to define types depending on the template instantiation + using ThreadAccumulators = TupleOfTuplesOfUnivariates; + using ExtendedUnivatiatesType = ExtendedUnivariates; + // Construct univariate accumulator containers; one per thread - std::vector thread_univariate_accumulators(num_threads); + std::vector thread_univariate_accumulators(num_threads); for (auto& accum : thread_univariate_accumulators) { // just normal relation lengths Utils::zero_univariates(accum); } // Construct extended univariates containers; one per thread - std::vector extended_univariates; + std::vector extended_univariates; extended_univariates.resize(num_threads); // Accumulate the contribution from each sub-relation @@ -335,14 +403,15 @@ template class ProtoGalaxyProver_ { size_t end = (thread_idx + 1) * iterations_per_thread; for (size_t idx = start; idx < end; idx++) { - // No need to initialise extended_univariates to 0, it's assigned to + extend_univariates(extended_univariates[thread_idx], instances, idx); FF pow_challenge = pow_betas[idx]; - // Accumulate the i-th row's univariate contribution. Note that the relation parameters passed to this - // function have already been folded. Moreover, linear-dependent relations that act over the entire - // execution trace rather than on rows, will not be multiplied by the pow challenge. + // Accumulate the i-th row's univariate contribution. Note that the relation parameters passed to + // this function have already been folded. Moreover, linear-dependent relations that act over the + // entire execution trace rather than on rows, will not be multiplied by the pow challenge. + accumulate_relation_univariates( thread_univariate_accumulators[thread_idx], extended_univariates[thread_idx], @@ -350,19 +419,115 @@ template class ProtoGalaxyProver_ { pow_challenge); } }); - + Utils::zero_univariates(univariate_accumulators); // Accumulate the per-thread univariate accumulators into a single set of accumulators for (auto& accumulators : thread_univariate_accumulators) { Utils::add_nested_tuples(univariate_accumulators, accumulators); } - // Batch the univariate contributions from each sub-relation to obtain the round univariate + return batch_over_relations(univariate_accumulators, instances.alphas); } + /** + * @brief Compute the combiner polynomial $G$ in the Protogalaxy paper using indice skippping optimisation + * + * @todo (https://github.com/AztecProtocol/barretenberg/issues/968) Make combiner tests better + * + */ + template = true> + ExtendedUnivariateWithRandomization compute_combiner(const ProverInstances& instances, PowPolynomial& pow_betas) + { + BB_OP_COUNT_TIME(); + size_t common_instance_size = instances[0]->proving_key.circuit_size; + pow_betas.compute_values(); + // Determine number of threads for multithreading. + // Note: Multithreading is "on" for every round but we reduce the number of threads from the max available based + // on a specified minimum number of iterations per thread. This eventually leads to the use of a + // single thread. For now we use a power of 2 number of threads simply to ensure the round size is evenly + // divided. + size_t max_num_threads = get_num_cpus_pow2(); // number of available threads (power of 2) + size_t min_iterations_per_thread = 1 << 6; // min number of iterations for which we'll spin up a unique thread + size_t desired_num_threads = common_instance_size / min_iterations_per_thread; + size_t num_threads = std::min(desired_num_threads, max_num_threads); // fewer than max if justified + num_threads = num_threads > 0 ? num_threads : 1; // ensure num threads is >= 1 + size_t iterations_per_thread = common_instance_size / num_threads; // actual iterations per thread + + // Univariates are optimised for usual PG, but we need the unoptimised version for tests (it's a version that + // doesn't skip computation), so we need to define types depending on the template instantiation + using ThreadAccumulators = OptimisedTupleOfTuplesOfUnivariates; + using ExtendedUnivatiatesType = OptimisedExtendedUnivariates; + + // Construct univariate accumulator containers; one per thread + std::vector thread_univariate_accumulators(num_threads); + for (auto& accum : thread_univariate_accumulators) { + // just normal relation lengths + Utils::zero_univariates(accum); + } + + // Construct extended univariates containers; one per thread + std::vector extended_univariates; + extended_univariates.resize(num_threads); + + // Accumulate the contribution from each sub-relation + parallel_for(num_threads, [&](size_t thread_idx) { + size_t start = thread_idx * iterations_per_thread; + size_t end = (thread_idx + 1) * iterations_per_thread; + + for (size_t idx = start; idx < end; idx++) { + // No need to initialise extended_univariates to 0, it's assigned to + // Instantiate univariates with skipping to ignore computation in those indices (they are still + // available for skipping relations, but all derived univariate will ignore those evaluations) + extend_univariates( + extended_univariates[thread_idx], instances, idx); + + FF pow_challenge = pow_betas[idx]; + + // Accumulate the i-th row's univariate contribution. Note that the relation parameters passed to + // this function have already been folded. Moreover, linear-dependent relations that act over the + // entire execution trace rather than on rows, will not be multiplied by the pow challenge. + accumulate_relation_univariates( + thread_univariate_accumulators[thread_idx], + extended_univariates[thread_idx], + instances.optimised_relation_parameters, // these parameters have already been folded + pow_challenge); + } + }); + Utils::zero_univariates(optimised_univariate_accumulators); + // Accumulate the per-thread univariate accumulators into a single set of accumulators + for (auto& accumulators : thread_univariate_accumulators) { + Utils::add_nested_tuples(optimised_univariate_accumulators, accumulators); + } + + // Convert from optimised version to non-optimised + deoptimise_univariates(optimised_univariate_accumulators, univariate_accumulators); + // Batch the univariate contributions from each sub-relation to obtain the round univariate + return batch_over_relations(univariate_accumulators, instances.alphas); + } + + /** + * @brief Convert univariates from optimised form to regular + * + * @details We need to convert before we batch relations, since optimised versions don't have enough information to + * extend the univariates to maximum length + * + * @param optimised_univariate_accumulators + * @param new_univariate_accumulators + */ + static void deoptimise_univariates(const OptimisedTupleOfTuplesOfUnivariates& optimised_univariate_accumulators, + TupleOfTuplesOfUnivariates& new_univariate_accumulators + + ) + { + auto deoptimise = [&](auto& element) { + auto& optimised_element = std::get(std::get(optimised_univariate_accumulators)); + element = optimised_element.convert(); + }; + + Utils::template apply_to_tuple_of_tuples<0, 0>(new_univariate_accumulators, deoptimise); + } static ExtendedUnivariateWithRandomization batch_over_relations(TupleOfTuplesOfUnivariates& univariate_accumulators, const CombinedRelationSeparator& alpha) { - // First relation does not get multiplied by a batching challenge auto result = std::get<0>(std::get<0>(univariate_accumulators)) .template extend_to(); @@ -432,7 +597,8 @@ template class ProtoGalaxyProver_ { { size_t param_idx = 0; auto to_fold = instances.relation_parameters.get_to_fold(); - for (auto& folded_parameter : to_fold) { + auto to_fold_optimised = instances.optimised_relation_parameters.get_to_fold(); + for (auto [folded_parameter, optimised_folded_parameter] : zip_view(to_fold, to_fold_optimised)) { Univariate tmp(0); size_t instance_idx = 0; for (auto& instance : instances) { @@ -440,6 +606,8 @@ template class ProtoGalaxyProver_ { instance_idx++; } folded_parameter = tmp.template extend_to(); + optimised_folded_parameter = + tmp.template extend_to(); param_idx++; } } diff --git a/barretenberg/cpp/src/barretenberg/relations/nested_containers.hpp b/barretenberg/cpp/src/barretenberg/relations/nested_containers.hpp index 46f2d246303..36a522eb161 100644 --- a/barretenberg/cpp/src/barretenberg/relations/nested_containers.hpp +++ b/barretenberg/cpp/src/barretenberg/relations/nested_containers.hpp @@ -10,30 +10,42 @@ namespace bb { * * @details Credit: https://stackoverflow.com/a/60440611 */ -template