diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index ada29474be7..fbe5bd1bf59 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -41,7 +41,7 @@ jobs: - name: Setup go uses: actions/setup-go@v3 with: - go-version: "1.19" + go-version: "1.20" cache: true - name: Build relic diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 11d402f8f51..94120bdf62c 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -105,6 +105,7 @@ jobs: - name: Build/Push ${{ matrix.role }} images env: IMAGE_TAG: ${{ inputs.docker_tag }} + GITHUB_CREDS: "machine github.com login ${{ secrets.REPO_SYNC_USER }} password ${{ secrets.REPO_SYNC }}" run: | make docker-build-${{ matrix.role }} docker-push-${{ matrix.role }} @@ -112,5 +113,6 @@ jobs: if: ${{ inputs.include_without_netgo }} env: IMAGE_TAG: ${{ inputs.docker_tag }} + GITHUB_CREDS: "machine github.com login ${{ secrets.REPO_SYNC_USER }} password ${{ secrets.REPO_SYNC }}" run: | make docker-build-${{ matrix.role }}-without-netgo docker-push-${{ matrix.role }}-without-netgo diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index eb28e840078..9079fb06a98 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -14,7 +14,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: '1.19' + go-version: "1.20" - name: Checkout repo uses: actions/checkout@v2 - name: Build relic diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0deec40adf..e41e747e0e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,9 @@ on: - 'v[0-9]+.[0-9]+' env: - GO_VERSION: 1.19 + GO_VERSION: "1.20" -concurrency: +concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true @@ -47,7 +47,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.49 + version: v1.51 args: -v --build-tags relic working-directory: ${{ matrix.dir }} # https://github.com/golangci/golangci-lint-action/issues/244 @@ -66,8 +66,8 @@ jobs: cache: true - name: Run tidy run: make tidy - - name: Emulator no relic check - run: make emulator-norelic-check + - name: code sanity check + run: make code-sanity-check shell-check: name: ShellCheck @@ -149,7 +149,7 @@ jobs: make1: install-tools make2: test retries: 3 - race: 1 + race: 0 - name: integration make1: install-tools make2: test @@ -198,10 +198,14 @@ jobs: - make -C integration mvp-tests - make -C integration network-tests - make -C integration verification-tests + - make -C integration upgrades-tests runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v3 + with: + # all tags are needed for integration tests + fetch-depth: 0 - name: Setup Go uses: actions/setup-go@v3 with: @@ -242,7 +246,7 @@ jobs: - name: Set up Localnet run: bash -c 'cd integration/localnet/ && make -e OBSERVER=2 bootstrap && make start-flow' - name: Ensure Observer is started - run: docker ps -f name=localnet_observer_1_1 | grep localnet_observer + run: docker ps -f name=localnet-observer_1-1 | grep localnet-observer - name: Get Client Version ensuring the client is provisioned run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer version - name: Wait for a default waiting period until a clean state diff --git a/.github/workflows/flaky-test-debug.yml b/.github/workflows/flaky-test-debug.yml index 3a5b47e2c2f..8058a656f29 100644 --- a/.github/workflows/flaky-test-debug.yml +++ b/.github/workflows/flaky-test-debug.yml @@ -5,7 +5,7 @@ on: branches: - '**/*flaky-test-debug*' env: - GO_VERSION: 1.19 + GO_VERSION: "1.20" #concurrency: # group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} @@ -36,7 +36,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.49 + version: v1.51 args: -v --build-tags relic working-directory: ${{ matrix.dir }} # https://github.com/golangci/golangci-lint-action/issues/244 diff --git a/.github/workflows/semver-tags.yaml b/.github/workflows/semver-tags.yaml new file mode 100644 index 00000000000..41ebe625744 --- /dev/null +++ b/.github/workflows/semver-tags.yaml @@ -0,0 +1,22 @@ +name: Verify Tag + +on: + push: + tags: + - '*' + +jobs: + SemVer-Check: + runs-on: ubuntu-latest + steps: + - name: Check if tag is SemVer compliant + # the tag should be in semver format, but can optionally be prepended by "any_text_with_slashes/" and "v" + # valid examples crypto/v0.24.5-fvm, tools/flaky_test_monitor/v0.23.5, v0.23.5, 0.23.5-fvm + run: | + TAG_NAME=${GITHUB_REF#refs/tags/} + if [[ "${TAG_NAME}" =~ ^(.+\/)*v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$ ]]; then + echo "Tag $TAG_NAME is SemVer compliant" + else + echo "Tag $TAG_NAME is not SemVer compliant" + exit 1 + fi diff --git a/.github/workflows/test-monitor-flaky.yml b/.github/workflows/test-monitor-flaky.yml index fcf215b734e..442d71c3e07 100644 --- a/.github/workflows/test-monitor-flaky.yml +++ b/.github/workflows/test-monitor-flaky.yml @@ -13,7 +13,7 @@ on: env: BIGQUERY_DATASET: production_src_flow_test_metrics BIGQUERY_TABLE: test_results - GO_VERSION: 1.19 + GO_VERSION: "1.20" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/test-monitor-regular-skipped.yml b/.github/workflows/test-monitor-regular-skipped.yml index 74736a00431..d9f696ab87c 100644 --- a/.github/workflows/test-monitor-regular-skipped.yml +++ b/.github/workflows/test-monitor-regular-skipped.yml @@ -15,7 +15,7 @@ env: BIGQUERY_DATASET: production_src_flow_test_metrics BIGQUERY_TABLE: skipped_tests BIGQUERY_TABLE2: test_results - GO_VERSION: 1.19 + GO_VERSION: "1.20" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index 2e297adb6ff..9f228f215ba 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: '1.19' + go-version: "1.20" - name: Set up Google Cloud SDK uses: google-github-actions/setup-gcloud@v1 with: diff --git a/.gitignore b/.gitignore index 472cc944ee4..1be2e18a99f 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,7 @@ go.work.sum # Ledger checkpoint status files **/checkpoint_status.json +**/export_report.json # Local testing result files tps-results*.json diff --git a/CODEOWNERS b/CODEOWNERS index 84e68154df7..0d8beae649c 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -23,11 +23,11 @@ /module/chunking/** @ramtinms /integration/tests/verification @ramtinms @yhassanzadeh13 -# Ledger Stream +# Ledger Stream /ledger/** @ramtinms @AlexHentschel -# FVM Stream -/fvm/** @ramtinms @janezpodhostnik @pattyshack +# FVM Stream +/fvm/** @ramtinms @janezpodhostnik # Networking Stream /network/** @yhassanzadeh13 @@ -45,11 +45,7 @@ /tools/test_monitor/** @gomisha # Performance Stream -/integration/benchmark/** @SaveTheRbtz @gomisha -/integration/localnet/** @SaveTheRbtz -/module/profiler/** @SaveTheRbtz @pattyshack -/module/trace/** @SaveTheRbtz @pattyshack -/module/tracer.go @SaveTheRbtz @pattyshack +/integration/benchmark/** @gomisha # Execution Sync /module/executiondatasync/** @peterargue diff --git a/CodingConventions.md b/CodingConventions.md index c8915e0b7b6..8fcd1545f78 100644 --- a/CodingConventions.md +++ b/CodingConventions.md @@ -91,7 +91,7 @@ happy path is either Benign failure cases are represented as typed sentinel errors ([basic errors](https://pkg.go.dev/errors#New) and [higher-level errors](https://dev.to/tigorlazuardi/go-creating-custom-error-wrapper-and-do-proper-error-equality-check-11k7)), so we can do type checks. - 2. _exception: the error a potential symptom of internal state corruption_. + 2. _exception: the error is a potential symptom of internal state corruption_. For example, a failed sanity check. In this case, the error is most likely fatal.

@@ -107,11 +107,71 @@ happy path is either where we treat everything beyond the known benign errors as critical failures. In unexpected failure cases, we assume that the vertex's in-memory state has been broken and proper functioning is no longer guaranteed. The only safe route of recovery is to restart the vertex from a previously persisted, safe state. Per convention, a vertex should throw any unexpected exceptions using the related [irrecoverable context](https://github.com/onflow/flow-go/blob/277b6515add6136946913747efebd508f0419a25/module/irrecoverable/irrecoverable.go). - * Many components in our BFT system can return benign errors (type (i)) and exceptions (type (ii)) - * Use the special `irrecoverable.exception` [error type](https://github.com/onflow/flow-go/blob/master/module/irrecoverable/exception.go#L7-L26) to denote an unexpected error (and strip any sentinel information from the error stack) - - -3. _Optional Simplification for components that solely return benign errors._ + * Many components in our BFT system can return benign errors (type (i)) and irrecoverable exceptions (type (ii)) + +3. **Whether a particular error is benign or an exception depends on the caller's context. Errors _cannot_ be categorized as benign or exception based on their type alone.** + + ![Error Handling](/docs/ErrorHandling.png) + + * For example, consider `storage.ErrNotFound` that could be returned by the storage lager, when querying a block by ID + (method [`Headers.ByBlockID(flow.Identifier)`](https://github.com/onflow/flow-go/blob/a918616c7b541b772c254e7eaaae3573561e6c0a/storage/headers.go#L15-L18)). + In many cases, `storage.ErrNotFound` is expected, for instance if a node is receiving a new block proposal and checks whether the parent has already been ingested + or needs to be requested from a different node. In contrast, if we are querying a block that we know is already finalized and the storage returns a `storage.ErrNotFound` + something is badly broken. + * Use the special `irrecoverable.exception` [error type](https://github.com/onflow/flow-go/blob/master/module/irrecoverable/exception.go#L7-L26) + to denote an unexpected error (and strip any sentinel information from the error stack). + + This is for any scenario when a higher-level function is interpreting a sentinel returned from a lower-level function as an exception. + To construct an example, lets look at our `storage.Blocks` API, which has a [`ByHeight` method](https://github.com/onflow/flow-go/blob/a918616c7b541b772c254e7eaaae3573561e6c0a/storage/blocks.go#L24-L26) + to retrieve _finalized_ blocks by height. The following could be a hypothetical implementation: + ```golang + // ByHeight retrieves the finalized block for the given height. + // From the perspective of the storage layer, the following errors are benign: + // - storage.ErrNotFound if no finalized block at the given height is known + func ByHeight(height uint64) (*flow.Block, error) { + // Step 1: retrieve the ID of the finalized block for the given height. We expect + // `storage.ErrNotFound` during normal operations, if no block at height has been + // finalized. We just bubble this sentinel up, as it already has the expected type. + blockID, err := retrieveBlockIdByHeight(height) + if err != nil { + return nil, fmt.Errorf("could not query block by height: %w", err) + } + + // Step 2: retrieve full block by ID. Function `retrieveBlockByID` returns + // `storage.ErrNotFound` in case no block with the given ID is known. In other parts + // of the code that also use `retrieveBlockByID`, this would be expected during normal + // operations. However, here we are querying a block, which the storage layer has + // already indexed. Failure to retrieve the block implies our storage is corrupted. + block, err := retrieveBlockByID(blockID) + if err != nil { + // We cannot bubble up `storage.ErrNotFound` as this would hide this irrecoverable + // failure behind a benign sentinel error. We use the `Exception` wrapper, which + // also implements the error `interface` but provides no `Unwrap` method. Thereby, + // the `err`s sentinel type is hidden from upstream type checks, and consequently + // classified as unexpected, i.e. irrecoverable exceptions. + return nil, irrecoverable.NewExceptionf("storage inconsistency, failed to" + + "retrieve full block for indexed and finalized block %x: %w", blockID, err) + } + return block, nil + } + ``` + Functions **may** use `irrecoverable.NewExceptionf` when: + - they are interpreting any error returning from a 3rd party module as unexpected + - they are reacting to an unexpected condition internal to their stack frame and returning a generic error + + Functions **must** usd `irrecoverable.NewExceptionf` when: + - they are interpreting any documented sentinel error returned from a flow-go module as unexpected + + For brief illustration, let us consider some function body, in which there are multiple subsequent calls to other lower-level functions. + In most scenarios, a particular sentinel type is either always or never expected during normal operations. If it is expected, + then the sentinel type should be documented. If it is consistently not expected, the error should _not_ be mentioned in the + function's godoc. In the absence of positive affirmation that `error` is an expected and benign sentinel, the error is to be + treated as an irrecoverable exception. So if a sentinel type `T` is consistently not expected throughout the function's body, make + sure the sentinel `T` is not mentioned in the function's godoc. The latter is fully sufficient to classify `T` as an irrecoverable + exception. + + +5. _Optional Simplification for components that solely return benign errors._ * In this case, you _can_ use untyped errors to represent benign error cases (e.g. using `fmt.Errorf`). * By using untyped errors, the code would be _breaking with our best practice guideline_ that benign errors should be represented as typed sentinel errors. Therefore, whenever all returned errors are benign, please clearly document this _for each public functions individually_. @@ -160,7 +220,8 @@ For example, a statement like the following would be sufficient: * Handle errors at a level, where you still have enough context to decide whether the error is expected during normal operations. * Errors of unexpected types are indicators that the node's internal state might be corrupted. - - Use the special `irrecoverable.exception` [error type](https://github.com/onflow/flow-go/blob/master/module/irrecoverable/exception.go#L7-L26) at the point an unexpected error is being returned, or when an error returned from another function is interpreted as unexpected + + ### Anti-Pattern Continuing on a best-effort basis is not an option, i.e. the following is an anti-pattern in the context of Flow: diff --git a/Makefile b/Makefile index b465aad4e31..d8b37127d1d 100644 --- a/Makefile +++ b/Makefile @@ -10,9 +10,6 @@ VERSION := $(shell git describe --tags --abbrev=2 --match "v*" --match "secure-c # dynamically split up CI jobs into smaller jobs that can be run in parallel GO_TEST_PACKAGES := ./... -FLOW_GO_TAG := v0.28.15 - - # Image tag: if image tag is not set, set it with version (or short commit if empty) ifeq (${IMAGE_TAG},) IMAGE_TAG := ${VERSION} @@ -22,7 +19,7 @@ ifeq (${IMAGE_TAG},) IMAGE_TAG := ${SHORT_COMMIT} endif -IMAGE_TAG_NO_NETGO := $(IMAGE_TAG)-without_netgo +IMAGE_TAG_NO_NETGO := $(IMAGE_TAG)-without-netgo # Name of the cover profile COVER_PROFILE := coverage.txt @@ -80,7 +77,7 @@ install-tools: crypto_setup_gopath check-go-version install-mock-generators go install golang.org/x/tools/cmd/stringer@master; .PHONY: verify-mocks -verify-mocks: generate-mocks +verify-mocks: tidy generate-mocks git diff --exit-code ############################################################################################ @@ -90,6 +87,23 @@ emulator-norelic-check: # test the fvm package compiles with Relic library disabled (required for the emulator build) cd ./fvm && go test ./... -run=NoTestHasThisPrefix +.SILENT: go-math-rand-check +go-math-rand-check: + # check that the insecure math/rand Go package isn't used by production code. + # `exclude` should only specify non production code (test, bench..). + # If this check fails, try updating your code by using: + # - "crypto/rand" or "flow-go/utils/rand" for non-deterministic randomness + # - "flow-go/crypto/random" for deterministic randomness + grep --include=\*.go \ + --exclude=*test* --exclude=*helper* --exclude=*example* --exclude=*fixture* --exclude=*benchmark* --exclude=*profiler* \ + --exclude-dir=*test* --exclude-dir=*helper* --exclude-dir=*example* --exclude-dir=*fixture* --exclude-dir=*benchmark* --exclude-dir=*profiler* -rnw '"math/rand"'; \ + if [ $$? -ne 1 ]; then \ + echo "[Error] Go production code should not use math/rand package"; exit 1; \ + fi + +.PHONY: code-sanity-check +code-sanity-check: go-math-rand-check emulator-norelic-check + .PHONY: fuzz-fvm fuzz-fvm: # run fuzz tests in the fvm package @@ -170,16 +184,16 @@ generate-mocks: install-mock-generators rm -rf ./fvm/environment/mock mockery --name '.*' --dir=fvm/environment --case=underscore --output="./fvm/environment/mock" --outpkg="mock" mockery --name '.*' --dir=ledger --case=underscore --output="./ledger/mock" --outpkg="mock" - mockery --name 'ViolationsConsumer' --dir=network/slashing --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork" + mockery --name 'ViolationsConsumer' --dir=network --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork" mockery --name '.*' --dir=network/p2p/ --case=underscore --output="./network/p2p/mock" --outpkg="mockp2p" + mockery --name '.*' --dir=network/alsp --case=underscore --output="./network/alsp/mock" --outpkg="mockalsp" mockery --name 'Vertex' --dir="./module/forest" --case=underscore --output="./module/forest/mock" --outpkg="mock" mockery --name '.*' --dir="./consensus/hotstuff" --case=underscore --output="./consensus/hotstuff/mocks" --outpkg="mocks" mockery --name '.*' --dir="./engine/access/wrapper" --case=underscore --output="./engine/access/mock" --outpkg="mock" mockery --name 'API' --dir="./access" --case=underscore --output="./access/mock" --outpkg="mock" mockery --name 'API' --dir="./engine/protocol" --case=underscore --output="./engine/protocol/mock" --outpkg="mock" - mockery --name 'API' --dir="./engine/access/state_stream" --case=underscore --output="./engine/access/state_stream/mock" --outpkg="mock" - mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/backend" --case=underscore --output="./engine/access/rpc/backend/mock" --outpkg="mock" - mockery --name 'IngestRPC' --dir="./engine/execution/ingestion" --case=underscore --tags relic --output="./engine/execution/ingestion/mock" --outpkg="mock" + mockery --name '.*' --dir="./engine/access/state_stream" --case=underscore --output="./engine/access/state_stream/mock" --outpkg="mock" + mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/connection" --case=underscore --output="./engine/access/rpc/connection/mock" --outpkg="mock" mockery --name '.*' --dir=model/fingerprint --case=underscore --output="./model/fingerprint/mock" --outpkg="mock" mockery --name 'ExecForkActor' --structname 'ExecForkActorMock' --dir=module/mempool/consensus/mock/ --case=underscore --output="./module/mempool/consensus/mock/" --outpkg="mock" mockery --name '.*' --dir=engine/verification/fetcher/ --case=underscore --output="./engine/verification/fetcher/mock" --outpkg="mockfetcher" @@ -253,13 +267,16 @@ docker-ci-integration: .PHONY: docker-build-collection docker-build-collection: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/collection --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --target production \ + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/collection:latest" -t "$(CONTAINER_REGISTRY)/collection:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG)" -t "$(CONTAINER_REGISTRY)/collection:$(FLOW_GO_TAG)" . + -t "$(CONTAINER_REGISTRY)/collection:latest" -t "$(CONTAINER_REGISTRY)/collection:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG)" . .PHONY: docker-build-collection-without-netgo docker-build-collection-without-netgo: docker build -f cmd/Dockerfile --build-arg TAGS=relic --build-arg TARGET=./cmd/collection --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --target production \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG_NO_NETGO)" . + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ + -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG_NO_NETGO)" . .PHONY: docker-build-collection-debug docker-build-collection-debug: @@ -269,13 +286,16 @@ docker-build-collection-debug: .PHONY: docker-build-consensus docker-build-consensus: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/consensus --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --target production \ + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/consensus:latest" -t "$(CONTAINER_REGISTRY)/consensus:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG)" -t "$(CONTAINER_REGISTRY)/consensus:$(FLOW_GO_TAG)" . + -t "$(CONTAINER_REGISTRY)/consensus:latest" -t "$(CONTAINER_REGISTRY)/consensus:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG)" . .PHONY: docker-build-consensus-without-netgo docker-build-consensus-without-netgo: docker build -f cmd/Dockerfile --build-arg TAGS=relic --build-arg TARGET=./cmd/consensus --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --target production \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG_NO_NETGO)" . + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ + -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG_NO_NETGO)" . .PHONY: docker-build-consensus-debug docker-build-consensus-debug: @@ -285,13 +305,16 @@ docker-build-consensus-debug: .PHONY: docker-build-execution docker-build-execution: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/execution --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --target production \ + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/execution:latest" -t "$(CONTAINER_REGISTRY)/execution:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG)" -t "$(CONTAINER_REGISTRY)/execution:$(FLOW_GO_TAG)" . + -t "$(CONTAINER_REGISTRY)/execution:latest" -t "$(CONTAINER_REGISTRY)/execution:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG)" . .PHONY: docker-build-execution-without-netgo docker-build-execution-without-netgo: docker build -f cmd/Dockerfile --build-arg TAGS=relic --build-arg TARGET=./cmd/execution --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --target production \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_NO_NETGO)" . + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ + -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_NO_NETGO)" . .PHONY: docker-build-execution-debug docker-build-execution-debug: @@ -311,13 +334,16 @@ docker-build-execution-corrupt: .PHONY: docker-build-verification docker-build-verification: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/verification --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --target production \ + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/verification:latest" -t "$(CONTAINER_REGISTRY)/verification:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG)" -t "$(CONTAINER_REGISTRY)/verification:$(FLOW_GO_TAG)" . + -t "$(CONTAINER_REGISTRY)/verification:latest" -t "$(CONTAINER_REGISTRY)/verification:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG)" . .PHONY: docker-build-verification-without-netgo docker-build-verification-without-netgo: docker build -f cmd/Dockerfile --build-arg TAGS=relic --build-arg TARGET=./cmd/verification --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --target production \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG_NO_NETGO)" . + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ + -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG_NO_NETGO)" . .PHONY: docker-build-verification-debug docker-build-verification-debug: @@ -337,13 +363,16 @@ docker-build-verification-corrupt: .PHONY: docker-build-access docker-build-access: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/access --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --target production \ + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/access:latest" -t "$(CONTAINER_REGISTRY)/access:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG)" -t "$(CONTAINER_REGISTRY)/access:$(FLOW_GO_TAG)" . + -t "$(CONTAINER_REGISTRY)/access:latest" -t "$(CONTAINER_REGISTRY)/access:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG)" . .PHONY: docker-build-access-without-netgo docker-build-access-without-netgo: docker build -f cmd/Dockerfile --build-arg TAGS=relic --build-arg TARGET=./cmd/access --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --target production \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG_NO_NETGO)" . + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ + -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG_NO_NETGO)" . .PHONY: docker-build-access-debug docker-build-access-debug: @@ -363,13 +392,16 @@ docker-build-access-corrupt: .PHONY: docker-build-observer docker-build-observer: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/observer --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --target production \ + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ -t "$(CONTAINER_REGISTRY)/observer:latest" -t "$(CONTAINER_REGISTRY)/observer:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG)" . .PHONY: docker-build-observer-without-netgo docker-build-observer-without-netgo: docker build -f cmd/Dockerfile --build-arg TAGS=relic --build-arg TARGET=./cmd/observer --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --target production \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" -t "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG_NO_NETGO)" . + --secret id=git_creds,env=GITHUB_CREDS --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ + -t "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG_NO_NETGO)" . .PHONY: docker-build-ghost @@ -425,7 +457,6 @@ docker-build-benchnet: docker-build-flow docker-build-loader docker-push-collection: docker push "$(CONTAINER_REGISTRY)/collection:$(SHORT_COMMIT)" docker push "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG)" - docker push "$(CONTAINER_REGISTRY)/collection:$(FLOW_GO_TAG)" .PHONY: docker-push-collection-without-netgo docker-push-collection-without-netgo: @@ -439,7 +470,6 @@ docker-push-collection-latest: docker-push-collection docker-push-consensus: docker push "$(CONTAINER_REGISTRY)/consensus:$(SHORT_COMMIT)" docker push "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG)" - docker push "$(CONTAINER_REGISTRY)/consensus:$(FLOW_GO_TAG)" .PHONY: docker-push-consensus-without-netgo docker-push-consensus-without-netgo: @@ -453,7 +483,6 @@ docker-push-consensus-latest: docker-push-consensus docker-push-execution: docker push "$(CONTAINER_REGISTRY)/execution:$(SHORT_COMMIT)" docker push "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG)" - docker push "$(CONTAINER_REGISTRY)/execution:$(FLOW_GO_TAG)" .PHONY: docker-push-execution-corrupt docker-push-execution-corrupt: @@ -473,7 +502,6 @@ docker-push-execution-latest: docker-push-execution docker-push-verification: docker push "$(CONTAINER_REGISTRY)/verification:$(SHORT_COMMIT)" docker push "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG)" - docker push "$(CONTAINER_REGISTRY)/verification:$(FLOW_GO_TAG)" .PHONY: docker-push-verification-corrupt docker-push-verification-corrupt: @@ -492,7 +520,6 @@ docker-push-verification-latest: docker-push-verification docker-push-access: docker push "$(CONTAINER_REGISTRY)/access:$(SHORT_COMMIT)" docker push "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG)" - docker push "$(CONTAINER_REGISTRY)/access:$(FLOW_GO_TAG)" .PHONY: docker-push-access-corrupt docker-push-access-corrupt: @@ -506,7 +533,7 @@ docker-push-access-without-netgo: .PHONY: docker-push-access-latest docker-push-access-latest: docker-push-access docker push "$(CONTAINER_REGISTRY)/access:latest" - + .PHONY: docker-push-observer docker-push-observer: @@ -652,4 +679,4 @@ monitor-rollout: kubectl --kubeconfig=$$kconfig rollout status statefulsets.apps flow-collection-node-v1; \ kubectl --kubeconfig=$$kconfig rollout status statefulsets.apps flow-consensus-node-v1; \ kubectl --kubeconfig=$$kconfig rollout status statefulsets.apps flow-execution-node-v1; \ - kubectl --kubeconfig=$$kconfig rollout status statefulsets.apps flow-verification-node-v1 \ No newline at end of file + kubectl --kubeconfig=$$kconfig rollout status statefulsets.apps flow-verification-node-v1 diff --git a/README.md b/README.md index 3298a00f465..39bd7a13e3e 100644 --- a/README.md +++ b/README.md @@ -53,17 +53,7 @@ The following table lists all work streams and links to their home directory and ## Installation -### Clone Repository - - Clone this repository -- Clone this repository's submodules: - - ```bash - git submodule update --init --recursive - ``` - -### Install Dependencies - - Install [Go](https://golang.org/doc/install) (Flow supports Go 1.18 and later) - Install [CMake](https://cmake.org/install/), which is used for building the crypto library - Install [Docker](https://docs.docker.com/get-docker/), which is used for running a local network and integration tests diff --git a/access/api.go b/access/api.go index a65c35ac752..4188c04c1c4 100644 --- a/access/api.go +++ b/access/api.go @@ -14,6 +14,7 @@ import ( type API interface { Ping(ctx context.Context) error GetNetworkParameters(ctx context.Context) NetworkParameters + GetNodeVersionInfo(ctx context.Context) (*NodeVersionInfo, error) GetLatestBlockHeader(ctx context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) GetBlockHeaderByHeight(ctx context.Context, height uint64) (*flow.Header, flow.BlockStatus, error) @@ -28,7 +29,7 @@ type API interface { SendTransaction(ctx context.Context, tx *flow.TransactionBody) error GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) GetTransactionsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*flow.TransactionBody, error) - GetTransactionResult(ctx context.Context, id flow.Identifier) (*TransactionResult, error) + GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*TransactionResult, error) GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32) (*TransactionResult, error) GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*TransactionResult, error) @@ -70,7 +71,7 @@ func TransactionResultToMessage(result *TransactionResult) *access.TransactionRe BlockId: result.BlockID[:], TransactionId: result.TransactionID[:], CollectionId: result.CollectionID[:], - BlockHeight: uint64(result.BlockHeight), + BlockHeight: result.BlockHeight, } } @@ -103,3 +104,11 @@ func MessageToTransactionResult(message *access.TransactionResultResponse) *Tran type NetworkParameters struct { ChainID flow.ChainID } + +// NodeVersionInfo contains information about node, such as semver, commit, sporkID, protocolVersion, etc +type NodeVersionInfo struct { + Semver string + Commit string + SporkId flow.Identifier + ProtocolVersion uint64 +} diff --git a/access/handler.go b/access/handler.go index 914fd2a805d..11e47dd3521 100644 --- a/access/handler.go +++ b/access/handler.go @@ -3,31 +3,38 @@ package access import ( "context" - "github.com/onflow/flow/protobuf/go/flow/access" - "github.com/onflow/flow/protobuf/go/flow/entities" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/timestamppb" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/signature" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + + "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/entities" ) type Handler struct { api API chain flow.Chain signerIndicesDecoder hotstuff.BlockSignerDecoder + finalizedHeaderCache module.FinalizedHeaderCache + me module.Local } // HandlerOption is used to hand over optional constructor parameters type HandlerOption func(*Handler) -func NewHandler(api API, chain flow.Chain, options ...HandlerOption) *Handler { +var _ access.AccessAPIServer = (*Handler)(nil) + +func NewHandler(api API, chain flow.Chain, finalizedHeader module.FinalizedHeaderCache, me module.Local, options ...HandlerOption) *Handler { h := &Handler{ api: api, chain: chain, + finalizedHeaderCache: finalizedHeader, + me: me, signerIndicesDecoder: &signature.NoopBlockSignerDecoder{}, } for _, opt := range options { @@ -46,6 +53,26 @@ func (h *Handler) Ping(ctx context.Context, _ *access.PingRequest) (*access.Ping return &access.PingResponse{}, nil } +// GetNodeVersionInfo gets node version information such as semver, commit, sporkID, protocolVersion, etc +func (h *Handler) GetNodeVersionInfo( + ctx context.Context, + _ *access.GetNodeVersionInfoRequest, +) (*access.GetNodeVersionInfoResponse, error) { + nodeVersionInfo, err := h.api.GetNodeVersionInfo(ctx) + if err != nil { + return nil, err + } + + return &access.GetNodeVersionInfoResponse{ + Info: &entities.NodeVersionInfo{ + Semver: nodeVersionInfo.Semver, + Commit: nodeVersionInfo.Commit, + SporkId: nodeVersionInfo.SporkId[:], + ProtocolVersion: nodeVersionInfo.ProtocolVersion, + }, + }, nil +} + func (h *Handler) GetNetworkParameters( ctx context.Context, _ *access.GetNetworkParametersRequest, @@ -142,6 +169,8 @@ func (h *Handler) GetCollectionByID( ctx context.Context, req *access.GetCollectionByIDRequest, ) (*access.CollectionResponse, error) { + metadata := h.buildMetadataResponse() + id, err := convert.CollectionID(req.GetId()) if err != nil { return nil, err @@ -159,6 +188,7 @@ func (h *Handler) GetCollectionByID( return &access.CollectionResponse{ Collection: colMsg, + Metadata: metadata, }, nil } @@ -167,6 +197,8 @@ func (h *Handler) SendTransaction( ctx context.Context, req *access.SendTransactionRequest, ) (*access.SendTransactionResponse, error) { + metadata := h.buildMetadataResponse() + txMsg := req.GetTransaction() tx, err := convert.MessageToTransaction(txMsg, h.chain) @@ -182,7 +214,8 @@ func (h *Handler) SendTransaction( txID := tx.ID() return &access.SendTransactionResponse{ - Id: txID[:], + Id: txID[:], + Metadata: metadata, }, nil } @@ -191,6 +224,8 @@ func (h *Handler) GetTransaction( ctx context.Context, req *access.GetTransactionRequest, ) (*access.TransactionResponse, error) { + metadata := h.buildMetadataResponse() + id, err := convert.TransactionID(req.GetId()) if err != nil { return nil, err @@ -203,6 +238,7 @@ func (h *Handler) GetTransaction( return &access.TransactionResponse{ Transaction: convert.TransactionToMessage(*tx), + Metadata: metadata, }, nil } @@ -211,23 +247,48 @@ func (h *Handler) GetTransactionResult( ctx context.Context, req *access.GetTransactionRequest, ) (*access.TransactionResultResponse, error) { - id, err := convert.TransactionID(req.GetId()) + metadata := h.buildMetadataResponse() + + transactionID, err := convert.TransactionID(req.GetId()) if err != nil { return nil, err } - result, err := h.api.GetTransactionResult(ctx, id) + blockId := flow.ZeroID + requestBlockId := req.GetBlockId() + if requestBlockId != nil { + blockId, err = convert.BlockID(requestBlockId) + if err != nil { + return nil, err + } + } + + collectionId := flow.ZeroID + requestCollectionId := req.GetCollectionId() + if requestCollectionId != nil { + collectionId, err = convert.CollectionID(requestCollectionId) + if err != nil { + return nil, err + } + } + + result, err := h.api.GetTransactionResult(ctx, transactionID, blockId, collectionId) if err != nil { return nil, err } - return TransactionResultToMessage(result), nil + message := TransactionResultToMessage(result) + message.Metadata = metadata + + return message, nil } func (h *Handler) GetTransactionResultsByBlockID( ctx context.Context, req *access.GetTransactionsByBlockIDRequest, ) (*access.TransactionResultsResponse, error) { + metadata := h.buildMetadataResponse() + id, err := convert.BlockID(req.GetBlockId()) if err != nil { return nil, err @@ -238,13 +299,18 @@ func (h *Handler) GetTransactionResultsByBlockID( return nil, err } - return TransactionResultsToMessage(results), nil + message := TransactionResultsToMessage(results) + message.Metadata = metadata + + return message, nil } func (h *Handler) GetTransactionsByBlockID( ctx context.Context, req *access.GetTransactionsByBlockIDRequest, ) (*access.TransactionsResponse, error) { + metadata := h.buildMetadataResponse() + id, err := convert.BlockID(req.GetBlockId()) if err != nil { return nil, err @@ -257,6 +323,7 @@ func (h *Handler) GetTransactionsByBlockID( return &access.TransactionsResponse{ Transactions: convert.TransactionsToMessages(transactions), + Metadata: metadata, }, nil } @@ -266,6 +333,8 @@ func (h *Handler) GetTransactionResultByIndex( ctx context.Context, req *access.GetTransactionByIndexRequest, ) (*access.TransactionResultResponse, error) { + metadata := h.buildMetadataResponse() + blockID, err := convert.BlockID(req.GetBlockId()) if err != nil { return nil, err @@ -276,7 +345,10 @@ func (h *Handler) GetTransactionResultByIndex( return nil, err } - return TransactionResultToMessage(result), nil + message := TransactionResultToMessage(result) + message.Metadata = metadata + + return message, nil } // GetAccount returns an account by address at the latest sealed block. @@ -284,6 +356,8 @@ func (h *Handler) GetAccount( ctx context.Context, req *access.GetAccountRequest, ) (*access.GetAccountResponse, error) { + metadata := h.buildMetadataResponse() + address := flow.BytesToAddress(req.GetAddress()) account, err := h.api.GetAccount(ctx, address) @@ -297,7 +371,8 @@ func (h *Handler) GetAccount( } return &access.GetAccountResponse{ - Account: accountMsg, + Account: accountMsg, + Metadata: metadata, }, nil } @@ -306,6 +381,8 @@ func (h *Handler) GetAccountAtLatestBlock( ctx context.Context, req *access.GetAccountAtLatestBlockRequest, ) (*access.AccountResponse, error) { + metadata := h.buildMetadataResponse() + address, err := convert.Address(req.GetAddress(), h.chain) if err != nil { return nil, err @@ -322,7 +399,8 @@ func (h *Handler) GetAccountAtLatestBlock( } return &access.AccountResponse{ - Account: accountMsg, + Account: accountMsg, + Metadata: metadata, }, nil } @@ -330,6 +408,8 @@ func (h *Handler) GetAccountAtBlockHeight( ctx context.Context, req *access.GetAccountAtBlockHeightRequest, ) (*access.AccountResponse, error) { + metadata := h.buildMetadataResponse() + address, err := convert.Address(req.GetAddress(), h.chain) if err != nil { return nil, err @@ -346,7 +426,8 @@ func (h *Handler) GetAccountAtBlockHeight( } return &access.AccountResponse{ - Account: accountMsg, + Account: accountMsg, + Metadata: metadata, }, nil } @@ -355,6 +436,8 @@ func (h *Handler) ExecuteScriptAtLatestBlock( ctx context.Context, req *access.ExecuteScriptAtLatestBlockRequest, ) (*access.ExecuteScriptResponse, error) { + metadata := h.buildMetadataResponse() + script := req.GetScript() arguments := req.GetArguments() @@ -364,7 +447,8 @@ func (h *Handler) ExecuteScriptAtLatestBlock( } return &access.ExecuteScriptResponse{ - Value: value, + Value: value, + Metadata: metadata, }, nil } @@ -373,6 +457,8 @@ func (h *Handler) ExecuteScriptAtBlockHeight( ctx context.Context, req *access.ExecuteScriptAtBlockHeightRequest, ) (*access.ExecuteScriptResponse, error) { + metadata := h.buildMetadataResponse() + script := req.GetScript() arguments := req.GetArguments() blockHeight := req.GetBlockHeight() @@ -383,7 +469,8 @@ func (h *Handler) ExecuteScriptAtBlockHeight( } return &access.ExecuteScriptResponse{ - Value: value, + Value: value, + Metadata: metadata, }, nil } @@ -392,6 +479,8 @@ func (h *Handler) ExecuteScriptAtBlockID( ctx context.Context, req *access.ExecuteScriptAtBlockIDRequest, ) (*access.ExecuteScriptResponse, error) { + metadata := h.buildMetadataResponse() + script := req.GetScript() arguments := req.GetArguments() blockID := convert.MessageToIdentifier(req.GetBlockId()) @@ -402,7 +491,8 @@ func (h *Handler) ExecuteScriptAtBlockID( } return &access.ExecuteScriptResponse{ - Value: value, + Value: value, + Metadata: metadata, }, nil } @@ -411,6 +501,8 @@ func (h *Handler) GetEventsForHeightRange( ctx context.Context, req *access.GetEventsForHeightRangeRequest, ) (*access.EventsResponse, error) { + metadata := h.buildMetadataResponse() + eventType, err := convert.EventType(req.GetType()) if err != nil { return nil, err @@ -424,12 +516,13 @@ func (h *Handler) GetEventsForHeightRange( return nil, err } - resultEvents, err := blockEventsToMessages(results) + resultEvents, err := convert.BlockEventsToMessages(results) if err != nil { return nil, err } return &access.EventsResponse{ - Results: resultEvents, + Results: resultEvents, + Metadata: metadata, }, nil } @@ -438,6 +531,8 @@ func (h *Handler) GetEventsForBlockIDs( ctx context.Context, req *access.GetEventsForBlockIDsRequest, ) (*access.EventsResponse, error) { + metadata := h.buildMetadataResponse() + eventType, err := convert.EventType(req.GetType()) if err != nil { return nil, err @@ -453,18 +548,21 @@ func (h *Handler) GetEventsForBlockIDs( return nil, err } - resultEvents, err := blockEventsToMessages(results) + resultEvents, err := convert.BlockEventsToMessages(results) if err != nil { return nil, err } return &access.EventsResponse{ - Results: resultEvents, + Results: resultEvents, + Metadata: metadata, }, nil } // GetLatestProtocolStateSnapshot returns the latest serializable Snapshot func (h *Handler) GetLatestProtocolStateSnapshot(ctx context.Context, req *access.GetLatestProtocolStateSnapshotRequest) (*access.ProtocolStateSnapshotResponse, error) { + metadata := h.buildMetadataResponse() + snapshot, err := h.api.GetLatestProtocolStateSnapshot(ctx) if err != nil { return nil, err @@ -472,6 +570,7 @@ func (h *Handler) GetLatestProtocolStateSnapshot(ctx context.Context, req *acces return &access.ProtocolStateSnapshotResponse{ SerializedSnapshot: snapshot, + Metadata: metadata, }, nil } @@ -479,6 +578,8 @@ func (h *Handler) GetLatestProtocolStateSnapshot(ctx context.Context, req *acces // AN might receive multiple receipts with conflicting results for unsealed blocks. // If this case happens, since AN is not able to determine which result is the correct one until the block is sealed, it has to pick one result to respond to this query. For now, we return the result from the latest received receipt. func (h *Handler) GetExecutionResultForBlockID(ctx context.Context, req *access.GetExecutionResultForBlockIDRequest) (*access.ExecutionResultForBlockIDResponse, error) { + metadata := h.buildMetadataResponse() + blockID := convert.MessageToIdentifier(req.GetBlockId()) result, err := h.api.GetExecutionResultForBlockID(ctx, blockID) @@ -486,10 +587,33 @@ func (h *Handler) GetExecutionResultForBlockID(ctx context.Context, req *access. return nil, err } - return executionResultToMessages(result) + return executionResultToMessages(result, metadata) +} + +// GetExecutionResultByID returns the execution result for the given ID. +func (h *Handler) GetExecutionResultByID(ctx context.Context, req *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) { + metadata := h.buildMetadataResponse() + + blockID := convert.MessageToIdentifier(req.GetId()) + + result, err := h.api.GetExecutionResultByID(ctx, blockID) + if err != nil { + return nil, err + } + + execResult, err := convert.ExecutionResultToMessage(result) + if err != nil { + return nil, err + } + return &access.ExecutionResultByIDResponse{ + ExecutionResult: execResult, + Metadata: metadata, + }, nil } func (h *Handler) blockResponse(block *flow.Block, fullResponse bool, status flow.BlockStatus) (*access.BlockResponse, error) { + metadata := h.buildMetadataResponse() + signerIDs, err := h.signerIndicesDecoder.DecodeSignerIDs(block.Header) if err != nil { return nil, err // the block was retrieved from local storage - so no errors are expected @@ -504,13 +628,17 @@ func (h *Handler) blockResponse(block *flow.Block, fullResponse bool, status flo } else { msg = convert.BlockToMessageLight(block) } + return &access.BlockResponse{ Block: msg, BlockStatus: entities.BlockStatus(status), + Metadata: metadata, }, nil } func (h *Handler) blockHeaderResponse(header *flow.Header, status flow.BlockStatus) (*access.BlockHeaderResponse, error) { + metadata := h.buildMetadataResponse() + signerIDs, err := h.signerIndicesDecoder.DecodeSignerIDs(header) if err != nil { return nil, err // the block was retrieved from local storage - so no errors are expected @@ -524,42 +652,31 @@ func (h *Handler) blockHeaderResponse(header *flow.Header, status flow.BlockStat return &access.BlockHeaderResponse{ Block: msg, BlockStatus: entities.BlockStatus(status), + Metadata: metadata, }, nil } -func executionResultToMessages(er *flow.ExecutionResult) (*access.ExecutionResultForBlockIDResponse, error) { - execResult, err := convert.ExecutionResultToMessage(er) - if err != nil { - return nil, err - } - return &access.ExecutionResultForBlockIDResponse{ExecutionResult: execResult}, nil -} - -func blockEventsToMessages(blocks []flow.BlockEvents) ([]*access.EventsResponse_Result, error) { - results := make([]*access.EventsResponse_Result, len(blocks)) +// buildMetadataResponse builds and returns the metadata response object. +func (h *Handler) buildMetadataResponse() *entities.Metadata { + lastFinalizedHeader := h.finalizedHeaderCache.Get() + blockId := lastFinalizedHeader.ID() + nodeId := h.me.NodeID() - for i, block := range blocks { - event, err := blockEventsToMessage(block) - if err != nil { - return nil, err - } - results[i] = event + return &entities.Metadata{ + LatestFinalizedBlockId: blockId[:], + LatestFinalizedHeight: lastFinalizedHeader.Height, + NodeId: nodeId[:], } - - return results, nil } -func blockEventsToMessage(block flow.BlockEvents) (*access.EventsResponse_Result, error) { - eventMessages := make([]*entities.Event, len(block.Events)) - for i, event := range block.Events { - eventMessages[i] = convert.EventToMessage(event) +func executionResultToMessages(er *flow.ExecutionResult, metadata *entities.Metadata) (*access.ExecutionResultForBlockIDResponse, error) { + execResult, err := convert.ExecutionResultToMessage(er) + if err != nil { + return nil, err } - timestamp := timestamppb.New(block.BlockTimestamp) - return &access.EventsResponse_Result{ - BlockId: block.BlockID[:], - BlockHeight: block.BlockHeight, - BlockTimestamp: timestamp, - Events: eventMessages, + return &access.ExecutionResultForBlockIDResponse{ + ExecutionResult: execResult, + Metadata: metadata, }, nil } diff --git a/access/legacy/handler.go b/access/legacy/handler.go index 0912464f203..48f4efc911d 100644 --- a/access/legacy/handler.go +++ b/access/legacy/handler.go @@ -189,7 +189,7 @@ func (h *Handler) GetTransactionResult( ) (*accessproto.TransactionResultResponse, error) { id := convert.MessageToIdentifier(req.GetId()) - result, err := h.api.GetTransactionResult(ctx, id) + result, err := h.api.GetTransactionResult(ctx, id, flow.ZeroID, flow.ZeroID) if err != nil { return nil, err } diff --git a/access/mock/api.go b/access/mock/api.go index c534e272364..b3a91590f80 100644 --- a/access/mock/api.go +++ b/access/mock/api.go @@ -541,6 +541,32 @@ func (_m *API) GetNetworkParameters(ctx context.Context) access.NetworkParameter return r0 } +// GetNodeVersionInfo provides a mock function with given fields: ctx +func (_m *API) GetNodeVersionInfo(ctx context.Context) (*access.NodeVersionInfo, error) { + ret := _m.Called(ctx) + + var r0 *access.NodeVersionInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*access.NodeVersionInfo, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *access.NodeVersionInfo); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*access.NodeVersionInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetTransaction provides a mock function with given fields: ctx, id func (_m *API) GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) { ret := _m.Called(ctx, id) @@ -567,25 +593,25 @@ func (_m *API) GetTransaction(ctx context.Context, id flow.Identifier) (*flow.Tr return r0, r1 } -// GetTransactionResult provides a mock function with given fields: ctx, id -func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier) (*access.TransactionResult, error) { - ret := _m.Called(ctx, id) +// GetTransactionResult provides a mock function with given fields: ctx, id, blockID, collectionID +func (_m *API) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*access.TransactionResult, error) { + ret := _m.Called(ctx, id, blockID, collectionID) var r0 *access.TransactionResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*access.TransactionResult, error)); ok { - return rf(ctx, id) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) (*access.TransactionResult, error)); ok { + return rf(ctx, id, blockID, collectionID) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *access.TransactionResult); ok { - r0 = rf(ctx, id) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) *access.TransactionResult); ok { + r0 = rf(ctx, id, blockID, collectionID) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*access.TransactionResult) } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { - r1 = rf(ctx, id) + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) error); ok { + r1 = rf(ctx, id, blockID, collectionID) } else { r1 = ret.Error(1) } diff --git a/admin/README.md b/admin/README.md index 05d9901f9f4..7e4e5831aa2 100644 --- a/admin/README.md +++ b/admin/README.md @@ -21,11 +21,6 @@ libp2p, badger, and other golog-based libraries: curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-golog-level", "data": "debug"}' ``` -### To turn on profiler -``` -curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-profiler-enabled", "data": true}' -``` - ### To get the latest finalized block ``` curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "read-blocks", "data": { "block": "final" }}' @@ -51,6 +46,17 @@ curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{" curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "get-transactions", "data": { "start-height": 340, "end-height": 343 }}' ``` +### To get blocks for ranges (works for any node type, for block payload, only prints the collection ids) +``` +curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "read-range-blocks", "data": { "start-height": 105172044, "end-height": 105172047 }}' +``` + +### To get cluster block for ranges (only available to collection nodes, only prints the transaction ids) + +``` +curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "read-range-cluster-blocks", "data": { "chain-id": "cluster-576-e8af4702d837acb77868a95f61eb212f90b14c6b7d61c89f48949fd27d1a269b", "start-height": 25077, "end-height": 25080 }}' +``` + ### To get execution data for a block by execution_data_id (only available execution nodes and access nodes with execution sync enabled) ``` curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "read-execution-data", "data": { "execution_data_id": "2fff2b05e7226c58e3c14b3549ab44a354754761c5baa721ea0d1ea26d069dc4" }}' @@ -71,6 +77,7 @@ curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{" ``` curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-config", "data": {"consensus-required-approvals-for-sealing": 1}}' ``` +TODO remove #### Example: set block rate delay to 750ms ``` curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-config", "data": {"hotstuff-block-rate-delay": "750ms"}}' diff --git a/admin/commands/execution/stop_at_height.go b/admin/commands/execution/stop_at_height.go index b39b03e904e..fd88a8c4f10 100644 --- a/admin/commands/execution/stop_at_height.go +++ b/admin/commands/execution/stop_at_height.go @@ -7,7 +7,7 @@ import ( "github.com/onflow/flow-go/admin" "github.com/onflow/flow-go/admin/commands" - "github.com/onflow/flow-go/engine/execution/ingestion" + "github.com/onflow/flow-go/engine/execution/ingestion/stop" ) var _ commands.AdminCommand = (*StopAtHeightCommand)(nil) @@ -15,11 +15,11 @@ var _ commands.AdminCommand = (*StopAtHeightCommand)(nil) // StopAtHeightCommand will send a signal to engine to stop/crash EN // at given height type StopAtHeightCommand struct { - stopControl *ingestion.StopControl + stopControl *stop.StopControl } // NewStopAtHeightCommand creates a new StopAtHeightCommand object -func NewStopAtHeightCommand(sah *ingestion.StopControl) *StopAtHeightCommand { +func NewStopAtHeightCommand(sah *stop.StopControl) *StopAtHeightCommand { return &StopAtHeightCommand{ stopControl: sah, } @@ -36,13 +36,22 @@ type StopAtHeightReq struct { func (s *StopAtHeightCommand) Handler(_ context.Context, req *admin.CommandRequest) (interface{}, error) { sah := req.ValidatorData.(StopAtHeightReq) - oldHeight, oldCrash, err := s.stopControl.SetStopHeight(sah.height, sah.crash) + oldParams := s.stopControl.GetStopParameters() + newParams := stop.StopParameters{ + StopBeforeHeight: sah.height, + ShouldCrash: sah.crash, + } + + err := s.stopControl.SetStopParameters(newParams) if err != nil { return nil, err } - log.Info().Msgf("admintool: EN will stop at height %d and crash: %t, previous values: %d %t", sah.height, sah.crash, oldHeight, oldCrash) + log.Info(). + Interface("newParams", newParams). + Interface("oldParams", oldParams). + Msgf("admintool: New En stop parameters set") return "ok", nil } diff --git a/admin/commands/execution/stop_at_height_test.go b/admin/commands/execution/stop_at_height_test.go index 961d19ee452..c2ebe4cc93d 100644 --- a/admin/commands/execution/stop_at_height_test.go +++ b/admin/commands/execution/stop_at_height_test.go @@ -3,12 +3,15 @@ package execution import ( "context" "testing" + "time" "github.com/rs/zerolog" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/admin" - "github.com/onflow/flow-go/engine/execution/ingestion" + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/execution/ingestion/stop" + "github.com/onflow/flow-go/model/flow" ) func TestCommandParsing(t *testing.T) { @@ -88,7 +91,18 @@ func TestCommandParsing(t *testing.T) { func TestCommandsSetsValues(t *testing.T) { - stopControl := ingestion.NewStopControl(zerolog.Nop(), false, 0) + stopControl := stop.NewStopControl( + engine.NewUnit(), + time.Second, + zerolog.Nop(), + nil, + nil, + nil, + nil, + &flow.Header{Height: 1}, + false, + false, + ) cmd := NewStopAtHeightCommand(stopControl) @@ -102,9 +116,9 @@ func TestCommandsSetsValues(t *testing.T) { _, err := cmd.Handler(context.TODO(), req) require.NoError(t, err) - height, crash := stopControl.GetStopHeight() + s := stopControl.GetStopParameters() - require.Equal(t, stopControl.GetState(), ingestion.StopControlSet) - require.Equal(t, uint64(37), height) - require.Equal(t, true, crash) + require.NotNil(t, s) + require.Equal(t, uint64(37), s.StopBeforeHeight) + require.Equal(t, true, s.ShouldCrash) } diff --git a/admin/commands/state_synchronization/read_execution_data.go b/admin/commands/state_synchronization/read_execution_data.go index 04268cd6f89..5b3e4a98f75 100644 --- a/admin/commands/state_synchronization/read_execution_data.go +++ b/admin/commands/state_synchronization/read_execution_data.go @@ -24,7 +24,7 @@ type ReadExecutionDataCommand struct { func (r *ReadExecutionDataCommand) Handler(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { data := req.ValidatorData.(*requestData) - ed, err := r.executionDataStore.GetExecutionData(ctx, data.rootID) + ed, err := r.executionDataStore.Get(ctx, data.rootID) if err != nil { return nil, fmt.Errorf("failed to get execution data: %w", err) diff --git a/admin/commands/storage/helper.go b/admin/commands/storage/helper.go index 9474f85131f..e8166cb731a 100644 --- a/admin/commands/storage/helper.go +++ b/admin/commands/storage/helper.go @@ -5,6 +5,7 @@ import ( "math" "strings" + "github.com/onflow/flow-go/admin" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" ) @@ -90,3 +91,69 @@ func getBlockHeader(state protocol.State, req *blocksRequest) (*flow.Header, err return nil, fmt.Errorf("invalid request type: %v", req.requestType) } } + +func parseHeightRangeRequestData(req *admin.CommandRequest) (*heightRangeReqData, error) { + input, ok := req.Data.(map[string]interface{}) + if !ok { + return nil, admin.NewInvalidAdminReqFormatError("missing 'data' field") + } + + startHeight, err := findUint64(input, "start-height") + if err != nil { + return nil, fmt.Errorf("invalid start-height: %w", err) + } + + endHeight, err := findUint64(input, "end-height") + if err != nil { + return nil, fmt.Errorf("invalid end-height: %w", err) + } + + if endHeight < startHeight { + return nil, admin.NewInvalidAdminReqErrorf("end-height %v should not be smaller than start-height %v", endHeight, startHeight) + } + + return &heightRangeReqData{ + startHeight: startHeight, + endHeight: endHeight, + }, nil +} + +func parseString(req *admin.CommandRequest, field string) (string, error) { + input, ok := req.Data.(map[string]interface{}) + if !ok { + return "", admin.NewInvalidAdminReqFormatError("missing 'data' field") + } + fieldValue, err := findString(input, field) + if err != nil { + return "", admin.NewInvalidAdminReqErrorf("missing %v field", field) + } + return fieldValue, nil +} + +// Returns admin.InvalidAdminReqError for invalid inputs +func findUint64(input map[string]interface{}, field string) (uint64, error) { + data, ok := input[field] + if !ok { + return 0, admin.NewInvalidAdminReqErrorf("missing required field '%s'", field) + } + val, err := parseN(data) + if err != nil { + return 0, admin.NewInvalidAdminReqErrorf("invalid 'n' field: %w", err) + } + + return uint64(val), nil +} + +func findString(input map[string]interface{}, field string) (string, error) { + data, ok := input[field] + if !ok { + return "", admin.NewInvalidAdminReqErrorf("missing required field '%s'", field) + } + + str, ok := data.(string) + if !ok { + return "", admin.NewInvalidAdminReqErrorf("field '%s' is not string", field) + } + + return strings.ToLower(strings.TrimSpace(str)), nil +} diff --git a/admin/commands/storage/read_blocks.go b/admin/commands/storage/read_blocks.go index 3405a5be6e2..efd74908240 100644 --- a/admin/commands/storage/read_blocks.go +++ b/admin/commands/storage/read_blocks.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "github.com/rs/zerolog/log" + "github.com/onflow/flow-go/admin" "github.com/onflow/flow-go/admin/commands" "github.com/onflow/flow-go/model/flow" @@ -28,6 +30,8 @@ func (r *ReadBlocksCommand) Handler(ctx context.Context, req *admin.CommandReque var result []*flow.Block var blockID flow.Identifier + log.Info().Str("module", "admin-tool").Msgf("read blocks, data: %v", data) + if header, err := getBlockHeader(r.state, data.blocksRequest); err != nil { return nil, fmt.Errorf("failed to get block header: %w", err) } else { diff --git a/admin/commands/storage/read_range_blocks.go b/admin/commands/storage/read_range_blocks.go new file mode 100644 index 00000000000..cc4d00d6354 --- /dev/null +++ b/admin/commands/storage/read_range_blocks.go @@ -0,0 +1,51 @@ +package storage + +import ( + "context" + + "github.com/rs/zerolog/log" + + "github.com/onflow/flow-go/admin" + "github.com/onflow/flow-go/admin/commands" + "github.com/onflow/flow-go/cmd/util/cmd/read-light-block" + "github.com/onflow/flow-go/storage" +) + +var _ commands.AdminCommand = (*ReadRangeBlocksCommand)(nil) + +// 10001 instead of 10000, because 10000 won't allow a range from 10000 to 20000, +// which is easier to type than [10001, 20000] +const Max_Range_Block_Limit = uint64(10001) + +type ReadRangeBlocksCommand struct { + blocks storage.Blocks +} + +func NewReadRangeBlocksCommand(blocks storage.Blocks) commands.AdminCommand { + return &ReadRangeBlocksCommand{ + blocks: blocks, + } +} + +func (c *ReadRangeBlocksCommand) Handler(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { + reqData, err := parseHeightRangeRequestData(req) + if err != nil { + return nil, err + } + + log.Info().Str("module", "admin-tool").Msgf("read range blocks, data: %v", reqData) + + if reqData.Range() > Max_Range_Block_Limit { + return nil, admin.NewInvalidAdminReqErrorf("getting for more than %v blocks at a time might have an impact to node's performance and is not allowed", Max_Range_Block_Limit) + } + + lights, err := read.ReadLightBlockByHeightRange(c.blocks, reqData.startHeight, reqData.endHeight) + if err != nil { + return nil, err + } + return commands.ConvertToInterfaceList(lights) +} + +func (c *ReadRangeBlocksCommand) Validator(req *admin.CommandRequest) error { + return nil +} diff --git a/admin/commands/storage/read_range_cluster_blocks.go b/admin/commands/storage/read_range_cluster_blocks.go new file mode 100644 index 00000000000..f28e3bcd7e7 --- /dev/null +++ b/admin/commands/storage/read_range_cluster_blocks.go @@ -0,0 +1,67 @@ +package storage + +import ( + "context" + "fmt" + + "github.com/dgraph-io/badger/v2" + "github.com/rs/zerolog/log" + + "github.com/onflow/flow-go/admin" + "github.com/onflow/flow-go/admin/commands" + "github.com/onflow/flow-go/cmd/util/cmd/read-light-block" + "github.com/onflow/flow-go/model/flow" + storage "github.com/onflow/flow-go/storage/badger" +) + +var _ commands.AdminCommand = (*ReadRangeClusterBlocksCommand)(nil) + +// 10001 instead of 10000, because 10000 won't allow a range from 10000 to 20000, +// which is easier to type than [10001, 20000] +const Max_Range_Cluster_Block_Limit = uint64(10001) + +type ReadRangeClusterBlocksCommand struct { + db *badger.DB + headers *storage.Headers + payloads *storage.ClusterPayloads +} + +func NewReadRangeClusterBlocksCommand(db *badger.DB, headers *storage.Headers, payloads *storage.ClusterPayloads) commands.AdminCommand { + return &ReadRangeClusterBlocksCommand{ + db: db, + headers: headers, + payloads: payloads, + } +} + +func (c *ReadRangeClusterBlocksCommand) Handler(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { + chainID, err := parseString(req, "chain-id") + if err != nil { + return nil, err + } + + reqData, err := parseHeightRangeRequestData(req) + if err != nil { + return nil, err + } + + log.Info().Str("module", "admin-tool").Msgf("read range cluster blocks, data: %v", reqData) + + if reqData.Range() > Max_Range_Cluster_Block_Limit { + return nil, admin.NewInvalidAdminReqErrorf("getting for more than %v blocks at a time might have an impact to node's performance and is not allowed", Max_Range_Cluster_Block_Limit) + } + + clusterBlocks := storage.NewClusterBlocks( + c.db, flow.ChainID(chainID), c.headers, c.payloads, + ) + + lights, err := read.ReadClusterLightBlockByHeightRange(clusterBlocks, reqData.startHeight, reqData.endHeight) + if err != nil { + return nil, fmt.Errorf("could not get with chainID id %v: %w", chainID, err) + } + return commands.ConvertToInterfaceList(lights) +} + +func (c *ReadRangeClusterBlocksCommand) Validator(req *admin.CommandRequest) error { + return nil +} diff --git a/admin/commands/storage/read_transactions.go b/admin/commands/storage/read_transactions.go index 8c38a8edc98..386c429d509 100644 --- a/admin/commands/storage/read_transactions.go +++ b/admin/commands/storage/read_transactions.go @@ -14,14 +14,15 @@ import ( var _ commands.AdminCommand = (*GetTransactionsCommand)(nil) -// max number of block height to query transactions from -var MAX_HEIGHT_RANGE = uint64(1000) - -type getTransactionsReqData struct { +type heightRangeReqData struct { startHeight uint64 endHeight uint64 } +func (d heightRangeReqData) Range() uint64 { + return d.endHeight - d.startHeight + 1 +} + type GetTransactionsCommand struct { state protocol.State payloads storage.Payloads @@ -37,7 +38,12 @@ func NewGetTransactionsCommand(state protocol.State, payloads storage.Payloads, } func (c *GetTransactionsCommand) Handler(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { - data := req.ValidatorData.(*getTransactionsReqData) + data := req.ValidatorData.(*heightRangeReqData) + + limit := uint64(10001) + if data.Range() > limit { + return nil, admin.NewInvalidAdminReqErrorf("getting transactions for more than %v blocks at a time might have an impact to node's performance and is not allowed", limit) + } finder := &transactions.Finder{ State: c.state, @@ -55,50 +61,13 @@ func (c *GetTransactionsCommand) Handler(ctx context.Context, req *admin.Command return commands.ConvertToInterfaceList(blocks) } -// Returns admin.InvalidAdminReqError for invalid inputs -func findUint64(input map[string]interface{}, field string) (uint64, error) { - data, ok := input[field] - if !ok { - return 0, admin.NewInvalidAdminReqErrorf("missing required field '%s'", field) - } - val, err := parseN(data) - if err != nil { - return 0, admin.NewInvalidAdminReqErrorf("invalid 'n' field: %w", err) - } - - return uint64(val), nil -} - // Validator validates the request. // Returns admin.InvalidAdminReqError for invalid/malformed requests. func (c *GetTransactionsCommand) Validator(req *admin.CommandRequest) error { - input, ok := req.Data.(map[string]interface{}) - if !ok { - return admin.NewInvalidAdminReqFormatError("expected map[string]any") - } - - startHeight, err := findUint64(input, "start-height") - if err != nil { - return err - } - - endHeight, err := findUint64(input, "end-height") + data, err := parseHeightRangeRequestData(req) if err != nil { return err } - - if endHeight < startHeight { - return admin.NewInvalidAdminReqErrorf("endHeight %v should not be smaller than startHeight %v", endHeight, startHeight) - } - - if endHeight-startHeight+1 > MAX_HEIGHT_RANGE { - return admin.NewInvalidAdminReqErrorf("getting transactions for more than %v blocks at a time might have an impact to node's performance and is not allowed", MAX_HEIGHT_RANGE) - } - - req.ValidatorData = &getTransactionsReqData{ - startHeight: startHeight, - endHeight: endHeight, - } - + req.ValidatorData = data return nil } diff --git a/admin/commands/storage/read_transactions_test.go b/admin/commands/storage/read_transactions_test.go index 664a27e065c..2a3917c48e9 100644 --- a/admin/commands/storage/read_transactions_test.go +++ b/admin/commands/storage/read_transactions_test.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "testing" @@ -14,13 +15,18 @@ func TestReadTransactionsRangeTooWide(t *testing.T) { data := map[string]interface{}{ "start-height": float64(1), - "end-height": float64(1001), + "end-height": float64(10002), } - err := c.Validator(&admin.CommandRequest{ + + req := &admin.CommandRequest{ Data: data, - }) + } + err := c.Validator(req) + require.NoError(t, err) + + _, err = c.Handler(context.Background(), req) require.Error(t, err) - require.Contains(t, fmt.Sprintf("%v", err), "more than 1000 blocks") + require.Contains(t, fmt.Sprintf("%v", err), "more than 10001 blocks") } func TestReadTransactionsRangeInvalid(t *testing.T) { diff --git a/apiproxy/access_api_proxy.go b/apiproxy/access_api_proxy.go deleted file mode 100644 index 8e0b781af5e..00000000000 --- a/apiproxy/access_api_proxy.go +++ /dev/null @@ -1,468 +0,0 @@ -package apiproxy - -import ( - "context" - "fmt" - "sync" - "time" - - "google.golang.org/grpc/connectivity" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/status" - - "github.com/onflow/flow/protobuf/go/flow/access" - - "github.com/onflow/flow-go/engine/access/rpc/backend" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/grpcutils" -) - -// NewFlowAccessAPIRouter creates a backend access API that forwards some requests to an upstream node. -// It is used by Observer services, Blockchain Data Service, etc. -// Make sure that this is just for observation and not a staked participant in the flow network. -// This means that observers see a copy of the data but there is no interaction to ensure integrity from the root block. -func NewFlowAccessAPIRouter(accessNodeAddressAndPort flow.IdentityList, timeout time.Duration) (*FlowAccessAPIRouter, error) { - ret := &FlowAccessAPIRouter{} - err := ret.upstream.setFlowAccessAPI(accessNodeAddressAndPort, timeout) - if err != nil { - return nil, err - } - return ret, nil -} - -// setFlowAccessAPI sets a backend access API that forwards some requests to an upstream node. -// It is used by Observer services, Blockchain Data Service, etc. -// Make sure that this is just for observation and not a staked participant in the flow network. -// This means that observers see a copy of the data but there is no interaction to ensure integrity from the root block. -func (ret *FlowAccessAPIForwarder) setFlowAccessAPI(accessNodeAddressAndPort flow.IdentityList, timeout time.Duration) error { - ret.timeout = timeout - ret.ids = accessNodeAddressAndPort - ret.upstream = make([]access.AccessAPIClient, accessNodeAddressAndPort.Count()) - ret.connections = make([]*grpc.ClientConn, accessNodeAddressAndPort.Count()) - for i, identity := range accessNodeAddressAndPort { - // Store the faultTolerantClient setup parameters such as address, public, key and timeout, so that - // we can refresh the API on connection loss - ret.ids[i] = identity - - // We fail on any single error on startup, so that - // we identify bootstrapping errors early - err := ret.reconnectingClient(i) - if err != nil { - return err - } - } - - ret.roundRobin = 0 - return nil -} - -// FlowAccessAPIRouter is a structure that represents the routing proxy algorithm. -// It splits requests between a local and a remote API service. -type FlowAccessAPIRouter struct { - access.AccessAPIServer - upstream FlowAccessAPIForwarder -} - -// SetLocalAPI sets the local backend that responds to block related calls -// Everything else is forwarded to a selected upstream node -func (h *FlowAccessAPIRouter) SetLocalAPI(local access.AccessAPIServer) { - h.AccessAPIServer = local -} - -// reconnectingClient returns an active client, or -// creates one, if the last one is not ready anymore. -func (h *FlowAccessAPIForwarder) reconnectingClient(i int) error { - timeout := h.timeout - - if h.connections[i] == nil || h.connections[i].GetState() != connectivity.Ready { - identity := h.ids[i] - var connection *grpc.ClientConn - var err error - if identity.NetworkPubKey == nil { - connection, err = grpc.Dial( - identity.Address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), - grpc.WithInsecure(), //nolint:staticcheck - backend.WithClientUnaryInterceptor(timeout)) - if err != nil { - return err - } - } else { - tlsConfig, err := grpcutils.DefaultClientTLSConfig(identity.NetworkPubKey) - if err != nil { - return fmt.Errorf("failed to get default TLS client config using public flow networking key %s %w", identity.NetworkPubKey.String(), err) - } - - connection, err = grpc.Dial( - identity.Address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), - grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), - backend.WithClientUnaryInterceptor(timeout)) - if err != nil { - return fmt.Errorf("cannot connect to %s %w", identity.Address, err) - } - } - connection.Connect() - time.Sleep(1 * time.Second) - state := connection.GetState() - if state != connectivity.Ready && state != connectivity.Connecting { - return fmt.Errorf("%v", state) - } - h.connections[i] = connection - h.upstream[i] = access.NewAccessAPIClient(connection) - } - - return nil -} - -// faultTolerantClient implements an upstream connection that reconnects on errors -// a reasonable amount of time. -func (h *FlowAccessAPIForwarder) faultTolerantClient() (access.AccessAPIClient, error) { - if h.upstream == nil || len(h.upstream) == 0 { - return nil, status.Errorf(codes.Unimplemented, "method not implemented") - } - - // Reasoning: A retry count of three gives an acceptable 5% failure ratio from a 37% failure ratio. - // A bigger number is problematic due to the DNS resolve and connection times, - // plus the need to log and debug each individual connection failure. - // - // This reasoning eliminates the need of making this parameter configurable. - // The logic works rolling over a single connection as well making clean code. - const retryMax = 3 - - h.lock.Lock() - defer h.lock.Unlock() - - var err error - for i := 0; i < retryMax; i++ { - h.roundRobin++ - h.roundRobin = h.roundRobin % len(h.upstream) - err = h.reconnectingClient(h.roundRobin) - if err != nil { - continue - } - state := h.connections[h.roundRobin].GetState() - if state != connectivity.Ready && state != connectivity.Connecting { - continue - } - return h.upstream[h.roundRobin], nil - } - - return nil, status.Errorf(codes.Unavailable, err.Error()) -} - -// Ping pings the service. It is special in the sense that it responds successful, -// only if all underlying services are ready. -func (h *FlowAccessAPIRouter) Ping(context context.Context, req *access.PingRequest) (*access.PingResponse, error) { - return h.AccessAPIServer.Ping(context, req) -} - -func (h *FlowAccessAPIRouter) GetLatestBlockHeader(context context.Context, req *access.GetLatestBlockHeaderRequest) (*access.BlockHeaderResponse, error) { - return h.AccessAPIServer.GetLatestBlockHeader(context, req) -} - -func (h *FlowAccessAPIRouter) GetBlockHeaderByID(context context.Context, req *access.GetBlockHeaderByIDRequest) (*access.BlockHeaderResponse, error) { - return h.AccessAPIServer.GetBlockHeaderByID(context, req) -} - -func (h *FlowAccessAPIRouter) GetBlockHeaderByHeight(context context.Context, req *access.GetBlockHeaderByHeightRequest) (*access.BlockHeaderResponse, error) { - return h.AccessAPIServer.GetBlockHeaderByHeight(context, req) -} - -func (h *FlowAccessAPIRouter) GetLatestBlock(context context.Context, req *access.GetLatestBlockRequest) (*access.BlockResponse, error) { - return h.AccessAPIServer.GetLatestBlock(context, req) -} - -func (h *FlowAccessAPIRouter) GetBlockByID(context context.Context, req *access.GetBlockByIDRequest) (*access.BlockResponse, error) { - return h.AccessAPIServer.GetBlockByID(context, req) -} - -func (h *FlowAccessAPIRouter) GetBlockByHeight(context context.Context, req *access.GetBlockByHeightRequest) (*access.BlockResponse, error) { - return h.AccessAPIServer.GetBlockByHeight(context, req) -} - -func (h *FlowAccessAPIRouter) GetCollectionByID(context context.Context, req *access.GetCollectionByIDRequest) (*access.CollectionResponse, error) { - return h.AccessAPIServer.GetCollectionByID(context, req) -} - -func (h *FlowAccessAPIRouter) SendTransaction(context context.Context, req *access.SendTransactionRequest) (*access.SendTransactionResponse, error) { - return h.upstream.SendTransaction(context, req) -} - -func (h *FlowAccessAPIRouter) GetTransaction(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResponse, error) { - return h.upstream.GetTransaction(context, req) -} - -func (h *FlowAccessAPIRouter) GetTransactionResult(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResultResponse, error) { - return h.upstream.GetTransactionResult(context, req) -} - -func (h *FlowAccessAPIRouter) GetTransactionResultByIndex(context context.Context, req *access.GetTransactionByIndexRequest) (*access.TransactionResultResponse, error) { - return h.upstream.GetTransactionResultByIndex(context, req) -} - -func (h *FlowAccessAPIRouter) GetAccount(context context.Context, req *access.GetAccountRequest) (*access.GetAccountResponse, error) { - return h.upstream.GetAccount(context, req) -} - -func (h *FlowAccessAPIRouter) GetAccountAtLatestBlock(context context.Context, req *access.GetAccountAtLatestBlockRequest) (*access.AccountResponse, error) { - return h.upstream.GetAccountAtLatestBlock(context, req) -} - -func (h *FlowAccessAPIRouter) GetAccountAtBlockHeight(context context.Context, req *access.GetAccountAtBlockHeightRequest) (*access.AccountResponse, error) { - return h.upstream.GetAccountAtBlockHeight(context, req) -} - -func (h *FlowAccessAPIRouter) ExecuteScriptAtLatestBlock(context context.Context, req *access.ExecuteScriptAtLatestBlockRequest) (*access.ExecuteScriptResponse, error) { - return h.upstream.ExecuteScriptAtLatestBlock(context, req) -} - -func (h *FlowAccessAPIRouter) ExecuteScriptAtBlockID(context context.Context, req *access.ExecuteScriptAtBlockIDRequest) (*access.ExecuteScriptResponse, error) { - return h.upstream.ExecuteScriptAtBlockID(context, req) -} - -func (h *FlowAccessAPIRouter) ExecuteScriptAtBlockHeight(context context.Context, req *access.ExecuteScriptAtBlockHeightRequest) (*access.ExecuteScriptResponse, error) { - return h.upstream.ExecuteScriptAtBlockHeight(context, req) -} - -func (h *FlowAccessAPIRouter) GetEventsForHeightRange(context context.Context, req *access.GetEventsForHeightRangeRequest) (*access.EventsResponse, error) { - return h.upstream.GetEventsForHeightRange(context, req) -} - -func (h *FlowAccessAPIRouter) GetEventsForBlockIDs(context context.Context, req *access.GetEventsForBlockIDsRequest) (*access.EventsResponse, error) { - return h.upstream.GetEventsForBlockIDs(context, req) -} - -func (h *FlowAccessAPIRouter) GetNetworkParameters(context context.Context, req *access.GetNetworkParametersRequest) (*access.GetNetworkParametersResponse, error) { - return h.AccessAPIServer.GetNetworkParameters(context, req) -} - -func (h *FlowAccessAPIRouter) GetLatestProtocolStateSnapshot(context context.Context, req *access.GetLatestProtocolStateSnapshotRequest) (*access.ProtocolStateSnapshotResponse, error) { - return h.AccessAPIServer.GetLatestProtocolStateSnapshot(context, req) -} - -func (h *FlowAccessAPIRouter) GetExecutionResultForBlockID(context context.Context, req *access.GetExecutionResultForBlockIDRequest) (*access.ExecutionResultForBlockIDResponse, error) { - return h.upstream.GetExecutionResultForBlockID(context, req) -} - -// FlowAccessAPIForwarder forwards all requests to a set of upstream access nodes or observers -type FlowAccessAPIForwarder struct { - lock sync.Mutex - roundRobin int - ids flow.IdentityList - upstream []access.AccessAPIClient - connections []*grpc.ClientConn - timeout time.Duration -} - -// Ping pings the service. It is special in the sense that it responds successful, -// only if all underlying services are ready. -func (h *FlowAccessAPIForwarder) Ping(context context.Context, req *access.PingRequest) (*access.PingResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.Ping(context, req) -} - -func (h *FlowAccessAPIForwarder) GetLatestBlockHeader(context context.Context, req *access.GetLatestBlockHeaderRequest) (*access.BlockHeaderResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetLatestBlockHeader(context, req) -} - -func (h *FlowAccessAPIForwarder) GetBlockHeaderByID(context context.Context, req *access.GetBlockHeaderByIDRequest) (*access.BlockHeaderResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetBlockHeaderByID(context, req) -} - -func (h *FlowAccessAPIForwarder) GetBlockHeaderByHeight(context context.Context, req *access.GetBlockHeaderByHeightRequest) (*access.BlockHeaderResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetBlockHeaderByHeight(context, req) -} - -func (h *FlowAccessAPIForwarder) GetLatestBlock(context context.Context, req *access.GetLatestBlockRequest) (*access.BlockResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetLatestBlock(context, req) -} - -func (h *FlowAccessAPIForwarder) GetBlockByID(context context.Context, req *access.GetBlockByIDRequest) (*access.BlockResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetBlockByID(context, req) -} - -func (h *FlowAccessAPIForwarder) GetBlockByHeight(context context.Context, req *access.GetBlockByHeightRequest) (*access.BlockResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetBlockByHeight(context, req) -} - -func (h *FlowAccessAPIForwarder) GetCollectionByID(context context.Context, req *access.GetCollectionByIDRequest) (*access.CollectionResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetCollectionByID(context, req) -} - -func (h *FlowAccessAPIForwarder) SendTransaction(context context.Context, req *access.SendTransactionRequest) (*access.SendTransactionResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.SendTransaction(context, req) -} - -func (h *FlowAccessAPIForwarder) GetTransaction(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetTransaction(context, req) -} - -func (h *FlowAccessAPIForwarder) GetTransactionResult(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResultResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetTransactionResult(context, req) -} - -func (h *FlowAccessAPIForwarder) GetTransactionResultByIndex(context context.Context, req *access.GetTransactionByIndexRequest) (*access.TransactionResultResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetTransactionResultByIndex(context, req) -} - -func (h *FlowAccessAPIForwarder) GetAccount(context context.Context, req *access.GetAccountRequest) (*access.GetAccountResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetAccount(context, req) -} - -func (h *FlowAccessAPIForwarder) GetAccountAtLatestBlock(context context.Context, req *access.GetAccountAtLatestBlockRequest) (*access.AccountResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetAccountAtLatestBlock(context, req) -} - -func (h *FlowAccessAPIForwarder) GetAccountAtBlockHeight(context context.Context, req *access.GetAccountAtBlockHeightRequest) (*access.AccountResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetAccountAtBlockHeight(context, req) -} - -func (h *FlowAccessAPIForwarder) ExecuteScriptAtLatestBlock(context context.Context, req *access.ExecuteScriptAtLatestBlockRequest) (*access.ExecuteScriptResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.ExecuteScriptAtLatestBlock(context, req) -} - -func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockID(context context.Context, req *access.ExecuteScriptAtBlockIDRequest) (*access.ExecuteScriptResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.ExecuteScriptAtBlockID(context, req) -} - -func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockHeight(context context.Context, req *access.ExecuteScriptAtBlockHeightRequest) (*access.ExecuteScriptResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.ExecuteScriptAtBlockHeight(context, req) -} - -func (h *FlowAccessAPIForwarder) GetEventsForHeightRange(context context.Context, req *access.GetEventsForHeightRangeRequest) (*access.EventsResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetEventsForHeightRange(context, req) -} - -func (h *FlowAccessAPIForwarder) GetEventsForBlockIDs(context context.Context, req *access.GetEventsForBlockIDsRequest) (*access.EventsResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetEventsForBlockIDs(context, req) -} - -func (h *FlowAccessAPIForwarder) GetNetworkParameters(context context.Context, req *access.GetNetworkParametersRequest) (*access.GetNetworkParametersResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetNetworkParameters(context, req) -} - -func (h *FlowAccessAPIForwarder) GetLatestProtocolStateSnapshot(context context.Context, req *access.GetLatestProtocolStateSnapshotRequest) (*access.ProtocolStateSnapshotResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetLatestProtocolStateSnapshot(context, req) -} - -func (h *FlowAccessAPIForwarder) GetExecutionResultForBlockID(context context.Context, req *access.GetExecutionResultForBlockIDRequest) (*access.ExecutionResultForBlockIDResponse, error) { - // This is a passthrough request - upstream, err := h.faultTolerantClient() - if err != nil { - return nil, err - } - return upstream.GetExecutionResultForBlockID(context, req) -} diff --git a/apiproxy/access_api_proxy_test.go b/apiproxy/access_api_proxy_test.go deleted file mode 100644 index 85be5054c09..00000000000 --- a/apiproxy/access_api_proxy_test.go +++ /dev/null @@ -1,272 +0,0 @@ -package apiproxy - -import ( - "context" - "fmt" - "net" - "testing" - "time" - - "github.com/onflow/flow/protobuf/go/flow/access" - "google.golang.org/grpc" - grpcinsecure "google.golang.org/grpc/credentials/insecure" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/grpcutils" -) - -// Methodology -// -// We test the proxy and fall-over logic to reach basic coverage. -// -// * Basic coverage means that all conditional checks happen once but only once. -// * We embrace the simplest adequate solution to reduce engineering cost. -// * Any use cases requiring multiple conditionals exercised in a row are considered ignorable due to cost constraints. - -// TestNetE2E tests the basic unix network first -func TestNetE2E(t *testing.T) { - done := make(chan int) - // Bring up 1st upstream server - charlie1, err := makeFlowLite("/tmp/TestProxyE2E1", done) - if err != nil { - t.Fatal(err) - } - // Wait until proxy call passes - err = callFlowLite("/tmp/TestProxyE2E1") - if err != nil { - t.Fatal(err) - } - // Bring up 2nd upstream server - charlie2, err := makeFlowLite("/tmp/TestProxyE2E2", done) - if err != nil { - t.Fatal(err) - } - // Both proxy calls should pass - err = callFlowLite("/tmp/TestProxyE2E1") - if err != nil { - t.Fatal(err) - } - err = callFlowLite("/tmp/TestProxyE2E2") - if err != nil { - t.Fatal(err) - } - // Stop 1st upstream server - _ = charlie1.Close() - // Proxy call falls through - err = callFlowLite("/tmp/TestProxyE2E1") - if err == nil { - t.Fatal(fmt.Errorf("backend still active after close")) - } - // Stop 2nd upstream server - _ = charlie2.Close() - // System errors out on shut down servers - err = callFlowLite("/tmp/TestProxyE2E1") - if err == nil { - t.Fatal(fmt.Errorf("backend still active after close")) - } - err = callFlowLite("/tmp/TestProxyE2E2") - if err == nil { - t.Fatal(fmt.Errorf("backend still active after close")) - } - // wait for all - <-done - <-done -} - -// TestgRPCE2E tests whether gRPC works -func TestGRPCE2E(t *testing.T) { - done := make(chan int) - // Bring up 1st upstream server - charlie1, _, err := newFlowLite("unix", "/tmp/TestProxyE2E1", done) - if err != nil { - t.Fatal(err) - } - // Wait until proxy call passes - err = openFlowLite("/tmp/TestProxyE2E1") - if err != nil { - t.Fatal(err) - } - // Bring up 2nd upstream server - charlie2, _, err := newFlowLite("unix", "/tmp/TestProxyE2E2", done) - if err != nil { - t.Fatal(err) - } - // Both proxy calls should pass - err = openFlowLite("/tmp/TestProxyE2E1") - if err != nil { - t.Fatal(err) - } - err = openFlowLite("/tmp/TestProxyE2E2") - if err != nil { - t.Fatal(err) - } - // Stop 1st upstream server - charlie1.Stop() - // Proxy call falls through - err = openFlowLite("/tmp/TestProxyE2E1") - if err == nil { - t.Fatal(fmt.Errorf("backend still active after close")) - } - // Stop 2nd upstream server - charlie2.Stop() - // System errors out on shut down servers - err = openFlowLite("/tmp/TestProxyE2E1") - if err == nil { - t.Fatal(fmt.Errorf("backend still active after close")) - } - err = openFlowLite("/tmp/TestProxyE2E2") - if err == nil { - t.Fatal(fmt.Errorf("backend still active after close")) - } - // wait for all - <-done - <-done -} - -// TestNewFlowCachedAccessAPIProxy tests the round robin end to end -func TestNewFlowCachedAccessAPIProxy(t *testing.T) { - done := make(chan int) - - // Bring up 1st upstream server - charlie1, _, err := newFlowLite("tcp", "127.0.0.1:11634", done) - if err != nil { - t.Fatal(err) - } - - // Prepare a proxy that fails due to the second connection being idle - l := flow.IdentityList{{Address: "127.0.0.1:11634"}, {Address: "127.0.0.1:11635"}} - c := FlowAccessAPIForwarder{} - err = c.setFlowAccessAPI(l, time.Second) - if err == nil { - t.Fatal(fmt.Errorf("should not start with one connection ready")) - } - - // Bring up 2nd upstream server - charlie2, _, err := newFlowLite("tcp", "127.0.0.1:11635", done) - if err != nil { - t.Fatal(err) - } - - background := context.Background() - - // Prepare a proxy - l = flow.IdentityList{{Address: "127.0.0.1:11634"}, {Address: "127.0.0.1:11635"}} - c = FlowAccessAPIForwarder{} - err = c.setFlowAccessAPI(l, time.Second) - if err != nil { - t.Fatal(err) - } - - // Wait until proxy call passes - _, err = c.Ping(background, &access.PingRequest{}) - if err != nil { - t.Fatal(err) - } - - // Wait until proxy call passes - _, err = c.Ping(background, &access.PingRequest{}) - if err != nil { - t.Fatal(err) - } - - // Wait until proxy call passes - _, err = c.Ping(background, &access.PingRequest{}) - if err != nil { - t.Fatal(err) - } - - charlie1.Stop() - charlie2.Stop() - - // Wait until proxy call fails - _, err = c.Ping(background, &access.PingRequest{}) - if err == nil { - t.Fatal(fmt.Errorf("should fail on no connections")) - } - - <-done - <-done -} - -func makeFlowLite(address string, done chan int) (net.Listener, error) { - l, err := net.Listen("unix", address) - if err != nil { - return nil, err - } - - go func(done chan int) { - for { - c, err := l.Accept() - if err != nil { - break - } - - b := make([]byte, 3) - _, _ = c.Read(b) - _, _ = c.Write(b) - _ = c.Close() - } - done <- 1 - }(done) - return l, err -} - -func callFlowLite(address string) error { - c, err := net.Dial("unix", address) - if err != nil { - return err - } - o := []byte("abc") - _, _ = c.Write(o) - i := make([]byte, 3) - _, _ = c.Read(i) - if string(o) != string(i) { - return fmt.Errorf("no match") - } - _ = c.Close() - _ = MockFlowAccessAPI{} - return err -} - -func newFlowLite(network string, address string, done chan int) (*grpc.Server, *net.Listener, error) { - l, err := net.Listen(network, address) - if err != nil { - return nil, nil, err - } - s := grpc.NewServer() - go func(done chan int) { - access.RegisterAccessAPIServer(s, MockFlowAccessAPI{}) - _ = s.Serve(l) - done <- 1 - }(done) - return s, &l, nil -} - -func openFlowLite(address string) error { - c, err := grpc.Dial( - "unix://"+address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), - grpc.WithTransportCredentials(grpcinsecure.NewCredentials())) - if err != nil { - return err - } - a := access.NewAccessAPIClient(c) - - background := context.Background() - - _, err = a.Ping(background, &access.PingRequest{}) - if err != nil { - return err - } - - return nil -} - -type MockFlowAccessAPI struct { - access.AccessAPIServer -} - -// Ping is used to check if the access node is alive and healthy. -func (p MockFlowAccessAPI) Ping(context.Context, *access.PingRequest) (*access.PingResponse, error) { - return &access.PingResponse{}, nil -} diff --git a/cmd/Dockerfile b/cmd/Dockerfile index 473effbef9b..d9d7800546c 100644 --- a/cmd/Dockerfile +++ b/cmd/Dockerfile @@ -3,7 +3,7 @@ #################################### ## (1) Setup the build environment -FROM golang:1.19-bullseye AS build-setup +FROM golang:1.20-bullseye AS build-setup RUN apt-get update RUN apt-get -y install cmake zip @@ -19,10 +19,13 @@ ARG TARGET ARG COMMIT ARG VERSION +ENV GOPRIVATE= + COPY . . RUN --mount=type=cache,sharing=locked,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=secret,id=git_creds,dst=/root/.netrc \ make crypto_setup_gopath #################################### @@ -39,6 +42,7 @@ ARG TAGS="relic,netgo" # https://github.com/golang/go/issues/27719#issuecomment-514747274 RUN --mount=type=cache,sharing=locked,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=secret,id=git_creds,dst=/root/.netrc \ CGO_ENABLED=1 GOOS=linux go build --tags "${TAGS}" -ldflags "-extldflags -static \ -X 'github.com/onflow/flow-go/cmd/build.commit=${COMMIT}' -X 'github.com/onflow/flow-go/cmd/build.semver=${VERSION}'" \ -o ./app ${TARGET} @@ -67,7 +71,7 @@ RUN --mount=type=ssh \ RUN chmod a+x /app/app ## (4) Add the statically linked debug binary to a distroless image configured for debugging -FROM golang:1.19-bullseye as debug +FROM golang:1.20-bullseye as debug RUN go install github.com/go-delve/delve/cmd/dlv@latest diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 1c9e058caef..f083aaed0fd 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -37,8 +37,10 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/access/ingestion" pingeng "github.com/onflow/flow-go/engine/access/ping" + "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" + rpcConnection "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/access/state_stream" followereng "github.com/onflow/flow-go/engine/common/follower" "github.com/onflow/flow-go/engine/common/requester" @@ -48,33 +50,36 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/blobs" "github.com/onflow/flow-go/module/chainsync" - modulecompliance "github.com/onflow/flow-go/module/compliance" "github.com/onflow/flow-go/module/executiondatasync/execution_data" + execdatacache "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" finalizer "github.com/onflow/flow-go/module/finalizer/consensus" + "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/module/id" + "github.com/onflow/flow-go/module/mempool/herocache" "github.com/onflow/flow-go/module/mempool/stdmap" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/metrics/unstaked" "github.com/onflow/flow-go/module/state_synchronization" edrequester "github.com/onflow/flow-go/module/state_synchronization/requester" "github.com/onflow/flow-go/network" + alspmgr "github.com/onflow/flow-go/network/alsp/manager" netcache "github.com/onflow/flow-go/network/cache" "github.com/onflow/flow-go/network/channels" cborcodec "github.com/onflow/flow-go/network/codec/cbor" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/blob" "github.com/onflow/flow-go/network/p2p/cache" + "github.com/onflow/flow-go/network/p2p/conduit" "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/middleware" "github.com/onflow/flow-go/network/p2p/p2pbuilder" - "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" relaynet "github.com/onflow/flow-go/network/relay" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/topology" "github.com/onflow/flow-go/network/validator" "github.com/onflow/flow-go/state/protocol" @@ -143,19 +148,28 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { collectionGRPCPort: 9000, executionGRPCPort: 9000, rpcConf: rpc.Config{ - UnsecureGRPCListenAddr: "0.0.0.0:9000", - SecureGRPCListenAddr: "0.0.0.0:9001", - HTTPListenAddr: "0.0.0.0:8000", - RESTListenAddr: "", - CollectionAddr: "", - HistoricalAccessAddrs: "", - CollectionClientTimeout: 3 * time.Second, - ExecutionClientTimeout: 3 * time.Second, - ConnectionPoolSize: backend.DefaultConnectionPoolSize, - MaxHeightRange: backend.DefaultMaxHeightRange, - PreferredExecutionNodeIDs: nil, - FixedExecutionNodeIDs: nil, - MaxMsgSize: grpcutils.DefaultMaxMsgSize, + UnsecureGRPCListenAddr: "0.0.0.0:9000", + SecureGRPCListenAddr: "0.0.0.0:9001", + HTTPListenAddr: "0.0.0.0:8000", + RESTListenAddr: "", + CollectionAddr: "", + HistoricalAccessAddrs: "", + BackendConfig: backend.Config{ + CollectionClientTimeout: 3 * time.Second, + ExecutionClientTimeout: 3 * time.Second, + ConnectionPoolSize: backend.DefaultConnectionPoolSize, + MaxHeightRange: backend.DefaultMaxHeightRange, + PreferredExecutionNodeIDs: nil, + FixedExecutionNodeIDs: nil, + ArchiveAddressList: nil, + CircuitBreakerConfig: rpcConnection.CircuitBreakerConfig{ + Enabled: false, + RestoreTimeout: 60 * time.Second, + MaxFailures: 5, + MaxRequests: 1, + }, + }, + MaxMsgSize: grpcutils.DefaultMaxMsgSize, }, stateStreamConf: state_stream.Config{ MaxExecutionDataMsgSize: grpcutils.DefaultMaxMsgSize, @@ -164,6 +178,7 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { ClientSendBufferSize: state_stream.DefaultSendBufferSize, MaxGlobalStreams: state_stream.DefaultMaxGlobalStreams, EventFilterConfig: state_stream.DefaultEventFilterConfig, + ResponseLimit: state_stream.DefaultResponseLimit, }, stateStreamFilterConf: nil, ExecutionNodeAddress: "localhost:9000", @@ -205,24 +220,25 @@ type FlowAccessNodeBuilder struct { FollowerState protocol.FollowerState SyncCore *chainsync.Core RpcEng *rpc.Engine - FinalizationDistributor *consensuspubsub.FinalizationDistributor - FinalizedHeader *synceng.FinalizedHeaderCache + FollowerDistributor *consensuspubsub.FollowerDistributor CollectionRPC access.AccessAPIClient TransactionTimings *stdmap.TransactionTimings CollectionsToMarkFinalized *stdmap.Times CollectionsToMarkExecuted *stdmap.Times BlocksToMarkExecuted *stdmap.Times - TransactionMetrics module.TransactionMetrics + TransactionMetrics *metrics.TransactionCollector + RestMetrics *metrics.RestCollector AccessMetrics module.AccessMetrics PingMetrics module.PingMetrics Committee hotstuff.DynamicCommittee - Finalized *flow.Header + Finalized *flow.Header // latest finalized block that the node knows of at startup time Pending []*flow.Header FollowerCore module.HotStuffFollower Validator hotstuff.Validator ExecutionDataDownloader execution_data.Downloader ExecutionDataRequester state_synchronization.ExecutionDataRequester ExecutionDataStore execution_data.ExecutionDataStore + ExecutionDataCache *execdatacache.ExecutionDataCache // The sync engine participants provider is the libp2p peer store for the access node // which is not available until after the network has started. @@ -235,6 +251,11 @@ type FlowAccessNodeBuilder struct { FollowerEng *followereng.ComplianceEngine SyncEng *synceng.Engine StateStreamEng *state_stream.Engine + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer + stateStreamGrpcServer *grpcserver.GrpcServer } func (builder *FlowAccessNodeBuilder) buildFollowerState() *FlowAccessNodeBuilder { @@ -313,12 +334,11 @@ func (builder *FlowAccessNodeBuilder) buildFollowerCore() *FlowAccessNodeBuilder followerCore, err := consensus.NewFollower( node.Logger, - builder.Committee, + node.Metrics.Mempool, node.Storage.Headers, final, - verifier, - builder.FinalizationDistributor, - node.RootBlock.Header, + builder.FollowerDistributor, + node.FinalizedRootBlock.Header, node.RootQC, builder.Finalized, builder.Pending, @@ -345,7 +365,7 @@ func (builder *FlowAccessNodeBuilder) buildFollowerEngine() *FlowAccessNodeBuild node.Logger, node.Metrics.Mempool, heroCacheCollector, - builder.FinalizationDistributor, + builder.FollowerDistributor, builder.FollowerState, builder.FollowerCore, builder.Validator, @@ -364,11 +384,12 @@ func (builder *FlowAccessNodeBuilder) buildFollowerEngine() *FlowAccessNodeBuild node.Storage.Headers, builder.Finalized, core, - followereng.WithComplianceConfigOpt(modulecompliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold)), + node.ComplianceConfig, ) if err != nil { return nil, fmt.Errorf("could not create follower engine: %w", err) } + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.FollowerEng.OnFinalizedBlock) return builder.FollowerEng, nil }) @@ -376,20 +397,6 @@ func (builder *FlowAccessNodeBuilder) buildFollowerEngine() *FlowAccessNodeBuild return builder } -func (builder *FlowAccessNodeBuilder) buildFinalizedHeader() *FlowAccessNodeBuilder { - builder.Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - finalizedHeader, err := synceng.NewFinalizedHeaderCache(node.Logger, node.State, builder.FinalizationDistributor) - if err != nil { - return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) - } - builder.FinalizedHeader = finalizedHeader - - return builder.FinalizedHeader, nil - }) - - return builder -} - func (builder *FlowAccessNodeBuilder) buildSyncEngine() *FlowAccessNodeBuilder { builder.Component("sync engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { sync, err := synceng.New( @@ -397,16 +404,17 @@ func (builder *FlowAccessNodeBuilder) buildSyncEngine() *FlowAccessNodeBuilder { node.Metrics.Engine, node.Network, node.Me, + node.State, node.Storage.Blocks, builder.FollowerEng, builder.SyncCore, - builder.FinalizedHeader, builder.SyncEngineParticipantsProviderFactory(), ) if err != nil { return nil, fmt.Errorf("could not create synchronization engine: %w", err) } builder.SyncEng = sync + builder.FollowerDistributor.AddFinalizationConsumer(sync) return builder.SyncEng, nil }) @@ -422,7 +430,6 @@ func (builder *FlowAccessNodeBuilder) BuildConsensusFollower() *FlowAccessNodeBu buildLatestHeader(). buildFollowerCore(). buildFollowerEngine(). - buildFinalizedHeader(). buildSyncEngine() return builder @@ -435,6 +442,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN var processedNotifications storage.ConsumerProgress var bsDependable *module.ProxiedReadyDoneAware var execDataDistributor *edrequester.ExecutionDataDistributor + var execDataCacheBackend *herocache.BlockExecutionData builder. AdminCommand("read-execution-data", func(config *cmd.NodeConfig) commands.AdminCommand { @@ -513,10 +521,10 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN Component("execution data requester", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { // Validation of the start block height needs to be done after loading state if builder.executionDataStartHeight > 0 { - if builder.executionDataStartHeight <= builder.RootBlock.Header.Height { + if builder.executionDataStartHeight <= builder.FinalizedRootBlock.Header.Height { return nil, fmt.Errorf( "execution data start block height (%d) must be greater than the root block height (%d)", - builder.executionDataStartHeight, builder.RootBlock.Header.Height) + builder.executionDataStartHeight, builder.FinalizedRootBlock.Header.Height) } latestSeal, err := builder.State.Sealed().Head() @@ -538,25 +546,40 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN // requester expects the initial last processed height, which is the first height - 1 builder.executionDataConfig.InitialBlockHeight = builder.executionDataStartHeight - 1 } else { - builder.executionDataConfig.InitialBlockHeight = builder.RootBlock.Header.Height + builder.executionDataConfig.InitialBlockHeight = builder.FinalizedRootBlock.Header.Height } execDataDistributor = edrequester.NewExecutionDataDistributor() + var heroCacheCollector module.HeroCacheMetrics = metrics.NewNoopCollector() + if builder.HeroCacheMetricsEnable { + heroCacheCollector = metrics.AccessNodeExecutionDataCacheMetrics(builder.MetricsRegisterer) + } + + execDataCacheBackend = herocache.NewBlockExecutionData(builder.stateStreamConf.ExecutionDataCacheSize, builder.Logger, heroCacheCollector) + // Execution Data cache with a downloader as the backend. This is used by the requester + // to download and cache execution data for each block. + executionDataCache := execdatacache.NewExecutionDataCache( + builder.ExecutionDataDownloader, + builder.Storage.Headers, + builder.Storage.Seals, + builder.Storage.Results, + execDataCacheBackend, + ) + builder.ExecutionDataRequester = edrequester.New( builder.Logger, metrics.NewExecutionDataRequesterCollector(), builder.ExecutionDataDownloader, + executionDataCache, processedBlockHeight, processedNotifications, builder.State, builder.Storage.Headers, - builder.Storage.Results, - builder.Storage.Seals, builder.executionDataConfig, ) - builder.FinalizationDistributor.AddOnBlockFinalizedConsumer(builder.ExecutionDataRequester.OnBlockFinalized) + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.ExecutionDataRequester.OnBlockFinalized) builder.ExecutionDataRequester.AddOnExecutionDataReceivedConsumer(execDataDistributor.OnExecutionDataReceived) return builder.ExecutionDataRequester, nil @@ -576,23 +599,36 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN } builder.stateStreamConf.RpcMetricsEnabled = builder.rpcMetricsEnabled - var heroCacheCollector module.HeroCacheMetrics = metrics.NewNoopCollector() - if builder.HeroCacheMetricsEnable { - heroCacheCollector = metrics.AccessNodeExecutionDataCacheMetrics(builder.MetricsRegisterer) + // Execution Data cache that uses a blobstore as the backend (instead of a downloader) + // This ensures that it simply returns a not found error if the blob doesn't exist + // instead of attempting to download it from the network. It shares a cache backend instance + // with the requester's implementation. + executionDataCache := execdatacache.NewExecutionDataCache( + builder.ExecutionDataStore, + builder.Storage.Headers, + builder.Storage.Seals, + builder.Storage.Results, + execDataCacheBackend, + ) + + highestAvailableHeight, err := builder.ExecutionDataRequester.HighestConsecutiveHeight() + if err != nil { + return nil, fmt.Errorf("could not get highest consecutive height: %w", err) } stateStreamEng, err := state_stream.NewEng( node.Logger, builder.stateStreamConf, builder.ExecutionDataStore, + executionDataCache, node.State, node.Storage.Headers, node.Storage.Seals, node.Storage.Results, node.RootChainID, - builder.apiRatelimits, - builder.apiBurstlimits, - heroCacheCollector, + builder.executionDataConfig.InitialBlockHeight, + highestAvailableHeight, + builder.stateStreamGrpcServer, ) if err != nil { return nil, fmt.Errorf("could not create state stream engine: %w", err) @@ -609,12 +645,12 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN } func FlowAccessNode(nodeBuilder *cmd.FlowNodeBuilder) *FlowAccessNodeBuilder { - dist := consensuspubsub.NewFinalizationDistributor() - dist.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + dist := consensuspubsub.NewFollowerDistributor() + dist.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) return &FlowAccessNodeBuilder{ - AccessNodeConfig: DefaultAccessNodeConfig(), - FlowNodeBuilder: nodeBuilder, - FinalizationDistributor: dist, + AccessNodeConfig: DefaultAccessNodeConfig(), + FlowNodeBuilder: nodeBuilder, + FollowerDistributor: dist, } } @@ -640,14 +676,15 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { flags.StringVar(&builder.rpcConf.RESTListenAddr, "rest-addr", defaultConfig.rpcConf.RESTListenAddr, "the address the REST server listens on (if empty the REST server will not be started)") flags.StringVarP(&builder.rpcConf.CollectionAddr, "static-collection-ingress-addr", "", defaultConfig.rpcConf.CollectionAddr, "the address (of the collection node) to send transactions to") flags.StringVarP(&builder.ExecutionNodeAddress, "script-addr", "s", defaultConfig.ExecutionNodeAddress, "the address (of the execution node) forward the script to") + flags.StringSliceVar(&builder.rpcConf.BackendConfig.ArchiveAddressList, "archive-address-list", defaultConfig.rpcConf.BackendConfig.ArchiveAddressList, "the list of address of the archive node to forward the script queries to") flags.StringVarP(&builder.rpcConf.HistoricalAccessAddrs, "historical-access-addr", "", defaultConfig.rpcConf.HistoricalAccessAddrs, "comma separated rpc addresses for historical access nodes") - flags.DurationVar(&builder.rpcConf.CollectionClientTimeout, "collection-client-timeout", defaultConfig.rpcConf.CollectionClientTimeout, "grpc client timeout for a collection node") - flags.DurationVar(&builder.rpcConf.ExecutionClientTimeout, "execution-client-timeout", defaultConfig.rpcConf.ExecutionClientTimeout, "grpc client timeout for an execution node") - flags.UintVar(&builder.rpcConf.ConnectionPoolSize, "connection-pool-size", defaultConfig.rpcConf.ConnectionPoolSize, "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") + flags.DurationVar(&builder.rpcConf.BackendConfig.CollectionClientTimeout, "collection-client-timeout", defaultConfig.rpcConf.BackendConfig.CollectionClientTimeout, "grpc client timeout for a collection node") + flags.DurationVar(&builder.rpcConf.BackendConfig.ExecutionClientTimeout, "execution-client-timeout", defaultConfig.rpcConf.BackendConfig.ExecutionClientTimeout, "grpc client timeout for an execution node") + flags.UintVar(&builder.rpcConf.BackendConfig.ConnectionPoolSize, "connection-pool-size", defaultConfig.rpcConf.BackendConfig.ConnectionPoolSize, "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") flags.UintVar(&builder.rpcConf.MaxMsgSize, "rpc-max-message-size", grpcutils.DefaultMaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") - flags.UintVar(&builder.rpcConf.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.MaxHeightRange, "maximum size for height range requests") - flags.StringSliceVar(&builder.rpcConf.PreferredExecutionNodeIDs, "preferred-execution-node-ids", defaultConfig.rpcConf.PreferredExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call e.g. b4a4dbdcd443d...,fb386a6a... etc.") - flags.StringSliceVar(&builder.rpcConf.FixedExecutionNodeIDs, "fixed-execution-node-ids", defaultConfig.rpcConf.FixedExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call if no matching preferred execution id is found e.g. b4a4dbdcd443d...,fb386a6a... etc.") + flags.UintVar(&builder.rpcConf.BackendConfig.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.BackendConfig.MaxHeightRange, "maximum size for height range requests") + flags.StringSliceVar(&builder.rpcConf.BackendConfig.PreferredExecutionNodeIDs, "preferred-execution-node-ids", defaultConfig.rpcConf.BackendConfig.PreferredExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call e.g. b4a4dbdcd443d...,fb386a6a... etc.") + flags.StringSliceVar(&builder.rpcConf.BackendConfig.FixedExecutionNodeIDs, "fixed-execution-node-ids", defaultConfig.rpcConf.BackendConfig.FixedExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call if no matching preferred execution id is found e.g. b4a4dbdcd443d...,fb386a6a... etc.") flags.BoolVar(&builder.logTxTimeToFinalized, "log-tx-time-to-finalized", defaultConfig.logTxTimeToFinalized, "log transaction time to finalized") flags.BoolVar(&builder.logTxTimeToExecuted, "log-tx-time-to-executed", defaultConfig.logTxTimeToExecuted, "log transaction time to executed") flags.BoolVar(&builder.logTxTimeToFinalizedExecuted, "log-tx-time-to-finalized-executed", defaultConfig.logTxTimeToFinalizedExecuted, "log transaction time to finalized and executed") @@ -659,7 +696,10 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { flags.StringToIntVar(&builder.apiBurstlimits, "api-burst-limits", defaultConfig.apiBurstlimits, "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") flags.BoolVar(&builder.supportsObserver, "supports-observer", defaultConfig.supportsObserver, "true if this staked access node supports observer or follower connections") flags.StringVar(&builder.PublicNetworkConfig.BindAddress, "public-network-address", defaultConfig.PublicNetworkConfig.BindAddress, "staked access node's public network bind address") - + flags.BoolVar(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.Enabled, "circuit-breaker-enabled", defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.Enabled, "specifies whether the circuit breaker is enabled for collection and execution API clients.") + flags.DurationVar(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.RestoreTimeout, "circuit-breaker-restore-timeout", defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.RestoreTimeout, "duration after which the circuit breaker will restore the connection to the client after closing it due to failures. Default value is 60s") + flags.Uint32Var(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxFailures, "circuit-breaker-max-failures", defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.MaxFailures, "maximum number of failed calls to the client that will cause the circuit breaker to close the connection. Default value is 5") + flags.Uint32Var(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests, "circuit-breaker-max-requests", defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests, "maximum number of requests to check if connection restored after timeout. Default value is 1") // ExecutionDataRequester config flags.BoolVar(&builder.executionDataSyncEnabled, "execution-data-sync-enabled", defaultConfig.executionDataSyncEnabled, "whether to enable the execution data sync protocol") flags.StringVar(&builder.executionDataDir, "execution-data-dir", defaultConfig.executionDataDir, "directory to use for Execution Data database") @@ -674,9 +714,10 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { flags.Uint32Var(&builder.stateStreamConf.ExecutionDataCacheSize, "execution-data-cache-size", defaultConfig.stateStreamConf.ExecutionDataCacheSize, "block execution data cache size") flags.Uint32Var(&builder.stateStreamConf.MaxGlobalStreams, "state-stream-global-max-streams", defaultConfig.stateStreamConf.MaxGlobalStreams, "global maximum number of concurrent streams") flags.UintVar(&builder.stateStreamConf.MaxExecutionDataMsgSize, "state-stream-max-message-size", defaultConfig.stateStreamConf.MaxExecutionDataMsgSize, "maximum size for a gRPC message containing block execution data") + flags.StringToIntVar(&builder.stateStreamFilterConf, "state-stream-event-filter-limits", defaultConfig.stateStreamFilterConf, "event filter limits for ExecutionData SubscribeEvents API e.g. EventTypes=100,Addresses=100,Contracts=100 etc.") flags.DurationVar(&builder.stateStreamConf.ClientSendTimeout, "state-stream-send-timeout", defaultConfig.stateStreamConf.ClientSendTimeout, "maximum wait before timing out while sending a response to a streaming client e.g. 30s") flags.UintVar(&builder.stateStreamConf.ClientSendBufferSize, "state-stream-send-buffer-size", defaultConfig.stateStreamConf.ClientSendBufferSize, "maximum number of responses to buffer within a stream") - flags.StringToIntVar(&builder.stateStreamFilterConf, "state-stream-event-filter-limits", defaultConfig.stateStreamFilterConf, "event filter limits for ExecutionData SubscribeEvents API e.g. EventTypes=100,Addresses=100,Contracts=100 etc.") + flags.Float64Var(&builder.stateStreamConf.ResponseLimit, "state-stream-response-limit", defaultConfig.stateStreamConf.ResponseLimit, "max number of responses per second to send over streaming endpoints. this helps manage resources consumed by each client querying data not in the cache e.g. 3 or 0.5. 0 means no limit") }).ValidateFlags(func() error { if builder.supportsObserver && (builder.PublicNetworkConfig.BindAddress == cmd.NotSet || builder.PublicNetworkConfig.BindAddress == "") { return errors.New("public-network-address must be set if supports-observer is true") @@ -718,6 +759,20 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { return errors.New("state-stream-event-filter-limits may only contain the keys EventTypes, Addresses, Contracts") } } + if builder.stateStreamConf.ResponseLimit < 0 { + return errors.New("state-stream-response-limit must be greater than or equal to 0") + } + } + if builder.rpcConf.BackendConfig.CircuitBreakerConfig.Enabled { + if builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxFailures == 0 { + return errors.New("circuit-breaker-max-failures must be greater than 0") + } + if builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests == 0 { + return errors.New("circuit-breaker-max-requests must be greater than 0") + } + if builder.rpcConf.BackendConfig.CircuitBreakerConfig.RestoreTimeout <= 0 { + return errors.New("circuit-breaker-restore-timeout must be greater than 0") + } } return nil @@ -733,9 +788,8 @@ func (builder *FlowAccessNodeBuilder) initNetwork(nodeID module.Local, topology network.Topology, receiveCache *netcache.ReceiveCache, ) (*p2p.Network, error) { - // creates network instance - net, err := p2p.NewNetwork(&p2p.NetworkParameters{ + net, err := p2p.NewNetwork(&p2p.NetworkConfig{ Logger: builder.Logger, Codec: cborcodec.NewCodec(), Me: nodeID, @@ -745,6 +799,17 @@ func (builder *FlowAccessNodeBuilder) initNetwork(nodeID module.Local, Metrics: networkMetrics, IdentityProvider: builder.IdentityProvider, ReceiveCache: receiveCache, + ConduitFactory: conduit.NewDefaultConduitFactory(), + AlspCfg: &alspmgr.MisbehaviorReportManagerConfig{ + Logger: builder.Logger, + SpamRecordCacheSize: builder.FlowConfig.NetworkConfig.AlspConfig.SpamRecordCacheSize, + SpamReportQueueSize: builder.FlowConfig.NetworkConfig.AlspConfig.SpamReportQueueSize, + DisablePenalty: builder.FlowConfig.NetworkConfig.AlspConfig.DisablePenalty, + HeartBeatInterval: builder.FlowConfig.NetworkConfig.AlspConfig.HearBeatInterval, + AlspMetrics: builder.Metrics.Network, + NetworkType: network.PublicNetwork, + HeroCacheMetricsFactory: builder.HeroCacheMetricsFactory(), + }, }) if err != nil { return nil, fmt.Errorf("could not initialize network: %w", err) @@ -776,11 +841,11 @@ func (builder *FlowAccessNodeBuilder) InitIDProviders() { } builder.IDTranslator = translator.NewHierarchicalIDTranslator(idCache, translator.NewPublicNetworkIDTranslator()) - builder.NodeDisallowListDistributor = cmd.BuildDisallowListNotificationDisseminator(builder.DisallowListNotificationCacheSize, builder.MetricsRegisterer, builder.Logger, builder.MetricsEnabled) - // The following wrapper allows to disallow-list byzantine nodes via an admin command: // the wrapper overrides the 'Ejected' flag of disallow-listed nodes to true - disallowListWrapper, err := cache.NewNodeBlocklistWrapper(idCache, node.DB, builder.NodeDisallowListDistributor) + disallowListWrapper, err := cache.NewNodeDisallowListWrapper(idCache, node.DB, func() network.DisallowListNotificationConsumer { + return builder.Middleware + }) if err != nil { return fmt.Errorf("could not initialize NodeBlockListWrapper: %w", err) } @@ -788,9 +853,9 @@ func (builder *FlowAccessNodeBuilder) InitIDProviders() { // register the wrapper for dynamic configuration via admin command err = node.ConfigManager.RegisterIdentifierListConfig("network-id-provider-blocklist", - disallowListWrapper.GetBlocklist, disallowListWrapper.Update) + disallowListWrapper.GetDisallowList, disallowListWrapper.Update) if err != nil { - return fmt.Errorf("failed to register blocklist with config manager: %w", err) + return fmt.Errorf("failed to register disallow-list wrapper with config manager: %w", err) } builder.SyncEngineParticipantsProviderFactory = func() module.IdentifierProvider { @@ -805,11 +870,6 @@ func (builder *FlowAccessNodeBuilder) InitIDProviders() { } return nil }) - - builder.Component("disallow list notification distributor", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - // distributor is returned as a component to be started and stopped. - return builder.NodeDisallowListDistributor, nil - }) } func (builder *FlowAccessNodeBuilder) Initialize() error { @@ -878,7 +938,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.rpcConf.CollectionAddr, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(builder.rpcConf.MaxMsgSize))), grpc.WithTransportCredentials(insecure.NewCredentials()), - backend.WithClientUnaryInterceptor(builder.rpcConf.CollectionClientTimeout)) + rpcConnection.WithClientTimeoutOption(builder.rpcConf.BackendConfig.CollectionClientTimeout)) if err != nil { return err } @@ -925,12 +985,29 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return err }). Module("transaction metrics", func(node *cmd.NodeConfig) error { - builder.TransactionMetrics = metrics.NewTransactionCollector(builder.TransactionTimings, node.Logger, builder.logTxTimeToFinalized, - builder.logTxTimeToExecuted, builder.logTxTimeToFinalizedExecuted) + builder.TransactionMetrics = metrics.NewTransactionCollector( + node.Logger, + builder.TransactionTimings, + builder.logTxTimeToFinalized, + builder.logTxTimeToExecuted, + builder.logTxTimeToFinalizedExecuted, + ) + return nil + }). + Module("rest metrics", func(node *cmd.NodeConfig) error { + m, err := metrics.NewRestCollector(routes.URLToRoute, node.MetricsRegisterer) + if err != nil { + return err + } + builder.RestMetrics = m return nil }). Module("access metrics", func(node *cmd.NodeConfig) error { - builder.AccessMetrics = metrics.NewAccessCollector() + builder.AccessMetrics = metrics.NewAccessCollector( + metrics.WithTransactionMetrics(builder.TransactionMetrics), + metrics.WithBackendScriptsMetrics(builder.TransactionMetrics), + metrics.WithRestMetrics(builder.RestMetrics), + ) return nil }). Module("ping metrics", func(node *cmd.NodeConfig) error { @@ -947,11 +1024,73 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.rpcConf.TransportCredentials = credentials.NewTLS(tlsConfig) return nil }). - Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - engineBuilder, err := rpc.NewBuilder( + Module("creating grpc servers", func(node *cmd.NodeConfig) error { + builder.secureGrpcServer = grpcserver.NewGrpcServerBuilder( + node.Logger, + builder.rpcConf.SecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits, + grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)).Build() + + builder.stateStreamGrpcServer = grpcserver.NewGrpcServerBuilder( node.Logger, + builder.stateStreamConf.ListenAddr, + builder.stateStreamConf.MaxExecutionDataMsgSize, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits, + grpcserver.WithStreamInterceptor()).Build() + + if builder.rpcConf.UnsecureGRPCListenAddr != builder.stateStreamConf.ListenAddr { + builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + builder.rpcConf.UnsecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits).Build() + } else { + builder.unsecureGrpcServer = builder.stateStreamGrpcServer + } + + return nil + }). + Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + config := builder.rpcConf + backendConfig := config.BackendConfig + accessMetrics := builder.AccessMetrics + + backendCache, cacheSize, err := backend.NewCache(node.Logger, + accessMetrics, + backendConfig.ConnectionPoolSize) + if err != nil { + return nil, fmt.Errorf("could not initialize backend cache: %w", err) + } + + var connBackendCache *rpcConnection.Cache + if backendCache != nil { + connBackendCache = rpcConnection.NewCache(backendCache, int(cacheSize)) + } + + connFactory := &rpcConnection.ConnectionFactoryImpl{ + CollectionGRPCPort: builder.collectionGRPCPort, + ExecutionGRPCPort: builder.executionGRPCPort, + CollectionNodeGRPCTimeout: backendConfig.CollectionClientTimeout, + ExecutionNodeGRPCTimeout: backendConfig.ExecutionClientTimeout, + AccessMetrics: accessMetrics, + Log: node.Logger, + Manager: rpcConnection.NewManager( + connBackendCache, + node.Logger, + accessMetrics, + config.MaxMsgSize, + backendConfig.CircuitBreakerConfig, + ), + } + + backend := backend.New( node.State, - builder.rpcConf, builder.CollectionRPC, builder.HistoricalAccessRPCs, node.Storage.Blocks, @@ -961,14 +1100,29 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { node.Storage.Receipts, node.Storage.Results, node.RootChainID, - builder.TransactionMetrics, builder.AccessMetrics, - builder.collectionGRPCPort, - builder.executionGRPCPort, + connFactory, builder.retryEnabled, + backendConfig.MaxHeightRange, + backendConfig.PreferredExecutionNodeIDs, + backendConfig.FixedExecutionNodeIDs, + node.Logger, + backend.DefaultSnapshotHistoryLimit, + backendConfig.ArchiveAddressList, + backendConfig.CircuitBreakerConfig.Enabled) + + engineBuilder, err := rpc.NewBuilder( + node.Logger, + node.State, + config, + node.RootChainID, + builder.AccessMetrics, builder.rpcMetricsEnabled, - builder.apiRatelimits, - builder.apiBurstlimits, + builder.Me, + backend, + backend, + builder.secureGrpcServer, + builder.unsecureGrpcServer, ) if err != nil { return nil, err @@ -981,6 +1135,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { if err != nil { return nil, err } + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.RpcEng.OnFinalizedBlock) return builder.RpcEng, nil }). @@ -1013,17 +1168,16 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { node.Storage.Transactions, node.Storage.Results, node.Storage.Receipts, - builder.TransactionMetrics, + builder.AccessMetrics, builder.CollectionsToMarkFinalized, builder.CollectionsToMarkExecuted, builder.BlocksToMarkExecuted, - builder.RpcEng, ) if err != nil { return nil, err } builder.RequestEng.WithHandle(builder.IngestEng.OnCollection) - builder.FinalizationDistributor.AddConsumer(builder.IngestEng) + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.IngestEng.OnFinalizedBlock) return builder.IngestEng, nil }). @@ -1041,14 +1195,14 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { unstaked.NewUnstakedEngineCollector(node.Metrics.Engine), builder.AccessNodeConfig.PublicNetworkConfig.Network, node.Me, + node.State, node.Storage.Blocks, builder.SyncCore, - builder.FinalizedHeader, ) - if err != nil { return nil, fmt.Errorf("could not create public sync request handler: %w", err) } + builder.FollowerDistributor.AddFinalizationConsumer(syncRequestHandler) return syncRequestHandler, nil }) @@ -1058,6 +1212,20 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.BuildExecutionDataRequester() } + builder.Component("secure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + return builder.secureGrpcServer, nil + }) + + builder.Component("state stream unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + return builder.stateStreamGrpcServer, nil + }) + + if builder.rpcConf.UnsecureGRPCListenAddr != builder.stateStreamConf.ListenAddr { + builder.Component("unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + return builder.unsecureGrpcServer, nil + }) + } + builder.Component("ping engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { ping, err := pingeng.New( node.Logger, @@ -1082,41 +1250,36 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { // enqueuePublicNetworkInit enqueues the public network component initialized for the staked node func (builder *FlowAccessNodeBuilder) enqueuePublicNetworkInit() { - var libp2pNode p2p.LibP2PNode + var publicLibp2pNode p2p.LibP2PNode builder. Module("public network metrics", func(node *cmd.NodeConfig) error { builder.PublicNetworkConfig.Metrics = metrics.NewNetworkCollector(builder.Logger, metrics.WithNetworkPrefix("public")) return nil }). Component("public libp2p node", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - - libP2PFactory := builder.initPublicLibP2PFactory(builder.NodeConfig.NetworkKey, builder.PublicNetworkConfig.BindAddress, builder.PublicNetworkConfig.Metrics) - var err error - libp2pNode, err = libP2PFactory() + publicLibp2pNode, err = builder.initPublicLibp2pNode( + builder.NodeConfig.NetworkKey, + builder.PublicNetworkConfig.BindAddress, + builder.PublicNetworkConfig.Metrics) if err != nil { return nil, fmt.Errorf("could not create public libp2p node: %w", err) } - return libp2pNode, nil + return publicLibp2pNode, nil }). Component("public network", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { msgValidators := publicNetworkMsgValidators(node.Logger.With().Bool("public", true).Logger(), node.IdentityProvider, builder.NodeID) - middleware := builder.initMiddleware(builder.NodeID, builder.PublicNetworkConfig.Metrics, libp2pNode, msgValidators...) + middleware := builder.initMiddleware(builder.NodeID, builder.PublicNetworkConfig.Metrics, publicLibp2pNode, msgValidators...) // topology returns empty list since peers are not known upfront top := topology.EmptyTopology{} - - var heroCacheCollector module.HeroCacheMetrics = metrics.NewNoopCollector() - if builder.HeroCacheMetricsEnable { - heroCacheCollector = metrics.PublicNetworkReceiveCacheMetricsFactory(builder.MetricsRegisterer) - } - receiveCache := netcache.NewHeroReceiveCache(builder.NetworkReceivedMessageCacheSize, + receiveCache := netcache.NewHeroReceiveCache(builder.FlowConfig.NetworkConfig.NetworkReceivedMessageCacheSize, builder.Logger, - heroCacheCollector) + metrics.NetworkReceiveCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork)) - err := node.Metrics.Mempool.Register(metrics.ResourcePublicNetworkingReceiveCache, receiveCache.Size) + err := node.Metrics.Mempool.Register(metrics.PrependPublicPrefix(metrics.ResourceNetworkingReceiveCache), receiveCache.Size) if err != nil { return nil, fmt.Errorf("could not register networking receive cache metric: %w", err) } @@ -1132,79 +1295,97 @@ func (builder *FlowAccessNodeBuilder) enqueuePublicNetworkInit() { return net, nil }). Component("public peer manager", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - return libp2pNode.PeerManagerComponent(), nil + return publicLibp2pNode.PeerManagerComponent(), nil }) } -// initPublicLibP2PFactory creates the LibP2P factory function for the given node ID and network key. -// The factory function is later passed into the initMiddleware function to eventually instantiate the p2p.LibP2PNode instance +// initPublicLibp2pNode initializes the public libp2p node for the public (unstaked) network. // The LibP2P host is created with the following options: // - DHT as server // - The address from the node config or the specified bind address as the listen address // - The passed in private key as the libp2p key // - No connection gater // - Default Flow libp2p pubsub options -func (builder *FlowAccessNodeBuilder) initPublicLibP2PFactory(networkKey crypto.PrivateKey, bindAddress string, networkMetrics module.LibP2PMetrics) p2p.LibP2PFactoryFunc { - return func() (p2p.LibP2PNode, error) { - connManager, err := connection.NewConnManager(builder.Logger, networkMetrics, builder.ConnectionManagerConfig) - if err != nil { - return nil, fmt.Errorf("could not create connection manager: %w", err) - } - - meshTracer := tracer.NewGossipSubMeshTracer( - builder.Logger, - networkMetrics, - builder.IdentityProvider, - builder.GossipSubConfig.LocalMeshLogInterval) - - // setup RPC inspectors - rpcInspectorBuilder := inspector.NewGossipSubInspectorBuilder(builder.Logger, builder.SporkID, builder.GossipSubRPCInspectorsConfig, builder.GossipSubInspectorNotifDistributor) - rpcInspectors, err := rpcInspectorBuilder. - SetPublicNetwork(p2p.PublicNetworkEnabled). - SetMetrics(builder.Metrics.Network, builder.MetricsRegisterer). - SetMetricsEnabled(builder.MetricsEnabled).Build() - if err != nil { - return nil, fmt.Errorf("failed to create gossipsub rpc inspectors: %w", err) - } +// +// Args: +// - networkKey: The private key to use for the libp2p node +// +// - bindAddress: The address to bind the libp2p node to. +// - networkMetrics: The metrics collector for the network +// Returns: +// - The libp2p node instance for the public network. +// - Any error encountered during initialization. Any error should be considered fatal. +func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.PrivateKey, bindAddress string, networkMetrics module.LibP2PMetrics) (p2p.LibP2PNode, error) { + connManager, err := connection.NewConnManager(builder.Logger, networkMetrics, &builder.FlowConfig.NetworkConfig.ConnectionManagerConfig) + if err != nil { + return nil, fmt.Errorf("could not create connection manager: %w", err) + } - libp2pNode, err := p2pbuilder.NewNodeBuilder( - builder.Logger, - networkMetrics, - bindAddress, - networkKey, - builder.SporkID, - builder.LibP2PResourceManagerConfig). - SetBasicResolver(builder.Resolver). - SetSubscriptionFilter( - subscription.NewRoleBasedFilter( - flow.RoleAccess, builder.IdentityProvider, - ), - ). - SetConnectionManager(connManager). - SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { - return dht.NewDHT( - ctx, - h, - protocols.FlowPublicDHTProtocolID(builder.SporkID), - builder.Logger, - networkMetrics, - dht.AsServer(), - ) - }). - // disable connection pruning for the access node which supports the observer - SetPeerManagerOptions(connection.ConnectionPruningDisabled, builder.PeerUpdateInterval). - SetStreamCreationRetryInterval(builder.UnicastCreateStreamRetryDelay). - SetGossipSubTracer(meshTracer). - SetGossipSubScoreTracerInterval(builder.GossipSubConfig.ScoreTracerInterval). - SetGossipSubRPCInspectors(rpcInspectors...). - Build() + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: builder.Logger, + Metrics: networkMetrics, + IDProvider: builder.IdentityProvider, + LoggerInterval: builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, + RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + RpcSentTrackerWorkerQueueCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, + RpcSentTrackerNumOfWorkers: builder.FlowConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, + HeroCacheMetricsFactory: builder.HeroCacheMetricsFactory(), + NetworkingType: network.PublicNetwork, + } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) - if err != nil { - return nil, fmt.Errorf("could not build libp2p node for staked access node: %w", err) - } + libp2pNode, err := p2pbuilder.NewNodeBuilder( + builder.Logger, + &p2pconfig.MetricsConfig{ + HeroCacheFactory: builder.HeroCacheMetricsFactory(), + Metrics: networkMetrics, + }, + network.PublicNetwork, + bindAddress, + networkKey, + builder.SporkID, + builder.IdentityProvider, + &builder.FlowConfig.NetworkConfig.ResourceManagerConfig, + &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, + &p2pconfig.PeerManagerConfig{ + // TODO: eventually, we need pruning enabled even on public network. However, it needs a modified version of + // the peer manager that also operate on the public identities. + ConnectionPruning: connection.PruningDisabled, + UpdateInterval: builder.FlowConfig.NetworkConfig.PeerUpdateInterval, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + }, + &p2p.DisallowListCacheConfig{ + MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, + Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), + }). + SetBasicResolver(builder.Resolver). + SetSubscriptionFilter( + subscription.NewRoleBasedFilter( + flow.RoleAccess, builder.IdentityProvider, + ), + ). + SetConnectionManager(connManager). + SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { + return dht.NewDHT( + ctx, + h, + protocols.FlowPublicDHTProtocolID(builder.SporkID), + builder.Logger, + networkMetrics, + dht.AsServer(), + ) + }). + // disable connection pruning for the access node which supports the observer + SetStreamCreationRetryInterval(builder.FlowConfig.NetworkConfig.UnicastCreateStreamRetryDelay). + SetGossipSubTracer(meshTracer). + SetGossipSubScoreTracerInterval(builder.FlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval). + Build() - return libp2pNode, nil + if err != nil { + return nil, fmt.Errorf("could not build libp2p node for staked access node: %w", err) } + + return libp2pNode, nil } // initMiddleware creates the network.Middleware implementation with the libp2p factory function, metrics, peer update @@ -1215,20 +1396,18 @@ func (builder *FlowAccessNodeBuilder) initMiddleware(nodeID flow.Identifier, validators ...network.MessageValidator, ) network.Middleware { logger := builder.Logger.With().Bool("staked", false).Logger() - slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(logger, networkMetrics) - mw := middleware.NewMiddleware( - logger, - libp2pNode, - nodeID, - builder.Metrics.Bitswap, - builder.SporkID, - middleware.DefaultUnicastTimeout, - builder.IDTranslator, - builder.CodecFactory(), - slashingViolationsConsumer, + mw := middleware.NewMiddleware(&middleware.Config{ + Logger: logger, + Libp2pNode: libp2pNode, + FlowId: nodeID, + BitSwapMetrics: builder.Metrics.Bitswap, + RootBlockID: builder.SporkID, + UnicastMessageTimeout: middleware.DefaultUnicastTimeout, + IdTranslator: builder.IDTranslator, + Codec: builder.CodecFactory(), + }, middleware.WithMessageValidators(validators...), // use default identifier provider ) - builder.NodeDisallowListDistributor.AddConsumer(mw) builder.Middleware = mw return builder.Middleware } diff --git a/cmd/bootstrap/cmd/clusters.go b/cmd/bootstrap/cmd/clusters.go index 8f6faa10505..441f573f429 100644 --- a/cmd/bootstrap/cmd/clusters.go +++ b/cmd/bootstrap/cmd/clusters.go @@ -1,6 +1,8 @@ package cmd import ( + "errors" + "github.com/onflow/flow-go/cmd/bootstrap/run" model "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/cluster" @@ -10,31 +12,62 @@ import ( "github.com/onflow/flow-go/model/flow/filter" ) -// Construct cluster assignment with internal and partner nodes uniformly -// distributed across clusters. This function will produce the same cluster -// assignments for the same partner and internal lists, and the same seed. -func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo, seed int64) (flow.AssignmentList, flow.ClusterList) { +// Construct random cluster assignment with internal and partner nodes. +// The number of clusters is read from the `flagCollectionClusters` flag. +// The number of nodes in each cluster is deterministic and only depends on the number of clusters +// and the number of nodes. The repartition of internal and partner nodes is also deterministic +// and only depends on the number of clusters and nodes. +// The identity of internal and partner nodes in each cluster is the non-deterministic and is randomized +// using the system entropy. +// The function guarantees a specific constraint when partitioning the nodes into clusters: +// Each cluster must contain strictly more than 2/3 of internal nodes. If the constraint can't be +// satisfied, an exception is returned. +// Note that if an exception is returned with a certain number of internal/partner nodes, there is no chance +// of succeeding the assignment by re-running the function without increasing the internal nodes ratio. +func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo) (flow.AssignmentList, flow.ClusterList, error) { partners := model.ToIdentityList(partnerNodes).Filter(filter.HasRole(flow.RoleCollection)) internals := model.ToIdentityList(internalNodes).Filter(filter.HasRole(flow.RoleCollection)) + nClusters := int(flagCollectionClusters) + nCollectors := len(partners) + len(internals) - // deterministically shuffle both collector lists based on the input seed - // by using a different seed each spork, we will have different clusters - // even with the same collectors - partners = partners.DeterministicShuffle(seed) - internals = internals.DeterministicShuffle(seed) + // ensure we have at least as many collection nodes as clusters + if nCollectors < int(flagCollectionClusters) { + log.Fatal().Msgf("network bootstrap is configured with %d collection nodes, but %d clusters - must have at least one collection node per cluster", + nCollectors, flagCollectionClusters) + } + + // shuffle both collector lists based on a non-deterministic algorithm + partners, err := partners.Shuffle() + if err != nil { + log.Fatal().Err(err).Msg("could not shuffle partners") + } + internals, err = internals.Shuffle() + if err != nil { + log.Fatal().Err(err).Msg("could not shuffle internals") + } - nClusters := flagCollectionClusters identifierLists := make([]flow.IdentifierList, nClusters) + // array to track the 2/3 internal-nodes constraint (internal_nodes > 2 * partner_nodes) + constraint := make([]int, nClusters) // first, round-robin internal nodes into each cluster for i, node := range internals { - identifierLists[i%len(identifierLists)] = append(identifierLists[i%len(identifierLists)], node.NodeID) + identifierLists[i%nClusters] = append(identifierLists[i%nClusters], node.NodeID) + constraint[i%nClusters] += 1 } // next, round-robin partner nodes into each cluster for i, node := range partners { identifierLists[i%len(identifierLists)] = append(identifierLists[i%len(identifierLists)], node.NodeID) + constraint[i%nClusters] -= 2 + } + + // check the 2/3 constraint: for every cluster `i`, constraint[i] must be strictly positive + for i := 0; i < nClusters; i++ { + if constraint[i] <= 0 { + return nil, nil, errors.New("there isn't enough internal nodes to have at least 2/3 internal nodes in each cluster") + } } assignments := assignment.FromIdentifierLists(identifierLists) @@ -45,7 +78,7 @@ func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo, se log.Fatal().Err(err).Msg("could not create cluster list") } - return assignments, clusters + return assignments, clusters, nil } func constructRootQCsForClusters( diff --git a/cmd/bootstrap/cmd/constants.go b/cmd/bootstrap/cmd/constants.go deleted file mode 100644 index 6f376d5032b..00000000000 --- a/cmd/bootstrap/cmd/constants.go +++ /dev/null @@ -1,5 +0,0 @@ -package cmd - -const ( - minNodesPerCluster = 3 -) diff --git a/cmd/bootstrap/cmd/constraints.go b/cmd/bootstrap/cmd/constraints.go index b7c17b07b4a..2b0487b56cc 100644 --- a/cmd/bootstrap/cmd/constraints.go +++ b/cmd/bootstrap/cmd/constraints.go @@ -8,10 +8,12 @@ import ( // ensureUniformNodeWeightsPerRole verifies that the following condition is satisfied for each role R: // * all node with role R must have the same weight +// The function assumes there is at least one node for each role. func ensureUniformNodeWeightsPerRole(allNodes flow.IdentityList) { // ensure all nodes of the same role have equal weight for _, role := range flow.Roles() { withRole := allNodes.Filter(filter.HasRole(role)) + // each role has at least one node so it's safe to access withRole[0] expectedWeight := withRole[0].Weight for _, node := range withRole { if node.Weight != expectedWeight { @@ -34,39 +36,4 @@ func checkConstraints(partnerNodes, internalNodes []model.NodeInfo) { all := append(partners, internals...) ensureUniformNodeWeightsPerRole(all) - - // check collection committee Byzantine threshold for each cluster - // for checking Byzantine constraints, the seed doesn't matter - _, clusters := constructClusterAssignment(partnerNodes, internalNodes, 0) - partnerCOLCount := uint(0) - internalCOLCount := uint(0) - for _, cluster := range clusters { - clusterPartnerCount := uint(0) - clusterInternalCount := uint(0) - for _, node := range cluster { - if _, exists := partners.ByNodeID(node.NodeID); exists { - clusterPartnerCount++ - } - if _, exists := internals.ByNodeID(node.NodeID); exists { - clusterInternalCount++ - } - } - if clusterInternalCount <= clusterPartnerCount*2 { - log.Fatal().Msgf( - "will not bootstrap configuration without Byzantine majority within cluster: "+ - "(partners=%d, internals=%d, min_internals=%d)", - clusterPartnerCount, clusterInternalCount, clusterPartnerCount*2+1) - } - partnerCOLCount += clusterPartnerCount - internalCOLCount += clusterInternalCount - } - - // ensure we have enough total collectors - totalCollectors := partnerCOLCount + internalCOLCount - if totalCollectors < flagCollectionClusters*minNodesPerCluster { - log.Fatal().Msgf( - "will not bootstrap configuration with insufficient # of collectors for cluster count: "+ - "(total_collectors=%d, clusters=%d, min_total_collectors=%d)", - totalCollectors, flagCollectionClusters, flagCollectionClusters*minNodesPerCluster) - } } diff --git a/cmd/bootstrap/cmd/dkg.go b/cmd/bootstrap/cmd/dkg.go index b190b1a7c2c..d7069534e64 100644 --- a/cmd/bootstrap/cmd/dkg.go +++ b/cmd/bootstrap/cmd/dkg.go @@ -11,7 +11,7 @@ import ( "github.com/onflow/flow-go/state/protocol/inmem" ) -func runDKG(nodes []model.NodeInfo) dkg.DKGData { +func runBeaconKG(nodes []model.NodeInfo) dkg.DKGData { n := len(nodes) log.Info().Msgf("read %v node infos for DKG", n) @@ -19,11 +19,7 @@ func runDKG(nodes []model.NodeInfo) dkg.DKGData { log.Debug().Msgf("will run DKG") var dkgData dkg.DKGData var err error - if flagFastKG { - dkgData, err = bootstrapDKG.RunFastKG(n, flagBootstrapRandomSeed) - } else { - dkgData, err = bootstrapDKG.RunDKG(n, GenerateRandomSeeds(n, crypto.SeedMinLenDKG)) - } + dkgData, err = bootstrapDKG.RandomBeaconKG(n, GenerateRandomSeed(crypto.SeedMinLenDKG)) if err != nil { log.Fatal().Err(err).Msg("error running DKG") } diff --git a/cmd/bootstrap/cmd/finalize.go b/cmd/bootstrap/cmd/finalize.go index 5d1eb74106a..6f5507fdcfc 100644 --- a/cmd/bootstrap/cmd/finalize.go +++ b/cmd/bootstrap/cmd/finalize.go @@ -1,7 +1,7 @@ package cmd import ( - "encoding/binary" + "crypto/rand" "encoding/hex" "encoding/json" "fmt" @@ -48,9 +48,6 @@ var ( flagNumViewsInStakingAuction uint64 flagNumViewsInDKGPhase uint64 flagEpochCommitSafetyThreshold uint64 - - // this flag is used to seed the DKG, clustering and cluster QC generation - flagBootstrapRandomSeed []byte ) // PartnerWeights is the format of the JSON file specifying partner node weights. @@ -101,7 +98,6 @@ func addFinalizeCmdFlags() { finalizeCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 100, "length of the epoch staking phase measured in views") finalizeCmd.Flags().Uint64Var(&flagNumViewsInDKGPhase, "epoch-dkg-phase-length", 1000, "length of each DKG phase measured in views") finalizeCmd.Flags().Uint64Var(&flagEpochCommitSafetyThreshold, "epoch-commit-safety-threshold", 500, "defines epoch commitment deadline") - finalizeCmd.Flags().BytesHexVar(&flagBootstrapRandomSeed, "random-seed", GenerateRandomSeed(flow.EpochSetupRandomSourceLength), "The seed used to for DKG, Clustering and Cluster QC generation") finalizeCmd.Flags().UintVar(&flagProtocolVersion, "protocol-version", flow.DefaultProtocolVersion, "major software version used for the duration of this spork") cmd.MarkFlagRequired(finalizeCmd, "root-block") @@ -143,14 +139,6 @@ func finalize(cmd *cobra.Command, args []string) { log.Fatal().Err(err).Msg("invalid or unsafe epoch commit threshold config") } - if len(flagBootstrapRandomSeed) != flow.EpochSetupRandomSourceLength { - log.Error().Int("expected", flow.EpochSetupRandomSourceLength).Int("actual", len(flagBootstrapRandomSeed)).Msg("random seed provided length is not valid") - return - } - - log.Info().Str("seed", hex.EncodeToString(flagBootstrapRandomSeed)).Msg("deterministic bootstrapping random seed") - log.Info().Msg("") - log.Info().Msg("collecting partner network and staking keys") partnerNodes := readPartnerNodeInfos() log.Info().Msg("") @@ -195,8 +183,10 @@ func finalize(cmd *cobra.Command, args []string) { log.Info().Msg("") log.Info().Msg("computing collection node clusters") - clusterAssignmentSeed := binary.BigEndian.Uint64(flagBootstrapRandomSeed) - assignments, clusters := constructClusterAssignment(partnerNodes, internalNodes, int64(clusterAssignmentSeed)) + assignments, clusters, err := constructClusterAssignment(partnerNodes, internalNodes) + if err != nil { + log.Fatal().Err(err).Msg("unable to generate cluster assignment") + } log.Info().Msg("") log.Info().Msg("constructing root blocks for collection node clusters") @@ -211,7 +201,6 @@ func finalize(cmd *cobra.Command, args []string) { if flagRootCommit == "0000000000000000000000000000000000000000000000000000000000000000" { generateEmptyExecutionState( block.Header.ChainID, - flagBootstrapRandomSeed, assignments, clusterQCs, dkgData, @@ -587,7 +576,6 @@ func loadRootProtocolSnapshot(path string) (*inmem.Snapshot, error) { // given configuration. Sets the flagRootCommit variable for future reads. func generateEmptyExecutionState( chainID flow.ChainID, - randomSource []byte, assignments flow.AssignmentList, clusterQCs []*flow.QuorumCertificate, dkgData dkg.DKGData, @@ -606,6 +594,10 @@ func generateEmptyExecutionState( log.Fatal().Err(err).Msg("invalid genesis token supply") } + randomSource := make([]byte, flow.EpochSetupRandomSourceLength) + if _, err = rand.Read(randomSource); err != nil { + log.Fatal().Err(err).Msg("failed to generate a random source") + } cdcRandomSource, err := cadence.NewString(hex.EncodeToString(randomSource)) if err != nil { log.Fatal().Err(err).Msg("invalid random source") diff --git a/cmd/bootstrap/cmd/finalize_test.go b/cmd/bootstrap/cmd/finalize_test.go index 033e29b6609..58929d21e81 100644 --- a/cmd/bootstrap/cmd/finalize_test.go +++ b/cmd/bootstrap/cmd/finalize_test.go @@ -2,7 +2,6 @@ package cmd import ( "encoding/hex" - "os" "path/filepath" "regexp" "strings" @@ -17,8 +16,7 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -const finalizeHappyPathLogs = "^deterministic bootstrapping random seed" + - "collecting partner network and staking keys" + +const finalizeHappyPathLogs = "collecting partner network and staking keys" + `read \d+ partner node configuration files` + `read \d+ weights for partner nodes` + "generating internal private networking and staking keys" + @@ -52,7 +50,6 @@ const finalizeHappyPathLogs = "^deterministic bootstrapping random seed" + var finalizeHappyPathRegex = regexp.MustCompile(finalizeHappyPathLogs) func TestFinalize_HappyPath(t *testing.T) { - deterministicSeed := GenerateRandomSeed(flow.EpochSetupRandomSourceLength) rootCommit := unittest.StateCommitmentFixture() rootParent := unittest.StateCommitmentFixture() chainName := "main" @@ -68,14 +65,10 @@ func TestFinalize_HappyPath(t *testing.T) { flagPartnerWeights = partnerWeights flagInternalNodePrivInfoDir = internalPrivDir - flagFastKG = true flagRootChain = chainName flagRootParent = hex.EncodeToString(rootParent[:]) flagRootHeight = rootHeight - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - // rootBlock will generate DKG and place it into bootDir/public-root-information rootBlock(nil, nil) @@ -102,232 +95,47 @@ func TestFinalize_HappyPath(t *testing.T) { }) } -func TestFinalize_Deterministic(t *testing.T) { - deterministicSeed := GenerateRandomSeed(flow.EpochSetupRandomSourceLength) - rootCommit := unittest.StateCommitmentFixture() - rootParent := unittest.StateCommitmentFixture() - chainName := "main" - rootHeight := uint64(1000) - epochCounter := uint64(0) - - utils.RunWithSporkBootstrapDir(t, func(bootDir, partnerDir, partnerWeights, internalPrivDir, configPath string) { - - flagOutdir = bootDir - - flagConfig = configPath - flagPartnerNodeInfoDir = partnerDir - flagPartnerWeights = partnerWeights - flagInternalNodePrivInfoDir = internalPrivDir - - flagFastKG = true - - flagRootCommit = hex.EncodeToString(rootCommit[:]) - flagRootParent = hex.EncodeToString(rootParent[:]) - flagRootChain = chainName - flagRootHeight = rootHeight - flagEpochCounter = epochCounter - flagNumViewsInEpoch = 100_000 - flagNumViewsInStakingAuction = 50_000 - flagNumViewsInDKGPhase = 2_000 - flagEpochCommitSafetyThreshold = 1_000 - - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - - // rootBlock will generate DKG and place it into model.PathRootDKGData - rootBlock(nil, nil) - - flagRootBlock = filepath.Join(bootDir, model.PathRootBlockData) - flagDKGDataPath = filepath.Join(bootDir, model.PathRootDKGData) - flagRootBlockVotesDir = filepath.Join(bootDir, model.DirnameRootBlockVotes) - - hook := zeroLoggerHook{logs: &strings.Builder{}} - log = log.Hook(hook) - - finalize(nil, nil) - require.Regexp(t, finalizeHappyPathRegex, hook.logs.String()) - hook.logs.Reset() - - // check if root protocol snapshot exists - snapshotPath := filepath.Join(bootDir, model.PathRootProtocolStateSnapshot) - assert.FileExists(t, snapshotPath) - - // read snapshot - _, err := utils.ReadRootProtocolSnapshot(bootDir) - require.NoError(t, err) - - // delete snapshot file - err = os.Remove(snapshotPath) - require.NoError(t, err) - - finalize(nil, nil) - require.Regexp(t, finalizeHappyPathRegex, hook.logs.String()) - hook.logs.Reset() - - // check if root protocol snapshot exists - assert.FileExists(t, snapshotPath) - - // read snapshot - _, err = utils.ReadRootProtocolSnapshot(bootDir) - require.NoError(t, err) - - // ATTENTION: we can't use next statement because QC generation is not deterministic - // assert.Equal(t, firstSnapshot, secondSnapshot) - // Meaning we don't have a guarantee that with same input arguments we will get same QC. - // This doesn't mean that QC is invalid, but it will result in different structures, - // different QC => different service events => different result => different seal - // We need to use a different mechanism for comparing. - // ToDo: Revisit if this test case is valid at all. - }) -} - -func TestFinalize_SameSeedDifferentStateCommits(t *testing.T) { - deterministicSeed := GenerateRandomSeed(flow.EpochSetupRandomSourceLength) - rootCommit := unittest.StateCommitmentFixture() - rootParent := unittest.StateCommitmentFixture() - chainName := "main" - rootHeight := uint64(1000) - epochCounter := uint64(0) - - utils.RunWithSporkBootstrapDir(t, func(bootDir, partnerDir, partnerWeights, internalPrivDir, configPath string) { - - flagOutdir = bootDir - - flagConfig = configPath - flagPartnerNodeInfoDir = partnerDir - flagPartnerWeights = partnerWeights - flagInternalNodePrivInfoDir = internalPrivDir - - flagFastKG = true - - flagRootCommit = hex.EncodeToString(rootCommit[:]) - flagRootParent = hex.EncodeToString(rootParent[:]) - flagRootChain = chainName - flagRootHeight = rootHeight - flagEpochCounter = epochCounter - flagNumViewsInEpoch = 100_000 - flagNumViewsInStakingAuction = 50_000 - flagNumViewsInDKGPhase = 2_000 - flagEpochCommitSafetyThreshold = 1_000 - - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - - // rootBlock will generate DKG and place it into bootDir/public-root-information - rootBlock(nil, nil) - - flagRootBlock = filepath.Join(bootDir, model.PathRootBlockData) - flagDKGDataPath = filepath.Join(bootDir, model.PathRootDKGData) - flagRootBlockVotesDir = filepath.Join(bootDir, model.DirnameRootBlockVotes) - - hook := zeroLoggerHook{logs: &strings.Builder{}} - log = log.Hook(hook) - - finalize(nil, nil) - require.Regexp(t, finalizeHappyPathRegex, hook.logs.String()) - hook.logs.Reset() - - // check if root protocol snapshot exists - snapshotPath := filepath.Join(bootDir, model.PathRootProtocolStateSnapshot) - assert.FileExists(t, snapshotPath) - - // read snapshot - snapshot1, err := utils.ReadRootProtocolSnapshot(bootDir) - require.NoError(t, err) - - // delete snapshot file - err = os.Remove(snapshotPath) - require.NoError(t, err) - - // change input state commitments - rootCommit2 := unittest.StateCommitmentFixture() - rootParent2 := unittest.StateCommitmentFixture() - flagRootCommit = hex.EncodeToString(rootCommit2[:]) - flagRootParent = hex.EncodeToString(rootParent2[:]) - - finalize(nil, nil) - require.Regexp(t, finalizeHappyPathRegex, hook.logs.String()) - hook.logs.Reset() - - // check if root protocol snapshot exists - assert.FileExists(t, snapshotPath) - - // read snapshot - snapshot2, err := utils.ReadRootProtocolSnapshot(bootDir) - require.NoError(t, err) - - // current epochs - currentEpoch1 := snapshot1.Epochs().Current() - currentEpoch2 := snapshot2.Epochs().Current() - - // check dkg - dkg1, err := currentEpoch1.DKG() - require.NoError(t, err) - dkg2, err := currentEpoch2.DKG() - require.NoError(t, err) - assert.Equal(t, dkg1, dkg2) - - // check clustering - clustering1, err := currentEpoch1.Clustering() - require.NoError(t, err) - clustering2, err := currentEpoch2.Clustering() - require.NoError(t, err) - assert.Equal(t, clustering1, clustering2) - - // verify random sources are same - randomSource1, err := currentEpoch1.RandomSource() - require.NoError(t, err) - randomSource2, err := currentEpoch2.RandomSource() - require.NoError(t, err) - assert.Equal(t, randomSource1, randomSource2) - assert.Equal(t, randomSource1, deterministicSeed) - assert.Equal(t, flow.EpochSetupRandomSourceLength, len(randomSource1)) - }) -} - -func TestFinalize_InvalidRandomSeedLength(t *testing.T) { - rootCommit := unittest.StateCommitmentFixture() - rootParent := unittest.StateCommitmentFixture() - chainName := "main" - rootHeight := uint64(12332) - epochCounter := uint64(2) - - // set random seed with smaller length - deterministicSeed, err := hex.DecodeString("a12354a343234aa44bbb43") +func TestClusterAssignment(t *testing.T) { + tmp := flagCollectionClusters + flagCollectionClusters = 5 + // Happy path (limit set-up, can't have one less internal node) + partnersLen := 7 + internalLen := 22 + partners := unittest.NodeInfosFixture(partnersLen, unittest.WithRole(flow.RoleCollection)) + internals := unittest.NodeInfosFixture(internalLen, unittest.WithRole(flow.RoleCollection)) + + // should not error + _, clusters, err := constructClusterAssignment(partners, internals) require.NoError(t, err) + require.True(t, checkClusterConstraint(clusters, partners, internals)) + + // unhappy Path + internals = internals[:21] // reduce one internal node + // should error + _, _, err = constructClusterAssignment(partners, internals) + require.Error(t, err) + // revert the flag value + flagCollectionClusters = tmp +} - // invalid length execution logs - expectedLogs := regexp.MustCompile("random seed provided length is not valid") - - utils.RunWithSporkBootstrapDir(t, func(bootDir, partnerDir, partnerWeights, internalPrivDir, configPath string) { - - flagOutdir = bootDir - - flagConfig = configPath - flagPartnerNodeInfoDir = partnerDir - flagPartnerWeights = partnerWeights - flagInternalNodePrivInfoDir = internalPrivDir - - flagFastKG = true - - flagRootCommit = hex.EncodeToString(rootCommit[:]) - flagRootParent = hex.EncodeToString(rootParent[:]) - flagRootChain = chainName - flagRootHeight = rootHeight - flagEpochCounter = epochCounter - flagNumViewsInEpoch = 100_000 - flagNumViewsInStakingAuction = 50_000 - flagNumViewsInDKGPhase = 2_000 - flagEpochCommitSafetyThreshold = 1_000 - - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - - hook := zeroLoggerHook{logs: &strings.Builder{}} - log = log.Hook(hook) - - finalize(nil, nil) - assert.Regexp(t, expectedLogs, hook.logs.String()) - hook.logs.Reset() - }) +// Check about the number of internal/partner nodes in each cluster. The identites +// in each cluster do not matter for this check. +func checkClusterConstraint(clusters flow.ClusterList, partnersInfo []model.NodeInfo, internalsInfo []model.NodeInfo) bool { + partners := model.ToIdentityList(partnersInfo) + internals := model.ToIdentityList(internalsInfo) + for _, cluster := range clusters { + var clusterPartnerCount, clusterInternalCount int + for _, node := range cluster { + if _, exists := partners.ByNodeID(node.NodeID); exists { + clusterPartnerCount++ + } + if _, exists := internals.ByNodeID(node.NodeID); exists { + clusterInternalCount++ + } + } + if clusterInternalCount <= clusterPartnerCount*2 { + return false + } + } + return true } diff --git a/cmd/bootstrap/cmd/machine_account_test.go b/cmd/bootstrap/cmd/machine_account_test.go index 5fab682e561..7a1627ca3ac 100644 --- a/cmd/bootstrap/cmd/machine_account_test.go +++ b/cmd/bootstrap/cmd/machine_account_test.go @@ -31,6 +31,7 @@ func TestMachineAccountHappyPath(t *testing.T) { flagRole = "consensus" flagAddress = "189.123.123.42:3869" addr, err := flow.Mainnet.Chain().AddressAtIndex(uint64(rand.Intn(1_000_000))) + t.Logf("address is %s", addr) require.NoError(t, err) flagMachineAccountAddress = addr.HexWithPrefix() diff --git a/cmd/bootstrap/cmd/rootblock.go b/cmd/bootstrap/cmd/rootblock.go index d9acfff8037..7060fdf1a4b 100644 --- a/cmd/bootstrap/cmd/rootblock.go +++ b/cmd/bootstrap/cmd/rootblock.go @@ -1,7 +1,6 @@ package cmd import ( - "encoding/hex" "time" "github.com/spf13/cobra" @@ -12,7 +11,6 @@ import ( ) var ( - flagFastKG bool flagRootChain string flagRootParent string flagRootHeight uint64 @@ -23,7 +21,7 @@ var ( var rootBlockCmd = &cobra.Command{ Use: "rootblock", Short: "Generate root block data", - Long: `Run DKG, generate root block and votes for root block needed for constructing QC. Serialize all info into file`, + Long: `Run Beacon KeyGen, generate root block and votes for root block needed for constructing QC. Serialize all info into file`, Run: rootBlock, } @@ -59,11 +57,6 @@ func addRootBlockCmdFlags() { cmd.MarkFlagRequired(rootBlockCmd, "root-chain") cmd.MarkFlagRequired(rootBlockCmd, "root-parent") cmd.MarkFlagRequired(rootBlockCmd, "root-height") - - rootBlockCmd.Flags().BytesHexVar(&flagBootstrapRandomSeed, "random-seed", GenerateRandomSeed(flow.EpochSetupRandomSourceLength), "The seed used to for DKG, Clustering and Cluster QC generation") - - // optional parameters to influence various aspects of identity generation - rootBlockCmd.Flags().BoolVar(&flagFastKG, "fast-kg", false, "use fast (centralized) random beacon key generation instead of DKG") } func rootBlock(cmd *cobra.Command, args []string) { @@ -78,14 +71,6 @@ func rootBlock(cmd *cobra.Command, args []string) { } } - if len(flagBootstrapRandomSeed) != flow.EpochSetupRandomSourceLength { - log.Error().Int("expected", flow.EpochSetupRandomSourceLength).Int("actual", len(flagBootstrapRandomSeed)).Msg("random seed provided length is not valid") - return - } - - log.Info().Str("seed", hex.EncodeToString(flagBootstrapRandomSeed)).Msg("deterministic bootstrapping random seed") - log.Info().Msg("") - log.Info().Msg("collecting partner network and staking keys") partnerNodes := readPartnerNodeInfos() log.Info().Msg("") @@ -104,7 +89,7 @@ func rootBlock(cmd *cobra.Command, args []string) { log.Info().Msg("") log.Info().Msg("running DKG for consensus nodes") - dkgData := runDKG(model.FilterByRole(stakingNodes, flow.RoleConsensus)) + dkgData := runBeaconKG(model.FilterByRole(stakingNodes, flow.RoleConsensus)) log.Info().Msg("") log.Info().Msg("constructing root block") diff --git a/cmd/bootstrap/cmd/rootblock_test.go b/cmd/bootstrap/cmd/rootblock_test.go index 0883037115f..a2ccb177e79 100644 --- a/cmd/bootstrap/cmd/rootblock_test.go +++ b/cmd/bootstrap/cmd/rootblock_test.go @@ -13,12 +13,10 @@ import ( "github.com/onflow/flow-go/cmd/bootstrap/utils" model "github.com/onflow/flow-go/model/bootstrap" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) -const rootBlockHappyPathLogs = "^deterministic bootstrapping random seed" + - "collecting partner network and staking keys" + +const rootBlockHappyPathLogs = "collecting partner network and staking keys" + `read \d+ partner node configuration files` + `read \d+ weights for partner nodes` + "generating internal private networking and staking keys" + @@ -42,7 +40,6 @@ const rootBlockHappyPathLogs = "^deterministic bootstrapping random seed" + var rootBlockHappyPathRegex = regexp.MustCompile(rootBlockHappyPathLogs) func TestRootBlock_HappyPath(t *testing.T) { - deterministicSeed := GenerateRandomSeed(flow.EpochSetupRandomSourceLength) rootParent := unittest.StateCommitmentFixture() chainName := "main" rootHeight := uint64(12332) @@ -56,15 +53,10 @@ func TestRootBlock_HappyPath(t *testing.T) { flagPartnerWeights = partnerWeights flagInternalNodePrivInfoDir = internalPrivDir - flagFastKG = true - flagRootParent = hex.EncodeToString(rootParent[:]) flagRootChain = chainName flagRootHeight = rootHeight - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - hook := zeroLoggerHook{logs: &strings.Builder{}} log = log.Hook(hook) @@ -79,7 +71,6 @@ func TestRootBlock_HappyPath(t *testing.T) { } func TestRootBlock_Deterministic(t *testing.T) { - deterministicSeed := GenerateRandomSeed(flow.EpochSetupRandomSourceLength) rootParent := unittest.StateCommitmentFixture() chainName := "main" rootHeight := uint64(1000) @@ -93,15 +84,10 @@ func TestRootBlock_Deterministic(t *testing.T) { flagPartnerWeights = partnerWeights flagInternalNodePrivInfoDir = internalPrivDir - flagFastKG = true - flagRootParent = hex.EncodeToString(rootParent[:]) flagRootChain = chainName flagRootHeight = rootHeight - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - hook := zeroLoggerHook{logs: &strings.Builder{}} log = log.Hook(hook) diff --git a/cmd/bootstrap/cmd/seal.go b/cmd/bootstrap/cmd/seal.go index 91533377a0e..1a34c394e13 100644 --- a/cmd/bootstrap/cmd/seal.go +++ b/cmd/bootstrap/cmd/seal.go @@ -41,7 +41,7 @@ func constructRootResultAndSeal( DKGPhase3FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*3 - 1, Participants: participants.Sort(order.Canonical), Assignments: assignments, - RandomSource: flagBootstrapRandomSeed, + RandomSource: GenerateRandomSeed(flow.EpochSetupRandomSourceLength), } qcsWithSignerIDs := make([]*flow.QuorumCertificateWithSignerIDs, 0, len(clusterQCs)) diff --git a/cmd/bootstrap/dkg/dkg.go b/cmd/bootstrap/dkg/dkg.go index b519c59829b..3b65f44964a 100644 --- a/cmd/bootstrap/dkg/dkg.go +++ b/cmd/bootstrap/dkg/dkg.go @@ -2,210 +2,19 @@ package dkg import ( "fmt" - "sync" - "time" - - "github.com/rs/zerolog/log" "github.com/onflow/flow-go/crypto" model "github.com/onflow/flow-go/model/dkg" "github.com/onflow/flow-go/module/signature" ) -// RunDKG simulates a distributed DKG protocol by running the protocol locally -// and generating the DKG output info -func RunDKG(n int, seeds [][]byte) (model.DKGData, error) { - - if n != len(seeds) { - return model.DKGData{}, fmt.Errorf("n needs to match the number of seeds (%v != %v)", n, len(seeds)) - } - - // separate the case whith one node - if n == 1 { - sk, pk, pkGroup, err := thresholdSignKeyGenOneNode(seeds[0]) - if err != nil { - return model.DKGData{}, fmt.Errorf("run dkg failed: %w", err) - } - - dkgData := model.DKGData{ - PrivKeyShares: sk, - PubGroupKey: pkGroup, - PubKeyShares: pk, - } - - return dkgData, nil - } - - processors := make([]localDKGProcessor, 0, n) - - // create the message channels for node communication - chans := make([]chan *message, n) - for i := 0; i < n; i++ { - chans[i] = make(chan *message, 5*n) - } - - // create processors for all nodes - for i := 0; i < n; i++ { - processors = append(processors, localDKGProcessor{ - current: i, - chans: chans, - }) - } - - // create DKG instances for all nodes - for i := 0; i < n; i++ { - var err error - processors[i].dkg, err = crypto.NewJointFeldman(n, - signature.RandomBeaconThreshold(n), i, &processors[i]) - if err != nil { - return model.DKGData{}, err - } - } - - var wg sync.WaitGroup - phase := 0 - - // start DKG in all nodes - // start listening on the channels - wg.Add(n) - for i := 0; i < n; i++ { - // start dkg could also run in parallel - // but they are run sequentially to avoid having non-deterministic - // output (the PRG used is common) - err := processors[i].dkg.Start(seeds[i]) - if err != nil { - return model.DKGData{}, err - } - go dkgRunChan(&processors[i], &wg, phase) - } - phase++ - - // sync the two timeouts and start the next phase - for ; phase <= 2; phase++ { - wg.Wait() - wg.Add(n) - for i := 0; i < n; i++ { - go dkgRunChan(&processors[i], &wg, phase) - } - } - - // synchronize the main thread to end all DKGs - wg.Wait() - - skShares := make([]crypto.PrivateKey, 0, n) - - for _, processor := range processors { - skShares = append(skShares, processor.privkey) - } - - dkgData := model.DKGData{ - PrivKeyShares: skShares, - PubGroupKey: processors[0].pubgroupkey, - PubKeyShares: processors[0].pubkeys, - } - - return dkgData, nil -} - -// localDKGProcessor implements DKGProcessor interface -type localDKGProcessor struct { - current int - dkg crypto.DKGState - chans []chan *message - privkey crypto.PrivateKey - pubgroupkey crypto.PublicKey - pubkeys []crypto.PublicKey -} - -const ( - broadcast int = iota - private -) - -type message struct { - orig int - channel int - data []byte -} - -// PrivateSend sends a message from one node to another -func (proc *localDKGProcessor) PrivateSend(dest int, data []byte) { - newMsg := &message{proc.current, private, data} - proc.chans[dest] <- newMsg -} - -// Broadcast a message from one node to all nodes -func (proc *localDKGProcessor) Broadcast(data []byte) { - newMsg := &message{proc.current, broadcast, data} - for i := 0; i < len(proc.chans); i++ { - if i != proc.current { - proc.chans[i] <- newMsg - } - } -} - -// Disqualify a node -func (proc *localDKGProcessor) Disqualify(node int, log string) { -} - -// FlagMisbehavior flags a node for misbehaviour -func (proc *localDKGProcessor) FlagMisbehavior(node int, log string) { -} - -// dkgRunChan simulates processing incoming messages by a node -// it assumes proc.dkg is already running -func dkgRunChan(proc *localDKGProcessor, sync *sync.WaitGroup, phase int) { - for { - select { - case newMsg := <-proc.chans[proc.current]: - var err error - if newMsg.channel == private { - err = proc.dkg.HandlePrivateMsg(newMsg.orig, newMsg.data) - } else { - err = proc.dkg.HandleBroadcastMsg(newMsg.orig, newMsg.data) - } - if err != nil { - log.Fatal().Err(err).Msg("failed to receive DKG mst") - } - // if timeout, stop and finalize - case <-time.After(1 * time.Second): - switch phase { - case 0: - err := proc.dkg.NextTimeout() - if err != nil { - log.Fatal().Err(err).Msg("failed to wait for next timeout") - } - case 1: - err := proc.dkg.NextTimeout() - if err != nil { - log.Fatal().Err(err).Msg("failed to wait for next timeout") - } - case 2: - privkey, pubgroupkey, pubkeys, err := proc.dkg.End() - if err != nil { - log.Fatal().Err(err).Msg("end dkg error should be nit") - } - if privkey == nil { - log.Fatal().Msg("privkey was nil") - } - - proc.privkey = privkey - proc.pubgroupkey = pubgroupkey - proc.pubkeys = pubkeys - } - sync.Done() - return - } - } -} - -// RunFastKG is an alternative to RunDKG that runs much faster by using a centralized threshold signature key generation. -func RunFastKG(n int, seed []byte) (model.DKGData, error) { +// RandomBeaconKG is centralized BLS threshold signature key generation. +func RandomBeaconKG(n int, seed []byte) (model.DKGData, error) { if n == 1 { sk, pk, pkGroup, err := thresholdSignKeyGenOneNode(seed) if err != nil { - return model.DKGData{}, fmt.Errorf("fast KeyGen failed: %w", err) + return model.DKGData{}, fmt.Errorf("Beacon KeyGen failed: %w", err) } dkgData := model.DKGData{ @@ -219,7 +28,7 @@ func RunFastKG(n int, seed []byte) (model.DKGData, error) { skShares, pkShares, pkGroup, err := crypto.BLSThresholdKeyGen(int(n), signature.RandomBeaconThreshold(int(n)), seed) if err != nil { - return model.DKGData{}, fmt.Errorf("fast KeyGen failed: %w", err) + return model.DKGData{}, fmt.Errorf("Beacon KeyGen failed: %w", err) } dkgData := model.DKGData{ @@ -231,7 +40,7 @@ func RunFastKG(n int, seed []byte) (model.DKGData, error) { return dkgData, nil } -// simulates DKG with one single node +// Beacon KG with one node func thresholdSignKeyGenOneNode(seed []byte) ([]crypto.PrivateKey, []crypto.PublicKey, crypto.PublicKey, error) { sk, err := crypto.GeneratePrivateKey(crypto.BLSBLS12381, seed) if err != nil { diff --git a/cmd/bootstrap/dkg/dkg_test.go b/cmd/bootstrap/dkg/dkg_test.go index 9835cdca538..a5d5a56de18 100644 --- a/cmd/bootstrap/dkg/dkg_test.go +++ b/cmd/bootstrap/dkg/dkg_test.go @@ -9,17 +9,20 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -func TestRunDKG(t *testing.T) { - seedLen := crypto.SeedMinLenDKG - _, err := RunDKG(0, unittest.SeedFixtures(2, seedLen)) - require.EqualError(t, err, "n needs to match the number of seeds (0 != 2)") +func TestBeaconKG(t *testing.T) { + seed := unittest.SeedFixture(2 * crypto.SeedMinLenDKG) - _, err = RunDKG(3, unittest.SeedFixtures(2, seedLen)) - require.EqualError(t, err, "n needs to match the number of seeds (3 != 2)") + // n = 0 + _, err := RandomBeaconKG(0, seed) + require.EqualError(t, err, "Beacon KeyGen failed: size should be between 2 and 254, got 0") - data, err := RunDKG(4, unittest.SeedFixtures(4, seedLen)) + // should work for case n = 1 + _, err = RandomBeaconKG(1, seed) require.NoError(t, err) + // n = 4 + data, err := RandomBeaconKG(4, seed) + require.NoError(t, err) require.Len(t, data.PrivKeyShares, 4) require.Len(t, data.PubKeyShares, 4) } diff --git a/cmd/bootstrap/transit/cmd/utils.go b/cmd/bootstrap/transit/cmd/utils.go index 380646671c0..1f8f2f7920b 100644 --- a/cmd/bootstrap/transit/cmd/utils.go +++ b/cmd/bootstrap/transit/cmd/utils.go @@ -36,7 +36,7 @@ var ( // commit and semver vars commit = build.Commit() - semver = build.Semver() + semver = build.Version() ) // readNodeID reads the NodeID file diff --git a/cmd/bootstrap/utils/file.go b/cmd/bootstrap/utils/file.go index b1c0585ba0e..fc5f35c7122 100644 --- a/cmd/bootstrap/utils/file.go +++ b/cmd/bootstrap/utils/file.go @@ -35,7 +35,7 @@ func ReadRootProtocolSnapshot(bootDir string) (*inmem.Snapshot, error) { func ReadRootBlock(rootBlockDataPath string) (*flow.Block, error) { bytes, err := io.ReadFile(rootBlockDataPath) if err != nil { - return nil, fmt.Errorf("could not read root block: %w", err) + return nil, fmt.Errorf("could not read root block file: %w", err) } var encodable flow.Block diff --git a/cmd/build/version.go b/cmd/build/version.go index e2d59b3f74d..d089c27f3fb 100644 --- a/cmd/build/version.go +++ b/cmd/build/version.go @@ -6,6 +6,13 @@ // go build -ldflags "-X github.com/onflow/flow-go/cmd/build.semver=v1.0.0" package build +import ( + "fmt" + "strings" + + smv "github.com/coreos/go-semver/semver" +) + // Default value for build-time-injected version strings. const undefined = "undefined" @@ -15,8 +22,8 @@ var ( commit string ) -// Semver returns the semantic version of this build. -func Semver() string { +// Version returns the raw version string of this build. +func Version() string { return semver } @@ -41,3 +48,40 @@ func init() { commit = undefined } } + +var UndefinedVersionError = fmt.Errorf("version is undefined") + +// Semver returns the semantic version of this build as a semver.Version +// if it is defined, or UndefinedVersionError otherwise. +// The version string is converted to a semver compliant one if it isn't already +// but this might fail if the version string is still not semver compliant. In that +// case, an error is returned. +func Semver() (*smv.Version, error) { + if !IsDefined(semver) { + return nil, UndefinedVersionError + } + ver, err := smv.NewVersion(makeSemverCompliant(semver)) + return ver, err +} + +// makeSemverCompliant converts a non-semver version string to a semver compliant one. +// This removes the leading 'v'. +// In the past we sometimes omitted the patch version, e.g. v1.0.0 became v1.0 so this +// also adds a 0 patch version if there's no patch version. +func makeSemverCompliant(version string) string { + if !IsDefined(version) { + return version + } + + // Remove the leading 'v' + version = strings.TrimPrefix(version, "v") + + // If there's no patch version, add .0 + parts := strings.SplitN(version, "-", 2) + if strings.Count(parts[0], ".") == 1 { + parts[0] = parts[0] + ".0" + } + + version = strings.Join(parts, "-") + return version +} diff --git a/cmd/build/version_test.go b/cmd/build/version_test.go new file mode 100644 index 00000000000..4f3232b56d2 --- /dev/null +++ b/cmd/build/version_test.go @@ -0,0 +1,26 @@ +package build + +import "testing" + +func TestMakeSemverV2Compliant(t *testing.T) { + testCases := []struct { + name string + input string + expected string + }{ + {"No hyphen", "v0.29", "0.29.0"}, + {"With hyphen", "v0.29.11-an-error-handling", "0.29.11-an-error-handling"}, + {"With hyphen no patch", "v0.29-an-error-handling", "0.29.0-an-error-handling"}, + {"All digits", "v0.29.1", "0.29.1"}, + {undefined, undefined, undefined}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + output := makeSemverCompliant(tc.input) + if output != tc.expected { + t.Errorf("Got %s; expected %s", output, tc.expected) + } + }) + } +} diff --git a/cmd/collection/main.go b/cmd/collection/main.go index da7e946a98c..27faa959ca2 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -7,16 +7,11 @@ import ( "github.com/spf13/pflag" client "github.com/onflow/flow-go-sdk/access/grpc" - "github.com/onflow/flow-go/cmd/util/cmd/common" - "github.com/onflow/flow-go/consensus/hotstuff/validator" - "github.com/onflow/flow-go/model/bootstrap" - modulecompliance "github.com/onflow/flow-go/module/compliance" - "github.com/onflow/flow-go/module/mempool/herocache" - "github.com/onflow/flow-go/module/mempool/queue" - "github.com/onflow/flow-go/utils/grpcutils" - sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/admin/commands" + storageCommands "github.com/onflow/flow-go/admin/commands/storage" "github.com/onflow/flow-go/cmd" + "github.com/onflow/flow-go/cmd/util/cmd/common" "github.com/onflow/flow-go/consensus" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees" @@ -24,10 +19,12 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" hotsignature "github.com/onflow/flow-go/consensus/hotstuff/signature" + "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/engine/collection/epochmgr" "github.com/onflow/flow-go/engine/collection/epochmgr/factories" + "github.com/onflow/flow-go/engine/collection/events" "github.com/onflow/flow-go/engine/collection/ingest" "github.com/onflow/flow-go/engine/collection/pusher" "github.com/onflow/flow-go/engine/collection/rpc" @@ -35,21 +32,27 @@ import ( "github.com/onflow/flow-go/engine/common/provider" consync "github.com/onflow/flow-go/engine/common/synchronization" "github.com/onflow/flow-go/fvm/systemcontracts" + "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/module" builder "github.com/onflow/flow-go/module/builder/collection" "github.com/onflow/flow-go/module/chainsync" + modulecompliance "github.com/onflow/flow-go/module/compliance" "github.com/onflow/flow-go/module/epochs" confinalizer "github.com/onflow/flow-go/module/finalizer/consensus" "github.com/onflow/flow-go/module/mempool" epochpool "github.com/onflow/flow-go/module/mempool/epochs" + "github.com/onflow/flow-go/module/mempool/herocache" + "github.com/onflow/flow-go/module/mempool/queue" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/state/protocol" badgerState "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/blocktimer" "github.com/onflow/flow-go/state/protocol/events/gadgets" + "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/grpcutils" ) func main() { @@ -68,7 +71,7 @@ func main() { hotstuffMinTimeout time.Duration hotstuffTimeoutAdjustmentFactor float64 hotstuffHappyPathMaxRoundFailures uint64 - blockRateDelay time.Duration + hotstuffProposalDuration time.Duration startupTimeString string startupTime time.Time @@ -78,9 +81,8 @@ func main() { rpcConf rpc.Config clusterComplianceConfig modulecompliance.Config - pools *epochpool.TransactionPools // epoch-scoped transaction pools - finalizationDistributor *pubsub.FinalizationDistributor - finalizedHeader *consync.FinalizedHeaderCache + pools *epochpool.TransactionPools // epoch-scoped transaction pools + followerDistributor *pubsub.FollowerDistributor push *pusher.Engine ing *ingest.Engine @@ -98,6 +100,7 @@ func main() { apiRatelimits map[string]int apiBurstlimits map[string]int ) + var deprecatedFlagBlockRateDelay time.Duration nodeBuilder := cmd.FlowNode(flow.RoleCollection.String()) nodeBuilder.ExtraFlags(func(flags *pflag.FlagSet) { @@ -135,17 +138,19 @@ func main() { "maximum byte size of the proposed collection") flags.Uint64Var(&maxCollectionTotalGas, "builder-max-collection-total-gas", flow.DefaultMaxCollectionTotalGas, "maximum total amount of maxgas of transactions in proposed collections") - flags.DurationVar(&hotstuffMinTimeout, "hotstuff-min-timeout", 2500*time.Millisecond, + // Collection Nodes use a lower min timeout than Consensus Nodes (1.5s vs 2.5s) because: + // - they tend to have higher happy-path view rate, allowing a shorter timeout + // - since they have smaller committees, 1-2 offline replicas has a larger negative impact, which is mitigating with a smaller timeout + flags.DurationVar(&hotstuffMinTimeout, "hotstuff-min-timeout", 1500*time.Millisecond, "the lower timeout bound for the hotstuff pacemaker, this is also used as initial timeout") flags.Float64Var(&hotstuffTimeoutAdjustmentFactor, "hotstuff-timeout-adjustment-factor", timeout.DefaultConfig.TimeoutAdjustmentFactor, "adjustment of timeout duration in case of time out event") flags.Uint64Var(&hotstuffHappyPathMaxRoundFailures, "hotstuff-happy-path-max-round-failures", timeout.DefaultConfig.HappyPathMaxRoundFailures, "number of failed rounds before first timeout increase") - flags.DurationVar(&blockRateDelay, "block-rate-delay", 250*time.Millisecond, - "the delay to broadcast block proposal in order to control block production rate") flags.Uint64Var(&clusterComplianceConfig.SkipNewProposalsThreshold, "cluster-compliance-skip-proposals-threshold", modulecompliance.DefaultConfig().SkipNewProposalsThreshold, "threshold at which new proposals are discarded rather than cached, if their height is this much above local finalized height (cluster compliance engine)") flags.StringVar(&startupTimeString, "hotstuff-startup-time", cmd.NotSet, "specifies date and time (in ISO 8601 format) after which the consensus participant may enter the first view (e.g (e.g 1996-04-24T15:04:05-07:00))") + flags.DurationVar(&hotstuffProposalDuration, "hotstuff-proposal-duration", time.Millisecond*250, "the target time between entering a view and broadcasting the proposal for that view (different and smaller than view time)") flags.Uint32Var(&maxCollectionRequestCacheSize, "max-collection-provider-cache-size", provider.DefaultEntityRequestCacheSize, "maximum number of collection requests to cache for collection provider") flags.UintVar(&collectionProviderWorkers, "collection-provider-workers", provider.DefaultRequestProviderWorkers, "number of workers to use for collection provider") // epoch qc contract flags @@ -154,6 +159,8 @@ func main() { flags.StringToIntVar(&apiRatelimits, "api-rate-limits", map[string]int{}, "per second rate limits for GRPC API methods e.g. Ping=300,SendTransaction=500 etc. note limits apply globally to all clients.") flags.StringToIntVar(&apiBurstlimits, "api-burst-limits", map[string]int{}, "burst limits for gRPC API methods e.g. Ping=100,SendTransaction=100 etc. note limits apply globally to all clients.") + // deprecated flags + flags.DurationVar(&deprecatedFlagBlockRateDelay, "block-rate-delay", 0, "the delay to broadcast block proposal in order to control block production rate") }).ValidateFlags(func() error { if startupTimeString != cmd.NotSet { t, err := time.Parse(time.RFC3339, startupTimeString) @@ -162,6 +169,9 @@ func main() { } startupTime = t } + if deprecatedFlagBlockRateDelay > 0 { + nodeBuilder.Logger.Warn().Msg("A deprecated flag was specified (--block-rate-delay). This flag is deprecated as of v0.30 (Jun 2023), has no effect, and will eventually be removed.") + } return nil }) @@ -171,9 +181,17 @@ func main() { nodeBuilder. PreInit(cmd.DynamicStartPreInit). - Module("finalization distributor", func(node *cmd.NodeConfig) error { - finalizationDistributor = pubsub.NewFinalizationDistributor() - finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + AdminCommand("read-range-cluster-blocks", func(conf *cmd.NodeConfig) commands.AdminCommand { + clusterPayloads := badger.NewClusterPayloads(&metrics.NoopCollector{}, conf.DB) + headers, ok := conf.Storage.Headers.(*badger.Headers) + if !ok { + panic("fail to initialize admin tool, conf.Storage.Headers can not be casted as badger headers") + } + return storageCommands.NewReadRangeClusterBlocksCommand(conf.DB, headers, clusterPayloads) + }). + Module("follower distributor", func(node *cmd.NodeConfig) error { + followerDistributor = pubsub.NewFollowerDistributor() + followerDistributor.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("mutable follower state", func(node *cmd.NodeConfig) error { @@ -258,14 +276,6 @@ func main() { return validator, err }). - Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - finalizedHeader, err = consync.NewFinalizedHeaderCache(node.Logger, node.State, finalizationDistributor) - if err != nil { - return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) - } - - return finalizedHeader, nil - }). Component("consensus committee", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { // initialize consensus committee's membership state // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS Committee @@ -282,18 +292,14 @@ func main() { if err != nil { return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - packer := hotsignature.NewConsensusSigDataPacker(mainConsensusCommittee) - // initialize the verifier for the protocol consensus - verifier := verification.NewCombinedVerifier(mainConsensusCommittee, packer) // creates a consensus follower with noop consumer as the notifier followerCore, err = consensus.NewFollower( node.Logger, - mainConsensusCommittee, + node.Metrics.Mempool, node.Storage.Headers, finalizer, - verifier, - finalizationDistributor, - node.RootBlock.Header, + followerDistributor, + node.FinalizedRootBlock.Header, node.RootQC, finalized, pending, @@ -319,7 +325,7 @@ func main() { node.Logger, node.Metrics.Mempool, heroCacheCollector, - finalizationDistributor, + followerDistributor, followerState, followerCore, validator, @@ -336,13 +342,14 @@ func main() { node.Me, node.Metrics.Engine, node.Storage.Headers, - finalizedHeader.Get(), + node.LastFinalizedHeader, core, - followereng.WithComplianceConfigOpt(modulecompliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold)), + node.ComplianceConfig, ) if err != nil { return nil, fmt.Errorf("could not create follower engine: %w", err) } + followerDistributor.AddOnBlockFinalizedConsumer(followerEng.OnFinalizedBlock) return followerEng, nil }). @@ -354,15 +361,16 @@ func main() { node.Metrics.Engine, node.Network, node.Me, + node.State, node.Storage.Blocks, followerEng, mainChainSyncCore, - finalizedHeader, node.SyncEngineIdentifierProvider, ) if err != nil { return nil, fmt.Errorf("could not create synchronization engine: %w", err) } + followerDistributor.AddFinalizationConsumer(sync) return sync, nil }). @@ -450,6 +458,7 @@ func main() { builderFactory, err := factories.NewBuilderFactory( node.DB, + node.State, node.Storage.Headers, node.Tracer, colMetrics, @@ -476,7 +485,7 @@ func main() { node.Metrics.Mempool, node.State, node.Storage.Transactions, - modulecompliance.WithSkipNewProposalsThreshold(clusterComplianceConfig.SkipNewProposalsThreshold), + clusterComplianceConfig, ) if err != nil { return nil, err @@ -502,7 +511,7 @@ func main() { } opts := []consensus.Option{ - consensus.WithBlockRateDelay(blockRateDelay), + consensus.WithStaticProposalDuration(hotstuffProposalDuration), consensus.WithMinTimeout(hotstuffMinTimeout), consensus.WithTimeoutAdjustmentFactor(hotstuffTimeoutAdjustmentFactor), consensus.WithHappyPathMaxRoundFailures(hotstuffHappyPathMaxRoundFailures), @@ -565,6 +574,8 @@ func main() { heightEvents := gadgets.NewHeights() node.ProtocolEvents.AddConsumer(heightEvents) + clusterEvents := events.NewDistributor() + manager, err := epochmgr.New( node.Logger, node.Me, @@ -573,6 +584,7 @@ func main() { rootQCVoter, factory, heightEvents, + clusterEvents, ) if err != nil { return nil, fmt.Errorf("could not create epoch manager: %w", err) @@ -580,7 +592,7 @@ func main() { // register the manager for protocol events node.ProtocolEvents.AddConsumer(manager) - + clusterEvents.AddConsumer(node.LibP2PNode) return manager, err }) diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 077215a5235..8fb294fedf4 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -20,6 +20,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/blockproducer" "github.com/onflow/flow-go/consensus/hotstuff/committees" + "github.com/onflow/flow-go/consensus/hotstuff/cruisectl" "github.com/onflow/flow-go/consensus/hotstuff/notifications" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" @@ -48,7 +49,6 @@ import ( builder "github.com/onflow/flow-go/module/builder/consensus" "github.com/onflow/flow-go/module/chainsync" chmodule "github.com/onflow/flow-go/module/chunks" - modulecompliance "github.com/onflow/flow-go/module/compliance" dkgmodule "github.com/onflow/flow-go/module/dkg" "github.com/onflow/flow-go/module/epochs" finalizer "github.com/onflow/flow-go/module/finalizer/consensus" @@ -73,26 +73,32 @@ import ( func main() { var ( - guaranteeLimit uint - resultLimit uint - approvalLimit uint - sealLimit uint - pendingReceiptsLimit uint - minInterval time.Duration - maxInterval time.Duration - maxSealPerBlock uint - maxGuaranteePerBlock uint - hotstuffMinTimeout time.Duration - hotstuffTimeoutAdjustmentFactor float64 - hotstuffHappyPathMaxRoundFailures uint64 - blockRateDelay time.Duration - chunkAlpha uint - requiredApprovalsForSealVerification uint - requiredApprovalsForSealConstruction uint - emergencySealing bool - dkgControllerConfig dkgmodule.ControllerConfig - startupTimeString string - startupTime time.Time + guaranteeLimit uint + resultLimit uint + approvalLimit uint + sealLimit uint + pendingReceiptsLimit uint + minInterval time.Duration + maxInterval time.Duration + maxSealPerBlock uint + maxGuaranteePerBlock uint + hotstuffMinTimeout time.Duration + hotstuffTimeoutAdjustmentFactor float64 + hotstuffHappyPathMaxRoundFailures uint64 + chunkAlpha uint + requiredApprovalsForSealVerification uint + requiredApprovalsForSealConstruction uint + emergencySealing bool + dkgControllerConfig dkgmodule.ControllerConfig + dkgMessagingEngineConfig = dkgeng.DefaultMessagingEngineConfig() + cruiseCtlConfig = cruisectl.DefaultConfig() + cruiseCtlTargetTransitionTimeFlag = cruiseCtlConfig.TargetTransition.String() + cruiseCtlFallbackProposalDurationFlag time.Duration + cruiseCtlMinViewDurationFlag time.Duration + cruiseCtlMaxViewDurationFlag time.Duration + cruiseCtlEnabledFlag bool + startupTimeString string + startupTime time.Time // DKG contract client machineAccountInfo *bootstrap.NodeMachineAccountInfo @@ -100,32 +106,33 @@ func main() { insecureAccessAPI bool accessNodeIDS []string - err error - mutableState protocol.ParticipantState - beaconPrivateKey *encodable.RandomBeaconPrivKey - guarantees mempool.Guarantees - receipts mempool.ExecutionTree - seals mempool.IncorporatedResultSeals - pendingReceipts mempool.PendingReceipts - receiptRequester *requester.Engine - syncCore *chainsync.Core - comp *compliance.Engine - hot module.HotStuff - conMetrics module.ConsensusMetrics - mainMetrics module.HotstuffMetrics - receiptValidator module.ReceiptValidator - chunkAssigner *chmodule.ChunkAssigner - finalizationDistributor *pubsub.FinalizationDistributor - dkgBrokerTunnel *dkgmodule.BrokerTunnel - blockTimer protocol.BlockTimer - finalizedHeader *synceng.FinalizedHeaderCache - committee *committees.Consensus - epochLookup *epochs.EpochLookup - hotstuffModules *consensus.HotstuffModules - dkgState *bstorage.DKGState - safeBeaconKeys *bstorage.SafeBeaconPrivateKeys - getSealingConfigs module.SealingConfigsGetter + err error + mutableState protocol.ParticipantState + beaconPrivateKey *encodable.RandomBeaconPrivKey + guarantees mempool.Guarantees + receipts mempool.ExecutionTree + seals mempool.IncorporatedResultSeals + pendingReceipts mempool.PendingReceipts + receiptRequester *requester.Engine + syncCore *chainsync.Core + comp *compliance.Engine + hot module.HotStuff + conMetrics module.ConsensusMetrics + mainMetrics module.HotstuffMetrics + receiptValidator module.ReceiptValidator + chunkAssigner *chmodule.ChunkAssigner + followerDistributor *pubsub.FollowerDistributor + dkgBrokerTunnel *dkgmodule.BrokerTunnel + blockTimer protocol.BlockTimer + proposalDurProvider hotstuff.ProposalDurationProvider + committee *committees.Consensus + epochLookup *epochs.EpochLookup + hotstuffModules *consensus.HotstuffModules + dkgState *bstorage.DKGState + safeBeaconKeys *bstorage.SafeBeaconPrivateKeys + getSealingConfigs module.SealingConfigsGetter ) + var deprecatedFlagBlockRateDelay time.Duration nodeBuilder := cmd.FlowNode(flow.RoleConsensus.String()) nodeBuilder.ExtraFlags(func(flags *pflag.FlagSet) { @@ -143,7 +150,11 @@ func main() { flags.DurationVar(&hotstuffMinTimeout, "hotstuff-min-timeout", 2500*time.Millisecond, "the lower timeout bound for the hotstuff pacemaker, this is also used as initial timeout") flags.Float64Var(&hotstuffTimeoutAdjustmentFactor, "hotstuff-timeout-adjustment-factor", timeout.DefaultConfig.TimeoutAdjustmentFactor, "adjustment of timeout duration in case of time out event") flags.Uint64Var(&hotstuffHappyPathMaxRoundFailures, "hotstuff-happy-path-max-round-failures", timeout.DefaultConfig.HappyPathMaxRoundFailures, "number of failed rounds before first timeout increase") - flags.DurationVar(&blockRateDelay, "block-rate-delay", 500*time.Millisecond, "the delay to broadcast block proposal in order to control block production rate") + flags.StringVar(&cruiseCtlTargetTransitionTimeFlag, "cruise-ctl-target-epoch-transition-time", cruiseCtlTargetTransitionTimeFlag, "the target epoch switchover schedule") + flags.DurationVar(&cruiseCtlFallbackProposalDurationFlag, "cruise-ctl-fallback-proposal-duration", cruiseCtlConfig.FallbackProposalDelay.Load(), "the proposal duration value to use when the controller is disabled, or in epoch fallback mode. In those modes, this value has the same as the old `--block-rate-delay`") + flags.DurationVar(&cruiseCtlMinViewDurationFlag, "cruise-ctl-min-view-duration", cruiseCtlConfig.MinViewDuration.Load(), "the lower bound of authority for the controller, when active. This is the smallest amount of time a view is allowed to take.") + flags.DurationVar(&cruiseCtlMaxViewDurationFlag, "cruise-ctl-max-view-duration", cruiseCtlConfig.MaxViewDuration.Load(), "the upper bound of authority for the controller when active. This is the largest amount of time a view is allowed to take.") + flags.BoolVar(&cruiseCtlEnabledFlag, "cruise-ctl-enabled", cruiseCtlConfig.Enabled.Load(), "whether the block time controller is enabled; when disabled, the FallbackProposalDelay is used") flags.UintVar(&chunkAlpha, "chunk-alpha", flow.DefaultChunkAssignmentAlpha, "number of verifiers that should be assigned to each chunk") flags.UintVar(&requiredApprovalsForSealVerification, "required-verification-seal-approvals", flow.DefaultRequiredApprovalsForSealValidation, "minimum number of approvals that are required to verify a seal") flags.UintVar(&requiredApprovalsForSealConstruction, "required-construction-seal-approvals", flow.DefaultRequiredApprovalsForSealConstruction, "minimum number of approvals that are required to construct a seal") @@ -153,7 +164,11 @@ func main() { flags.DurationVar(&dkgControllerConfig.BaseStartDelay, "dkg-controller-base-start-delay", dkgmodule.DefaultBaseStartDelay, "used to define the range for jitter prior to DKG start (eg. 500µs) - the base value is scaled quadratically with the # of DKG participants") flags.DurationVar(&dkgControllerConfig.BaseHandleFirstBroadcastDelay, "dkg-controller-base-handle-first-broadcast-delay", dkgmodule.DefaultBaseHandleFirstBroadcastDelay, "used to define the range for jitter prior to DKG handling the first broadcast messages (eg. 50ms) - the base value is scaled quadratically with the # of DKG participants") flags.DurationVar(&dkgControllerConfig.HandleSubsequentBroadcastDelay, "dkg-controller-handle-subsequent-broadcast-delay", dkgmodule.DefaultHandleSubsequentBroadcastDelay, "used to define the constant delay introduced prior to DKG handling subsequent broadcast messages (eg. 2s)") + flags.DurationVar(&dkgMessagingEngineConfig.RetryBaseWait, "dkg-messaging-engine-retry-base-wait", dkgMessagingEngineConfig.RetryBaseWait, "the inter-attempt wait time for the first attempt (base of exponential retry)") + flags.Uint64Var(&dkgMessagingEngineConfig.RetryMax, "dkg-messaging-engine-retry-max", dkgMessagingEngineConfig.RetryMax, "the maximum number of retry attempts for an outbound DKG message") + flags.Uint64Var(&dkgMessagingEngineConfig.RetryJitterPercent, "dkg-messaging-engine-retry-jitter-percent", dkgMessagingEngineConfig.RetryJitterPercent, "the percentage of jitter to apply to each inter-attempt wait time") flags.StringVar(&startupTimeString, "hotstuff-startup-time", cmd.NotSet, "specifies date and time (in ISO 8601 format) after which the consensus participant may enter the first view (e.g 1996-04-24T15:04:05-07:00)") + flags.DurationVar(&deprecatedFlagBlockRateDelay, "block-rate-delay", 0, "[deprecated in v0.30; Jun 2023] Use `cruise-ctl-*` flags instead, this flag has no effect and will eventually be removed") }).ValidateFlags(func() error { nodeBuilder.Logger.Info().Str("startup_time_str", startupTimeString).Msg("got startup_time_str") if startupTimeString != cmd.NotSet { @@ -164,6 +179,31 @@ func main() { startupTime = t nodeBuilder.Logger.Info().Time("startup_time", startupTime).Msg("got startup_time") } + // parse target transition time string, if set + if cruiseCtlTargetTransitionTimeFlag != cruiseCtlConfig.TargetTransition.String() { + transitionTime, err := cruisectl.ParseTransition(cruiseCtlTargetTransitionTimeFlag) + if err != nil { + return fmt.Errorf("invalid epoch transition time string: %w", err) + } + cruiseCtlConfig.TargetTransition = *transitionTime + } + // convert local flag variables to atomic config variables, for dynamically updatable fields + if cruiseCtlEnabledFlag != cruiseCtlConfig.Enabled.Load() { + cruiseCtlConfig.Enabled.Store(cruiseCtlEnabledFlag) + } + if cruiseCtlFallbackProposalDurationFlag != cruiseCtlConfig.FallbackProposalDelay.Load() { + cruiseCtlConfig.FallbackProposalDelay.Store(cruiseCtlFallbackProposalDurationFlag) + } + if cruiseCtlMinViewDurationFlag != cruiseCtlConfig.MinViewDuration.Load() { + cruiseCtlConfig.MinViewDuration.Store(cruiseCtlMinViewDurationFlag) + } + if cruiseCtlMaxViewDurationFlag != cruiseCtlConfig.MaxViewDuration.Load() { + cruiseCtlConfig.MaxViewDuration.Store(cruiseCtlMaxViewDurationFlag) + } + // log a warning about deprecated flags + if deprecatedFlagBlockRateDelay > 0 { + nodeBuilder.Logger.Warn().Msg("A deprecated flag was specified (--block-rate-delay). This flag is deprecated as of v0.30 (Jun 2023), has no effect, and will eventually be removed.") + } return nil }) @@ -267,7 +307,7 @@ func main() { // their first beacon private key through the DKG in the EpochSetup phase // prior to their first epoch as network participant). - rootSnapshot := node.State.AtBlockID(node.RootBlock.ID()) + rootSnapshot := node.State.AtBlockID(node.FinalizedRootBlock.ID()) isSporkRoot, err := protocol.IsSporkRootSnapshot(rootSnapshot) if err != nil { return fmt.Errorf("could not check whether root snapshot is spork root: %w", err) @@ -287,7 +327,7 @@ func main() { return fmt.Errorf("could not load beacon key file: %w", err) } - rootEpoch := node.State.AtBlockID(node.RootBlock.ID()).Epochs().Current() + rootEpoch := node.State.AtBlockID(node.FinalizedRootBlock.ID()).Epochs().Current() epochCounter, err := rootEpoch.Counter() if err != nil { return fmt.Errorf("could not get root epoch counter: %w", err) @@ -364,9 +404,8 @@ func main() { syncCore, err = chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector(node.RootChainID), node.RootChainID) return err }). - Module("finalization distributor", func(node *cmd.NodeConfig) error { - finalizationDistributor = pubsub.NewFinalizationDistributor() - finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) + Module("follower distributor", func(node *cmd.NodeConfig) error { + followerDistributor = pubsub.NewFollowerDistributor() return nil }). Module("machine account config", func(node *cmd.NodeConfig) error { @@ -432,8 +471,8 @@ func main() { ) // subscribe for finalization events from hotstuff - finalizationDistributor.AddOnBlockFinalizedConsumer(e.OnFinalizedBlock) - finalizationDistributor.AddOnBlockIncorporatedConsumer(e.OnBlockIncorporated) + followerDistributor.AddOnBlockFinalizedConsumer(e.OnFinalizedBlock) + followerDistributor.AddOnBlockIncorporatedConsumer(e.OnBlockIncorporated) return e, err }). @@ -487,8 +526,8 @@ func main() { // subscribe engine to inputs from other node-internal components receiptRequester.WithHandle(e.HandleReceipt) - finalizationDistributor.AddOnBlockFinalizedConsumer(e.OnFinalizedBlock) - finalizationDistributor.AddOnBlockIncorporatedConsumer(e.OnBlockIncorporated) + followerDistributor.AddOnBlockFinalizedConsumer(e.OnFinalizedBlock) + followerDistributor.AddOnBlockIncorporatedConsumer(e.OnBlockIncorporated) return e, err }). @@ -554,13 +593,18 @@ func main() { // create consensus logger logger := createLogger(node.Logger, node.RootChainID) + telemetryConsumer := notifications.NewTelemetryConsumer(logger) + slashingViolationConsumer := notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger) + followerDistributor.AddProposalViolationConsumer(slashingViolationConsumer) + // initialize a logging notifier for hotstuff notifier := createNotifier( logger, mainMetrics, ) - notifier.AddConsumer(finalizationDistributor) + notifier.AddParticipantConsumer(telemetryConsumer) + notifier.AddFollowerConsumer(followerDistributor) // initialize the persister persist := persister.New(node.DB, node.RootChainID) @@ -575,16 +619,20 @@ func main() { node.Storage.Headers, finalize, notifier, - node.RootBlock.Header, + node.FinalizedRootBlock.Header, node.RootQC, ) if err != nil { return nil, err } - qcDistributor := pubsub.NewQCCreatedDistributor() + // create producer and connect it to consumers + voteAggregationDistributor := pubsub.NewVoteAggregationDistributor() + voteAggregationDistributor.AddVoteCollectorConsumer(telemetryConsumer) + voteAggregationDistributor.AddVoteAggregationViolationConsumer(slashingViolationConsumer) + validator := consensus.NewValidator(mainMetrics, wrappedCommittee) - voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(wrappedCommittee, qcDistributor.OnQcConstructedFromVotes) + voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(wrappedCommittee, voteAggregationDistributor.OnQcConstructedFromVotes) lowestViewForVoteProcessing := finalizedBlock.View + 1 voteAggregator, err := consensus.NewVoteAggregator( logger, @@ -592,17 +640,21 @@ func main() { node.Metrics.Engine, node.Metrics.Mempool, lowestViewForVoteProcessing, - notifier, + voteAggregationDistributor, voteProcessorFactory, - finalizationDistributor) + followerDistributor) if err != nil { return nil, fmt.Errorf("could not initialize vote aggregator: %w", err) } - timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() + // create producer and connect it to consumers + timeoutAggregationDistributor := pubsub.NewTimeoutAggregationDistributor() + timeoutAggregationDistributor.AddTimeoutCollectorConsumer(telemetryConsumer) + timeoutAggregationDistributor.AddTimeoutAggregationViolationConsumer(slashingViolationConsumer) + timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory( logger, - timeoutCollectorDistributor, + timeoutAggregationDistributor, committee, validator, msig.ConsensusTimeoutTag, @@ -614,7 +666,7 @@ func main() { node.Metrics.Mempool, notifier, timeoutProcessorFactory, - timeoutCollectorDistributor, + timeoutAggregationDistributor, lowestViewForVoteProcessing, ) if err != nil { @@ -626,9 +678,8 @@ func main() { Committee: wrappedCommittee, Signer: signer, Persist: persist, - QCCreatedDistributor: qcDistributor, - FinalizationDistributor: finalizationDistributor, - TimeoutCollectorDistributor: timeoutCollectorDistributor, + VoteCollectorDistributor: voteAggregationDistributor.VoteCollectorDistributor, + TimeoutCollectorDistributor: timeoutAggregationDistributor.TimeoutCollectorDistributor, Forks: forks, Validator: validator, VoteAggregator: voteAggregator, @@ -637,6 +688,39 @@ func main() { return util.MergeReadyDone(voteAggregator, timeoutAggregator), nil }). + Component("block rate cruise control", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + livenessData, err := hotstuffModules.Persist.GetLivenessData() + if err != nil { + return nil, err + } + ctl, err := cruisectl.NewBlockTimeController(node.Logger, metrics.NewCruiseCtlMetrics(), cruiseCtlConfig, node.State, livenessData.CurrentView) + if err != nil { + return nil, err + } + proposalDurProvider = ctl + hotstuffModules.Notifier.AddOnBlockIncorporatedConsumer(ctl.OnBlockIncorporated) + node.ProtocolEvents.AddConsumer(ctl) + + // set up admin commands for dynamically updating configs + err = node.ConfigManager.RegisterBoolConfig("cruise-ctl-enabled", cruiseCtlConfig.GetEnabled, cruiseCtlConfig.SetEnabled) + if err != nil { + return nil, err + } + err = node.ConfigManager.RegisterDurationConfig("cruise-ctl-fallback-proposal-duration", cruiseCtlConfig.GetFallbackProposalDuration, cruiseCtlConfig.SetFallbackProposalDuration) + if err != nil { + return nil, err + } + err = node.ConfigManager.RegisterDurationConfig("cruise-ctl-min-view-duration", cruiseCtlConfig.GetMinViewDuration, cruiseCtlConfig.SetMinViewDuration) + if err != nil { + return nil, err + } + err = node.ConfigManager.RegisterDurationConfig("cruise-ctl-max-view-duration", cruiseCtlConfig.GetMaxViewDuration, cruiseCtlConfig.SetMaxViewDuration) + if err != nil { + return nil, err + } + + return ctl, nil + }). Component("consensus participant", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { // initialize the block builder var build module.Builder @@ -667,8 +751,7 @@ func main() { consensus.WithMinTimeout(hotstuffMinTimeout), consensus.WithTimeoutAdjustmentFactor(hotstuffTimeoutAdjustmentFactor), consensus.WithHappyPathMaxRoundFailures(hotstuffHappyPathMaxRoundFailures), - consensus.WithBlockRateDelay(blockRateDelay), - consensus.WithConfigRegistrar(node.ConfigManager), + consensus.WithProposalDurationProvider(proposalDurProvider), } if !startupTime.IsZero() { @@ -683,6 +766,7 @@ func main() { hot, err = consensus.NewParticipant( createLogger(node.Logger, node.RootChainID), mainMetrics, + node.Metrics.Mempool, build, finalizedBlock, pending, @@ -705,6 +789,7 @@ func main() { node.Metrics.Mempool, mainMetrics, node.Metrics.Compliance, + followerDistributor, node.Tracer, node.Storage.Headers, node.Storage.Payloads, @@ -715,7 +800,7 @@ func main() { hot, hotstuffModules.VoteAggregator, hotstuffModules.TimeoutAggregator, - modulecompliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold), + node.ComplianceConfig, ) if err != nil { return nil, fmt.Errorf("could not initialize compliance core: %w", err) @@ -730,8 +815,7 @@ func main() { if err != nil { return nil, fmt.Errorf("could not initialize compliance engine: %w", err) } - - finalizationDistributor.AddOnBlockFinalizedConsumer(comp.OnFinalizedBlock) + followerDistributor.AddOnBlockFinalizedConsumer(comp.OnFinalizedBlock) return comp, nil }). @@ -754,29 +838,22 @@ func main() { hotstuffModules.Notifier.AddConsumer(messageHub) return messageHub, nil }). - Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - finalizedHeader, err = synceng.NewFinalizedHeaderCache(node.Logger, node.State, finalizationDistributor) - if err != nil { - return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) - } - - return finalizedHeader, nil - }). Component("sync engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { sync, err := synceng.New( node.Logger, node.Metrics.Engine, node.Network, node.Me, + node.State, node.Storage.Blocks, comp, syncCore, - finalizedHeader, node.SyncEngineIdentifierProvider, ) if err != nil { return nil, fmt.Errorf("could not initialize synchronization engine: %w", err) } + followerDistributor.AddFinalizationConsumer(sync) return sync, nil }). @@ -797,6 +874,8 @@ func main() { node.Network, node.Me, dkgBrokerTunnel, + node.Metrics.Mempool, + dkgMessagingEngineConfig, ) if err != nil { return nil, fmt.Errorf("could not initialize DKG messaging engine: %w", err) diff --git a/cmd/consensus/notifier.go b/cmd/consensus/notifier.go index 94fc57782e6..3826060cf63 100644 --- a/cmd/consensus/notifier.go +++ b/cmd/consensus/notifier.go @@ -17,11 +17,9 @@ func createLogger(log zerolog.Logger, chainID flow.ChainID) zerolog.Logger { // createNotifier creates a pubsub distributor and connects it to consensus consumers. func createNotifier(log zerolog.Logger, metrics module.HotstuffMetrics) *pubsub.Distributor { - telemetryConsumer := notifications.NewTelemetryConsumer(log) metricsConsumer := metricsconsumer.NewMetricsConsumer(metrics) logsConsumer := notifications.NewLogConsumer(log) dis := pubsub.NewDistributor() - dis.AddConsumer(telemetryConsumer) dis.AddConsumer(metricsConsumer) dis.AddConsumer(logsConsumer) return dis diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index b21736e9cd3..871ab90b8f6 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -28,6 +28,7 @@ import ( stateSyncCommands "github.com/onflow/flow-go/admin/commands/state_synchronization" storageCommands "github.com/onflow/flow-go/admin/commands/storage" uploaderCommands "github.com/onflow/flow-go/admin/commands/uploader" + "github.com/onflow/flow-go/cmd/build" "github.com/onflow/flow-go/consensus" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees" @@ -37,6 +38,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" + "github.com/onflow/flow-go/engine" followereng "github.com/onflow/flow-go/engine/common/follower" "github.com/onflow/flow-go/engine/common/provider" "github.com/onflow/flow-go/engine/common/requester" @@ -45,13 +47,15 @@ import ( "github.com/onflow/flow-go/engine/execution/computation" "github.com/onflow/flow-go/engine/execution/computation/committer" "github.com/onflow/flow-go/engine/execution/ingestion" + "github.com/onflow/flow-go/engine/execution/ingestion/stop" "github.com/onflow/flow-go/engine/execution/ingestion/uploader" exeprovider "github.com/onflow/flow-go/engine/execution/provider" "github.com/onflow/flow-go/engine/execution/rpc" + "github.com/onflow/flow-go/engine/execution/scripts" "github.com/onflow/flow-go/engine/execution/state" "github.com/onflow/flow-go/engine/execution/state/bootstrap" "github.com/onflow/flow-go/fvm" - fvmState "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/ledger/common/pathfinder" ledger "github.com/onflow/flow-go/ledger/complete" @@ -62,7 +66,6 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/blobs" "github.com/onflow/flow-go/module/chainsync" - "github.com/onflow/flow-go/module/compliance" "github.com/onflow/flow-go/module/executiondatasync/execution_data" exedataprovider "github.com/onflow/flow-go/module/executiondatasync/provider" "github.com/onflow/flow-go/module/executiondatasync/pruner" @@ -109,39 +112,41 @@ type ExecutionNode struct { builder *FlowNodeBuilder // This is needed for accessing the ShutdownFunc exeConf *ExecutionConfig - collector module.ExecutionMetrics - executionState state.ExecutionState - followerState protocol.FollowerState - committee hotstuff.DynamicCommittee - ledgerStorage *ledger.Ledger - events *storage.Events - serviceEvents *storage.ServiceEvents - txResults *storage.TransactionResults - results *storage.ExecutionResults - myReceipts *storage.MyExecutionReceipts - providerEngine *exeprovider.Engine - checkerEng *checker.Engine - syncCore *chainsync.Core - syncEngine *synchronization.Engine - followerCore *hotstuff.FollowerLoop // follower hotstuff logic - followerEng *followereng.ComplianceEngine // to sync blocks from consensus nodes - computationManager *computation.Manager - collectionRequester *requester.Engine - ingestionEng *ingestion.Engine - finalizationDistributor *pubsub.FinalizationDistributor - finalizedHeader *synchronization.FinalizedHeaderCache - checkAuthorizedAtBlock func(blockID flow.Identifier) (bool, error) - diskWAL *wal.DiskWAL - blockDataUploader *uploader.Manager - executionDataStore execution_data.ExecutionDataStore - toTriggerCheckpoint *atomic.Bool // create the checkpoint trigger to be controlled by admin tool, and listened by the compactor - stopControl *ingestion.StopControl // stop the node at given block height - executionDataDatastore *badger.Datastore - executionDataPruner *pruner.Pruner - executionDataBlobstore blobs.Blobstore - executionDataTracker tracker.Storage - blobService network.BlobService - blobserviceDependable *module.ProxiedReadyDoneAware + ingestionUnit *engine.Unit + + collector module.ExecutionMetrics + executionState state.ExecutionState + followerState protocol.FollowerState + committee hotstuff.DynamicCommittee + ledgerStorage *ledger.Ledger + events *storage.Events + serviceEvents *storage.ServiceEvents + txResults *storage.TransactionResults + results *storage.ExecutionResults + myReceipts *storage.MyExecutionReceipts + providerEngine *exeprovider.Engine + checkerEng *checker.Engine + syncCore *chainsync.Core + syncEngine *synchronization.Engine + followerCore *hotstuff.FollowerLoop // follower hotstuff logic + followerEng *followereng.ComplianceEngine // to sync blocks from consensus nodes + computationManager *computation.Manager + collectionRequester *requester.Engine + ingestionEng *ingestion.Engine + scriptsEng *scripts.Engine + followerDistributor *pubsub.FollowerDistributor + checkAuthorizedAtBlock func(blockID flow.Identifier) (bool, error) + diskWAL *wal.DiskWAL + blockDataUploader *uploader.Manager + executionDataStore execution_data.ExecutionDataStore + toTriggerCheckpoint *atomic.Bool // create the checkpoint trigger to be controlled by admin tool, and listened by the compactor + stopControl *stop.StopControl // stop the node at given block height + executionDataDatastore *badger.Datastore + executionDataPruner *pruner.Pruner + executionDataBlobstore blobs.Blobstore + executionDataTracker tracker.Storage + blobService network.BlobService + blobserviceDependable *module.ProxiedReadyDoneAware } func (builder *ExecutionNodeBuilder) LoadComponentsAndModules() { @@ -150,6 +155,7 @@ func (builder *ExecutionNodeBuilder) LoadComponentsAndModules() { builder: builder.FlowNodeBuilder, exeConf: builder.exeConf, toTriggerCheckpoint: atomic.NewBool(false), + ingestionUnit: engine.NewUnit(), } builder.FlowNodeBuilder. @@ -173,7 +179,7 @@ func (builder *ExecutionNodeBuilder) LoadComponentsAndModules() { Module("execution metrics", exeNode.LoadExecutionMetrics). Module("sync core", exeNode.LoadSyncCore). Module("execution receipts storage", exeNode.LoadExecutionReceiptsStorage). - Module("finalization distributor", exeNode.LoadFinalizationDistributor). + Module("follower distributor", exeNode.LoadFollowerDistributor). Module("authorization checking function", exeNode.LoadAuthorizationCheckingFunction). Module("execution data datastore", exeNode.LoadExecutionDataDatastore). Module("execution data getter", exeNode.LoadExecutionDataGetter). @@ -198,7 +204,7 @@ func (builder *ExecutionNodeBuilder) LoadComponentsAndModules() { Component("provider engine", exeNode.LoadProviderEngine). Component("checker engine", exeNode.LoadCheckerEngine). Component("ingestion engine", exeNode.LoadIngestionEngine). - Component("finalized snapshot", exeNode.LoadFinalizedSnapshot). + Component("scripts engine", exeNode.LoadScriptsEngine). Component("consensus committee", exeNode.LoadConsensusCommittee). Component("follower core", exeNode.LoadFollowerCore). Component("follower engine", exeNode.LoadFollowerEngine). @@ -272,9 +278,9 @@ func (exeNode *ExecutionNode) LoadExecutionReceiptsStorage( return nil } -func (exeNode *ExecutionNode) LoadFinalizationDistributor(node *NodeConfig) error { - exeNode.finalizationDistributor = pubsub.NewFinalizationDistributor() - exeNode.finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) +func (exeNode *ExecutionNode) LoadFollowerDistributor(node *NodeConfig) error { + exeNode.followerDistributor = pubsub.NewFollowerDistributor() + exeNode.followerDistributor.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil } @@ -473,7 +479,14 @@ func (exeNode *ExecutionNode) LoadProviderEngine( exeNode.executionDataTracker, ) - vmCtx := fvm.NewContext(node.FvmOptions...) + // in case node.FvmOptions already set a logger, we don't want to override it + opts := append([]fvm.Option{ + fvm.WithLogger( + node.Logger.With().Str("module", "FVM").Logger(), + )}, + node.FvmOptions..., + ) + vmCtx := fvm.NewContext(opts...) ledgerViewCommitter := committer.NewLedgerViewCommitter(exeNode.ledgerStorage, node.Tracer) manager, err := computation.New( @@ -653,17 +666,44 @@ func (exeNode *ExecutionNode) LoadStopControl( module.ReadyDoneAware, error, ) { - lastExecutedHeight, _, err := exeNode.executionState.GetHighestExecutedBlockID(context.TODO()) + ver, err := build.Semver() + if err != nil { + err = fmt.Errorf("could not set semver version for stop control. "+ + "version %s is not semver compliant: %w", build.Version(), err) + + // The node would not know its own version. Without this the node would not know + // how to reach to version boundaries. + exeNode.builder.Logger. + Err(err). + Msg("error starting stop control") + + return nil, err + } + + latestFinalizedBlock, err := node.State.Final().Head() if err != nil { - return nil, fmt.Errorf("cannot get the latest executed block height for stop control: %w", err) + return nil, fmt.Errorf("could not get latest finalized block: %w", err) } - exeNode.stopControl = ingestion.NewStopControl( - exeNode.builder.Logger.With().Str("compontent", "stop_control").Logger(), + stopControl := stop.NewStopControl( + exeNode.ingestionUnit, + exeNode.exeConf.maxGracefulStopDuration, + exeNode.builder.Logger, + exeNode.executionState, + node.Storage.Headers, + node.Storage.VersionBeacons, + ver, + latestFinalizedBlock, + // TODO: rename to exeNode.exeConf.executionStopped to make it more consistent exeNode.exeConf.pauseExecution, - lastExecutedHeight) + true, + ) + // stopControl needs to consume BlockFinalized events. + node.ProtocolEvents.AddConsumer(stopControl) - return &module.NoopReadyDoneAware{}, nil + exeNode.stopControl = stopControl + + return stopControl, nil } func (exeNode *ExecutionNode) LoadExecutionStateLedger( @@ -789,11 +829,13 @@ func (exeNode *ExecutionNode) LoadIngestionEngine( } exeNode.ingestionEng, err = ingestion.New( + exeNode.ingestionUnit, node.Logger, node.Network, node.Me, exeNode.collectionRequester, node.State, + node.Storage.Headers, node.Storage.Blocks, node.Storage.Collections, exeNode.events, @@ -809,6 +851,7 @@ func (exeNode *ExecutionNode) LoadIngestionEngine( exeNode.executionDataPruner, exeNode.blockDataUploader, exeNode.stopControl, + exeNode.exeConf.onflowOnlyLNs, ) // TODO: we should solve these mutual dependencies better @@ -820,6 +863,19 @@ func (exeNode *ExecutionNode) LoadIngestionEngine( return exeNode.ingestionEng, err } +// create scripts engine for handling script execution +func (exeNode *ExecutionNode) LoadScriptsEngine(node *NodeConfig) (module.ReadyDoneAware, error) { + // for RPC to load it + exeNode.scriptsEng = scripts.New( + node.Logger, + node.State, + exeNode.computationManager, + exeNode.executionState, + ) + + return exeNode.scriptsEng, nil +} + func (exeNode *ExecutionNode) LoadConsensusCommittee( node *NodeConfig, ) ( @@ -849,27 +905,22 @@ func (exeNode *ExecutionNode) LoadFollowerCore( // state when the follower detects newly finalized blocks final := finalizer.NewFinalizer(node.DB, node.Storage.Headers, exeNode.followerState, node.Tracer) - packer := signature.NewConsensusSigDataPacker(exeNode.committee) - // initialize the verifier for the protocol consensus - verifier := verification.NewCombinedVerifier(exeNode.committee, packer) - finalized, pending, err := recovery.FindLatest(node.State, node.Storage.Headers) if err != nil { return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - exeNode.finalizationDistributor.AddConsumer(exeNode.checkerEng) + exeNode.followerDistributor.AddFinalizationConsumer(exeNode.checkerEng) // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block exeNode.followerCore, err = consensus.NewFollower( node.Logger, - exeNode.committee, + node.Metrics.Mempool, node.Storage.Headers, final, - verifier, - exeNode.finalizationDistributor, - node.RootBlock.Header, + exeNode.followerDistributor, + node.FinalizedRootBlock.Header, node.RootQC, finalized, pending, @@ -901,7 +952,7 @@ func (exeNode *ExecutionNode) LoadFollowerEngine( node.Logger, node.Metrics.Mempool, heroCacheCollector, - exeNode.finalizationDistributor, + exeNode.followerDistributor, exeNode.followerState, exeNode.followerCore, validator, @@ -918,13 +969,14 @@ func (exeNode *ExecutionNode) LoadFollowerEngine( node.Me, node.Metrics.Engine, node.Storage.Headers, - exeNode.finalizedHeader.Get(), + node.LastFinalizedHeader, core, - followereng.WithComplianceConfigOpt(compliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold)), + node.ComplianceConfig, ) if err != nil { return nil, fmt.Errorf("could not create follower engine: %w", err) } + exeNode.followerDistributor.AddOnBlockFinalizedConsumer(exeNode.followerEng.OnFinalizedBlock) return exeNode.followerEng, nil } @@ -975,21 +1027,6 @@ func (exeNode *ExecutionNode) LoadReceiptProviderEngine( return eng, err } -func (exeNode *ExecutionNode) LoadFinalizedSnapshot( - node *NodeConfig, -) ( - module.ReadyDoneAware, - error, -) { - var err error - exeNode.finalizedHeader, err = synchronization.NewFinalizedHeaderCache(node.Logger, node.State, exeNode.finalizationDistributor) - if err != nil { - return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) - } - - return exeNode.finalizedHeader, nil -} - func (exeNode *ExecutionNode) LoadSynchronizationEngine( node *NodeConfig, ) ( @@ -1003,15 +1040,16 @@ func (exeNode *ExecutionNode) LoadSynchronizationEngine( node.Metrics.Engine, node.Network, node.Me, + node.State, node.Storage.Blocks, exeNode.followerEng, exeNode.syncCore, - exeNode.finalizedHeader, node.SyncEngineIdentifierProvider, ) if err != nil { return nil, fmt.Errorf("could not initialize synchronization engine: %w", err) } + exeNode.followerDistributor.AddFinalizationConsumer(exeNode.syncEngine) return exeNode.syncEngine, nil } @@ -1025,7 +1063,7 @@ func (exeNode *ExecutionNode) LoadGrpcServer( return rpc.New( node.Logger, exeNode.exeConf.rpcConf, - exeNode.ingestionEng, + exeNode.scriptsEng, node.Storage.Headers, node.State, exeNode.events, @@ -1060,7 +1098,7 @@ func (exeNode *ExecutionNode) LoadBootstrapper(node *NodeConfig) error { // TODO: check that the checkpoint file contains the root block's statecommit hash - err = bootstrapper.BootstrapExecutionDatabase(node.DB, node.RootSeal.FinalState, node.RootBlock.Header) + err = bootstrapper.BootstrapExecutionDatabase(node.DB, node.RootSeal) if err != nil { return fmt.Errorf("could not bootstrap execution database: %w", err) } @@ -1082,7 +1120,7 @@ func (exeNode *ExecutionNode) LoadBootstrapper(node *NodeConfig) error { func getContractEpochCounter( vm fvm.VM, vmCtx fvm.Context, - snapshot fvmState.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) ( uint64, error, diff --git a/cmd/execution_config.go b/cmd/execution_config.go index 860a5257593..1d7de8e5c8f 100644 --- a/cmd/execution_config.go +++ b/cmd/execution_config.go @@ -17,6 +17,7 @@ import ( "github.com/onflow/flow-go/utils/grpcutils" "github.com/onflow/flow-go/engine/execution/computation" + "github.com/onflow/flow-go/engine/execution/ingestion/stop" "github.com/onflow/flow-go/engine/execution/rpc" "github.com/onflow/flow-go/fvm/storage/derived" storage "github.com/onflow/flow-go/storage/badger" @@ -49,10 +50,17 @@ type ExecutionConfig struct { blobstoreRateLimit int blobstoreBurstLimit int chunkDataPackRequestWorkers uint + maxGracefulStopDuration time.Duration computationConfig computation.ComputationConfig receiptRequestWorkers uint // common provider engine workers receiptRequestsCacheSize uint32 // common provider engine cache size + + // This is included to temporarily work around an issue observed on a small number of ENs. + // It works around an issue where some collection nodes are not configured with enough + // this works around an issue where some collection nodes are not configured with enough + // file descriptors causing connection failures. + onflowOnlyLNs bool } func (exeConf *ExecutionConfig) SetupFlags(flags *pflag.FlagSet) { @@ -71,6 +79,7 @@ func (exeConf *ExecutionConfig) SetupFlags(flags *pflag.FlagSet) { "cache size for Cadence execution") flags.BoolVar(&exeConf.computationConfig.ExtensiveTracing, "extensive-tracing", false, "adds high-overhead tracing to execution") flags.BoolVar(&exeConf.computationConfig.CadenceTracing, "cadence-tracing", false, "enables cadence runtime level tracing") + flags.IntVar(&exeConf.computationConfig.MaxConcurrency, "computer-max-concurrency", 1, "set to greater than 1 to enable concurrent transaction execution") flags.UintVar(&exeConf.chunkDataPackCacheSize, "chdp-cache", storage.DefaultCacheSize, "cache size for chunk data packs") flags.Uint32Var(&exeConf.chunkDataPackRequestsCacheSize, "chdp-request-queue", mempool.DefaultChunkDataPackRequestQueueSize, "queue size for chunk data pack requests") flags.DurationVar(&exeConf.requestInterval, "request-interval", 60*time.Second, "the interval between requests for the requester engine") @@ -97,6 +106,9 @@ func (exeConf *ExecutionConfig) SetupFlags(flags *pflag.FlagSet) { flags.StringToIntVar(&exeConf.apiBurstlimits, "api-burst-limits", map[string]int{}, "burst limits for gRPC API methods e.g. Ping=100,ExecuteScriptAtBlockID=100 etc. note limits apply globally to all clients.") flags.IntVar(&exeConf.blobstoreRateLimit, "blobstore-rate-limit", 0, "per second outgoing rate limit for Execution Data blobstore") flags.IntVar(&exeConf.blobstoreBurstLimit, "blobstore-burst-limit", 0, "outgoing burst limit for Execution Data blobstore") + flags.DurationVar(&exeConf.maxGracefulStopDuration, "max-graceful-stop-duration", stop.DefaultMaxGracefulStopDuration, "the maximum amount of time stop control will wait for ingestion engine to gracefully shutdown before crashing") + + flags.BoolVar(&exeConf.onflowOnlyLNs, "temp-onflow-only-lns", false, "do not use unless required. forces node to only request collections from onflow collection nodes") } func (exeConf *ExecutionConfig) ValidateFlags() error { diff --git a/cmd/node_builder.go b/cmd/node_builder.go index 97d0ea40093..9fb490d3f02 100644 --- a/cmd/node_builder.go +++ b/cmd/node_builder.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/pflag" "github.com/onflow/flow-go/admin/commands" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/model/flow" @@ -25,13 +26,6 @@ import ( "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/codec/cbor" "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/connection" - "github.com/onflow/flow-go/network/p2p/distributor" - "github.com/onflow/flow-go/network/p2p/dns" - "github.com/onflow/flow-go/network/p2p/middleware" - "github.com/onflow/flow-go/network/p2p/p2pbuilder" - inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" - "github.com/onflow/flow-go/network/p2p/unicast" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/events" bstorage "github.com/onflow/flow-go/storage/badger" @@ -141,7 +135,6 @@ type NodeBuilder interface { // For a node running as a standalone process, the config fields will be populated from the command line params, // while for a node running as a library, the config fields are expected to be initialized by the caller. type BaseConfig struct { - NetworkConfig nodeIDHex string AdminAddr string AdminCert string @@ -177,47 +170,9 @@ type BaseConfig struct { // ComplianceConfig configures either the compliance engine (consensus nodes) // or the follower engine (all other node roles) ComplianceConfig compliance.Config -} - -type NetworkConfig struct { - // NetworkConnectionPruning determines whether connections to nodes - // that are not part of protocol state should be trimmed - // TODO: solely a fallback mechanism, can be removed upon reliable behavior in production. - NetworkConnectionPruning bool - // GossipSubConfig core gossipsub configuration. - GossipSubConfig *p2pbuilder.GossipSubConfig - // GossipSubRPCInspectorsConfig configuration for all gossipsub RPC control message inspectors. - GossipSubRPCInspectorsConfig *inspectorbuilder.GossipSubRPCInspectorsConfig - // PreferredUnicastProtocols list of unicast protocols in preferred order - PreferredUnicastProtocols []string - NetworkReceivedMessageCacheSize uint32 - - PeerUpdateInterval time.Duration - UnicastMessageTimeout time.Duration - DNSCacheTTL time.Duration - LibP2PResourceManagerConfig *p2pbuilder.ResourceManagerConfig - ConnectionManagerConfig *connection.ManagerConfig - // UnicastCreateStreamRetryDelay initial delay used in the exponential backoff for create stream retries - UnicastCreateStreamRetryDelay time.Duration - // size of the queue for notifications about new peers in the disallow list. - DisallowListNotificationCacheSize uint32 - // UnicastRateLimitersConfig configuration for all unicast rate limiters. - UnicastRateLimitersConfig *UnicastRateLimitersConfig -} -// UnicastRateLimitersConfig unicast rate limiter configuration for the message and bandwidth rate limiters. -type UnicastRateLimitersConfig struct { - // DryRun setting this to true will disable connection disconnects and gating when unicast rate limiters are configured - DryRun bool - // LockoutDuration the number of seconds a peer will be forced to wait before being allowed to successful reconnect to the node - // after being rate limited. - LockoutDuration time.Duration - // MessageRateLimit amount of unicast messages that can be sent by a peer per second. - MessageRateLimit int - // BandwidthRateLimit bandwidth size in bytes a peer is allowed to send via unicast streams per second. - BandwidthRateLimit int - // BandwidthBurstLimit bandwidth size in bytes a peer is allowed to send via unicast streams at once. - BandwidthBurstLimit int + // FlowConfig Flow configuration. + FlowConfig config.FlowConfig } // NodeConfig contains all the derived parameters such the NodeID, private keys etc. and initialized instances of @@ -260,23 +215,31 @@ type NodeConfig struct { // root state information RootSnapshot protocol.Snapshot - // cached properties of RootSnapshot for convenience - RootBlock *flow.Block - RootQC *flow.QuorumCertificate - RootResult *flow.ExecutionResult - RootSeal *flow.Seal - RootChainID flow.ChainID - SporkID flow.Identifier + // excerpt of root snapshot and latest finalized snapshot, when we boot up + StateExcerptAtBoot // bootstrapping options SkipNwAddressBasedValidations bool // UnicastRateLimiterDistributor notifies consumers when a peer's unicast message is rate limited. UnicastRateLimiterDistributor p2p.UnicastRateLimiterDistributor - // NodeDisallowListDistributor notifies consumers of updates to disallow listing of nodes. - NodeDisallowListDistributor p2p.DisallowListNotificationDistributor - // GossipSubInspectorNotifDistributor notifies consumers when an invalid RPC message is encountered. - GossipSubInspectorNotifDistributor p2p.GossipSubInspectorNotificationDistributor +} + +// StateExcerptAtBoot stores information about the root snapshot and latest finalized block for use in bootstrapping. +type StateExcerptAtBoot struct { + // properties of RootSnapshot for convenience + // For node bootstrapped with a root snapshot for the first block of a spork, + // FinalizedRootBlock and SealedRootBlock are the same block (special case of self-sealing block) + // For node bootstrapped with a root snapshot for a block above the first block of a spork (dynamically bootstrapped), + // FinalizedRootBlock.Height > SealedRootBlock.Height + FinalizedRootBlock *flow.Block // The last finalized block when bootstrapped. + SealedRootBlock *flow.Block // The last sealed block when bootstrapped. + RootQC *flow.QuorumCertificate // QC for Finalized Root Block + RootResult *flow.ExecutionResult // Result for SealedRootBlock + RootSeal *flow.Seal //Seal for RootResult + RootChainID flow.ChainID + SporkID flow.Identifier + LastFinalizedHeader *flow.Header // last finalized header when the node boots up } func DefaultBaseConfig() *BaseConfig { @@ -288,26 +251,6 @@ func DefaultBaseConfig() *BaseConfig { codecFactory := func() network.Codec { return cbor.NewCodec() } return &BaseConfig{ - NetworkConfig: NetworkConfig{ - UnicastCreateStreamRetryDelay: unicast.DefaultRetryDelay, - PeerUpdateInterval: connection.DefaultPeerUpdateInterval, - UnicastMessageTimeout: middleware.DefaultUnicastTimeout, - NetworkReceivedMessageCacheSize: p2p.DefaultReceiveCacheSize, - UnicastRateLimitersConfig: &UnicastRateLimitersConfig{ - DryRun: true, - LockoutDuration: 10, - MessageRateLimit: 0, - BandwidthRateLimit: 0, - BandwidthBurstLimit: middleware.LargeMsgMaxUnicastMsgSize, - }, - GossipSubConfig: p2pbuilder.DefaultGossipSubConfig(), - GossipSubRPCInspectorsConfig: inspectorbuilder.DefaultGossipSubRPCInspectorsConfig(), - DNSCacheTTL: dns.DefaultTimeToLive, - LibP2PResourceManagerConfig: p2pbuilder.DefaultResourceManagerConfig(), - ConnectionManagerConfig: connection.DefaultConnManagerConfig(), - NetworkConnectionPruning: connection.ConnectionPruningEnabled, - DisallowListNotificationCacheSize: distributor.DefaultDisallowListNotificationQueueCacheSize, - }, nodeIDHex: NotSet, AdminAddr: NotSet, AdminCert: NotSet, diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index c28e215fa2c..1e7687578c2 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -6,17 +6,13 @@ import ( "encoding/json" "errors" "fmt" - "os" - "path/filepath" "strings" "time" - badger "github.com/ipfs/go-ds-badger2" dht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" - "github.com/onflow/go-bitswap" "github.com/rs/zerolog" "github.com/spf13/pflag" @@ -32,8 +28,11 @@ import ( recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/access/apiproxy" + restapiproxy "github.com/onflow/flow-go/engine/access/rest/apiproxy" + "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" + rpcConnection "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/common/follower" synceng "github.com/onflow/flow-go/engine/common/synchronization" "github.com/onflow/flow-go/engine/protocol" @@ -42,41 +41,36 @@ import ( "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/chainsync" - "github.com/onflow/flow-go/module/compliance" - "github.com/onflow/flow-go/module/executiondatasync/execution_data" finalizer "github.com/onflow/flow-go/module/finalizer/consensus" + "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/local" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/module/state_synchronization" - edrequester "github.com/onflow/flow-go/module/state_synchronization/requester" consensus_follower "github.com/onflow/flow-go/module/upstream" "github.com/onflow/flow-go/network" + alspmgr "github.com/onflow/flow-go/network/alsp/manager" netcache "github.com/onflow/flow-go/network/cache" "github.com/onflow/flow-go/network/channels" cborcodec "github.com/onflow/flow-go/network/codec/cbor" "github.com/onflow/flow-go/network/converter" "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/blob" "github.com/onflow/flow-go/network/p2p/cache" + "github.com/onflow/flow-go/network/p2p/conduit" p2pdht "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/keyutils" "github.com/onflow/flow-go/network/p2p/middleware" "github.com/onflow/flow-go/network/p2p/p2pbuilder" - "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/validator" stateprotocol "github.com/onflow/flow-go/state/protocol" badgerState "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/blocktimer" "github.com/onflow/flow-go/state/protocol/events/gadgets" - "github.com/onflow/flow-go/storage" - bstorage "github.com/onflow/flow-go/storage/badger" "github.com/onflow/flow-go/utils/grpcutils" "github.com/onflow/flow-go/utils/io" ) @@ -109,10 +103,6 @@ type ObserverServiceConfig struct { apiBurstlimits map[string]int rpcConf rpc.Config rpcMetricsEnabled bool - executionDataSyncEnabled bool - executionDataDir string - executionDataStartHeight uint64 - executionDataConfig edrequester.ExecutionDataConfig apiTimeout time.Duration upstreamNodeAddresses []string upstreamNodePublicKeys []string @@ -121,21 +111,24 @@ type ObserverServiceConfig struct { // DefaultObserverServiceConfig defines all the default values for the ObserverServiceConfig func DefaultObserverServiceConfig() *ObserverServiceConfig { - homedir, _ := os.UserHomeDir() return &ObserverServiceConfig{ rpcConf: rpc.Config{ - UnsecureGRPCListenAddr: "0.0.0.0:9000", - SecureGRPCListenAddr: "0.0.0.0:9001", - HTTPListenAddr: "0.0.0.0:8000", - RESTListenAddr: "", - CollectionAddr: "", - HistoricalAccessAddrs: "", - CollectionClientTimeout: 3 * time.Second, - ExecutionClientTimeout: 3 * time.Second, - MaxHeightRange: backend.DefaultMaxHeightRange, - PreferredExecutionNodeIDs: nil, - FixedExecutionNodeIDs: nil, - MaxMsgSize: grpcutils.DefaultMaxMsgSize, + UnsecureGRPCListenAddr: "0.0.0.0:9000", + SecureGRPCListenAddr: "0.0.0.0:9001", + HTTPListenAddr: "0.0.0.0:8000", + RESTListenAddr: "", + CollectionAddr: "", + HistoricalAccessAddrs: "", + BackendConfig: backend.Config{ + CollectionClientTimeout: 3 * time.Second, + ExecutionClientTimeout: 3 * time.Second, + ConnectionPoolSize: backend.DefaultConnectionPoolSize, + MaxHeightRange: backend.DefaultMaxHeightRange, + PreferredExecutionNodeIDs: nil, + FixedExecutionNodeIDs: nil, + ArchiveAddressList: nil, + }, + MaxMsgSize: grpcutils.DefaultMaxMsgSize, }, rpcMetricsEnabled: false, apiRatelimits: nil, @@ -143,19 +136,9 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig { bootstrapNodeAddresses: []string{}, bootstrapNodePublicKeys: []string{}, observerNetworkingKeyPath: cmd.NotSet, - executionDataSyncEnabled: false, - executionDataDir: filepath.Join(homedir, ".flow", "execution_data"), - executionDataStartHeight: 0, - executionDataConfig: edrequester.ExecutionDataConfig{ - InitialBlockHeight: 0, - MaxSearchAhead: edrequester.DefaultMaxSearchAhead, - FetchTimeout: edrequester.DefaultFetchTimeout, - RetryDelay: edrequester.DefaultRetryDelay, - MaxRetryDelay: edrequester.DefaultMaxRetryDelay, - }, - apiTimeout: 3 * time.Second, - upstreamNodeAddresses: []string{}, - upstreamNodePublicKeys: []string{}, + apiTimeout: 3 * time.Second, + upstreamNodeAddresses: []string{}, + upstreamNodePublicKeys: []string{}, } } @@ -166,19 +149,16 @@ type ObserverServiceBuilder struct { *ObserverServiceConfig // components - LibP2PNode p2p.LibP2PNode - FollowerState stateprotocol.FollowerState - SyncCore *chainsync.Core - RpcEng *rpc.Engine - FinalizationDistributor *pubsub.FinalizationDistributor - FinalizedHeader *synceng.FinalizedHeaderCache - Committee hotstuff.DynamicCommittee - Finalized *flow.Header - Pending []*flow.Header - FollowerCore module.HotStuffFollower - Validator hotstuff.Validator - ExecutionDataDownloader execution_data.Downloader - ExecutionDataRequester state_synchronization.ExecutionDataRequester // for the observer, the sync engine participants provider is the libp2p peer store which is not + LibP2PNode p2p.LibP2PNode + FollowerState stateprotocol.FollowerState + SyncCore *chainsync.Core + RpcEng *rpc.Engine + FollowerDistributor *pubsub.FollowerDistributor + Committee hotstuff.DynamicCommittee + Finalized *flow.Header + Pending []*flow.Header + FollowerCore module.HotStuffFollower + // available until after the network has started. Hence, a factory function that needs to be called just before // creating the sync engine SyncEngineParticipantsProviderFactory func() module.IdentifierProvider @@ -189,6 +169,12 @@ type ObserverServiceBuilder struct { // Public network peerID peer.ID + + RestMetrics *metrics.RestCollector + AccessMetrics module.AccessMetrics + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer } // deriveBootstrapPeerIdentities derives the Flow Identity of the bootstrap peers from the parameters. @@ -329,19 +315,13 @@ func (builder *ObserverServiceBuilder) buildFollowerCore() *ObserverServiceBuild // state when the follower detects newly finalized blocks final := finalizer.NewFinalizer(node.DB, node.Storage.Headers, builder.FollowerState, node.Tracer) - packer := hotsignature.NewConsensusSigDataPacker(builder.Committee) - // initialize the verifier for the protocol consensus - verifier := verification.NewCombinedVerifier(builder.Committee, packer) - builder.Validator = hotstuffvalidator.New(builder.Committee, verifier) - followerCore, err := consensus.NewFollower( node.Logger, - builder.Committee, + node.Metrics.Mempool, node.Storage.Headers, final, - verifier, - builder.FinalizationDistributor, - node.RootBlock.Header, + builder.FollowerDistributor, + node.FinalizedRootBlock.Header, node.RootQC, builder.Finalized, builder.Pending, @@ -363,14 +343,18 @@ func (builder *ObserverServiceBuilder) buildFollowerEngine() *ObserverServiceBui if node.HeroCacheMetricsEnable { heroCacheCollector = metrics.FollowerCacheMetrics(node.MetricsRegisterer) } + packer := hotsignature.NewConsensusSigDataPacker(builder.Committee) + verifier := verification.NewCombinedVerifier(builder.Committee, packer) // verifier for HotStuff signature constructs (QCs, TCs, votes) + val := hotstuffvalidator.New(builder.Committee, verifier) + core, err := follower.NewComplianceCore( node.Logger, node.Metrics.Mempool, heroCacheCollector, - builder.FinalizationDistributor, + builder.FollowerDistributor, builder.FollowerState, builder.FollowerCore, - builder.Validator, + val, builder.SyncCore, node.Tracer, ) @@ -386,12 +370,13 @@ func (builder *ObserverServiceBuilder) buildFollowerEngine() *ObserverServiceBui node.Storage.Headers, builder.Finalized, core, - follower.WithComplianceConfigOpt(compliance.WithSkipNewProposalsThreshold(builder.ComplianceConfig.SkipNewProposalsThreshold)), + builder.ComplianceConfig, follower.WithChannel(channels.PublicReceiveBlocks), ) if err != nil { return nil, fmt.Errorf("could not create follower engine: %w", err) } + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.FollowerEng.OnFinalizedBlock) return builder.FollowerEng, nil }) @@ -399,20 +384,6 @@ func (builder *ObserverServiceBuilder) buildFollowerEngine() *ObserverServiceBui return builder } -func (builder *ObserverServiceBuilder) buildFinalizedHeader() *ObserverServiceBuilder { - builder.Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - finalizedHeader, err := synceng.NewFinalizedHeaderCache(node.Logger, node.State, builder.FinalizationDistributor) - if err != nil { - return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) - } - builder.FinalizedHeader = finalizedHeader - - return builder.FinalizedHeader, nil - }) - - return builder -} - func (builder *ObserverServiceBuilder) buildSyncEngine() *ObserverServiceBuilder { builder.Component("sync engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { sync, err := synceng.New( @@ -420,16 +391,17 @@ func (builder *ObserverServiceBuilder) buildSyncEngine() *ObserverServiceBuilder node.Metrics.Engine, node.Network, node.Me, + node.State, node.Storage.Blocks, builder.FollowerEng, builder.SyncCore, - builder.FinalizedHeader, builder.SyncEngineParticipantsProviderFactory(), ) if err != nil { return nil, fmt.Errorf("could not create synchronization engine: %w", err) } builder.SyncEng = sync + builder.FollowerDistributor.AddFinalizationConsumer(sync) return builder.SyncEng, nil }) @@ -445,118 +417,11 @@ func (builder *ObserverServiceBuilder) BuildConsensusFollower() cmd.NodeBuilder buildLatestHeader(). buildFollowerCore(). buildFollowerEngine(). - buildFinalizedHeader(). buildSyncEngine() return builder } -func (builder *ObserverServiceBuilder) BuildExecutionDataRequester() *ObserverServiceBuilder { - var ds *badger.Datastore - var bs network.BlobService - var processedBlockHeight storage.ConsumerProgress - var processedNotifications storage.ConsumerProgress - - builder. - Module("execution data datastore and blobstore", func(node *cmd.NodeConfig) error { - err := os.MkdirAll(builder.executionDataDir, 0700) - if err != nil { - return err - } - - ds, err = badger.NewDatastore(builder.executionDataDir, &badger.DefaultOptions) - if err != nil { - return err - } - - builder.ShutdownFunc(func() error { - if err := ds.Close(); err != nil { - return fmt.Errorf("could not close execution data datastore: %w", err) - } - return nil - }) - - return nil - }). - Module("processed block height consumer progress", func(node *cmd.NodeConfig) error { - // uses the datastore's DB - processedBlockHeight = bstorage.NewConsumerProgress(ds.DB, module.ConsumeProgressExecutionDataRequesterBlockHeight) - return nil - }). - Module("processed notifications consumer progress", func(node *cmd.NodeConfig) error { - // uses the datastore's DB - processedNotifications = bstorage.NewConsumerProgress(ds.DB, module.ConsumeProgressExecutionDataRequesterNotification) - return nil - }). - Component("execution data service", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - var err error - bs, err = node.Network.RegisterBlobService(channels.ExecutionDataService, ds, - blob.WithBitswapOptions( - bitswap.WithTracer( - blob.NewTracer(node.Logger.With().Str("blob_service", channels.ExecutionDataService.String()).Logger()), - ), - ), - ) - if err != nil { - return nil, fmt.Errorf("could not register blob service: %w", err) - } - - builder.ExecutionDataDownloader = execution_data.NewDownloader(bs) - - return builder.ExecutionDataDownloader, nil - }). - Component("execution data requester", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - // Validation of the start block height needs to be done after loading state - if builder.executionDataStartHeight > 0 { - if builder.executionDataStartHeight <= builder.RootBlock.Header.Height { - return nil, fmt.Errorf( - "execution data start block height (%d) must be greater than the root block height (%d)", - builder.executionDataStartHeight, builder.RootBlock.Header.Height) - } - - latestSeal, err := builder.State.Sealed().Head() - if err != nil { - return nil, fmt.Errorf("failed to get latest sealed height") - } - - // Note: since the root block of a spork is also sealed in the root protocol state, the - // latest sealed height is always equal to the root block height. That means that at the - // very beginning of a spork, this check will always fail. Operators should not specify - // an InitialBlockHeight when starting from the beginning of a spork. - if builder.executionDataStartHeight > latestSeal.Height { - return nil, fmt.Errorf( - "execution data start block height (%d) must be less than or equal to the latest sealed block height (%d)", - builder.executionDataStartHeight, latestSeal.Height) - } - - // executionDataStartHeight is provided as the first block to sync, but the - // requester expects the initial last processed height, which is the first height - 1 - builder.executionDataConfig.InitialBlockHeight = builder.executionDataStartHeight - 1 - } else { - builder.executionDataConfig.InitialBlockHeight = builder.RootBlock.Header.Height - } - - builder.ExecutionDataRequester = edrequester.New( - builder.Logger, - metrics.NewExecutionDataRequesterCollector(), - builder.ExecutionDataDownloader, - processedBlockHeight, - processedNotifications, - builder.State, - builder.Storage.Headers, - builder.Storage.Results, - builder.Storage.Seals, - builder.executionDataConfig, - ) - - builder.FinalizationDistributor.AddOnBlockFinalizedConsumer(builder.ExecutionDataRequester.OnBlockFinalized) - - return builder.ExecutionDataRequester, nil - }) - - return builder -} - type Option func(*ObserverServiceConfig) func NewFlowObserverServiceBuilder(opts ...Option) *ObserverServiceBuilder { @@ -565,11 +430,11 @@ func NewFlowObserverServiceBuilder(opts ...Option) *ObserverServiceBuilder { opt(config) } anb := &ObserverServiceBuilder{ - ObserverServiceConfig: config, - FlowNodeBuilder: cmd.FlowNode("observer"), - FinalizationDistributor: pubsub.NewFinalizationDistributor(), + ObserverServiceConfig: config, + FlowNodeBuilder: cmd.FlowNode("observer"), + FollowerDistributor: pubsub.NewFollowerDistributor(), } - anb.FinalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) + anb.FollowerDistributor.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(anb.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address anb.FlowNodeBuilder.SkipNwAddressBasedValidations = true @@ -594,7 +459,8 @@ func (builder *ObserverServiceBuilder) extraFlags() { flags.StringVarP(&builder.rpcConf.HTTPListenAddr, "http-addr", "h", defaultConfig.rpcConf.HTTPListenAddr, "the address the http proxy server listens on") flags.StringVar(&builder.rpcConf.RESTListenAddr, "rest-addr", defaultConfig.rpcConf.RESTListenAddr, "the address the REST server listens on (if empty the REST server will not be started)") flags.UintVar(&builder.rpcConf.MaxMsgSize, "rpc-max-message-size", defaultConfig.rpcConf.MaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") - flags.UintVar(&builder.rpcConf.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.MaxHeightRange, "maximum size for height range requests") + flags.UintVar(&builder.rpcConf.BackendConfig.ConnectionPoolSize, "connection-pool-size", defaultConfig.rpcConf.BackendConfig.ConnectionPoolSize, "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") + flags.UintVar(&builder.rpcConf.BackendConfig.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.BackendConfig.MaxHeightRange, "maximum size for height range requests") flags.StringToIntVar(&builder.apiRatelimits, "api-rate-limits", defaultConfig.apiRatelimits, "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.") flags.StringToIntVar(&builder.apiBurstlimits, "api-burst-limits", defaultConfig.apiBurstlimits, "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") flags.StringVar(&builder.observerNetworkingKeyPath, "observer-networking-key-path", defaultConfig.observerNetworkingKeyPath, "path to the networking key for observer") @@ -604,31 +470,6 @@ func (builder *ObserverServiceBuilder) extraFlags() { flags.StringSliceVar(&builder.upstreamNodeAddresses, "upstream-node-addresses", defaultConfig.upstreamNodeAddresses, "the gRPC network addresses of the upstream access node. e.g. access-001.mainnet.flow.org:9000,access-002.mainnet.flow.org:9000") flags.StringSliceVar(&builder.upstreamNodePublicKeys, "upstream-node-public-keys", defaultConfig.upstreamNodePublicKeys, "the networking public key of the upstream access node (in the same order as the upstream node addresses) e.g. \"d57a5e9c5.....\",\"44ded42d....\"") flags.BoolVar(&builder.rpcMetricsEnabled, "rpc-metrics-enabled", defaultConfig.rpcMetricsEnabled, "whether to enable the rpc metrics") - - // ExecutionDataRequester config - flags.BoolVar(&builder.executionDataSyncEnabled, "execution-data-sync-enabled", defaultConfig.executionDataSyncEnabled, "whether to enable the execution data sync protocol") - flags.StringVar(&builder.executionDataDir, "execution-data-dir", defaultConfig.executionDataDir, "directory to use for Execution Data database") - flags.Uint64Var(&builder.executionDataStartHeight, "execution-data-start-height", defaultConfig.executionDataStartHeight, "height of first block to sync execution data from when starting with an empty Execution Data database") - flags.Uint64Var(&builder.executionDataConfig.MaxSearchAhead, "execution-data-max-search-ahead", defaultConfig.executionDataConfig.MaxSearchAhead, "max number of heights to search ahead of the lowest outstanding execution data height") - flags.DurationVar(&builder.executionDataConfig.FetchTimeout, "execution-data-fetch-timeout", defaultConfig.executionDataConfig.FetchTimeout, "timeout to use when fetching execution data from the network e.g. 300s") - flags.DurationVar(&builder.executionDataConfig.RetryDelay, "execution-data-retry-delay", defaultConfig.executionDataConfig.RetryDelay, "initial delay for exponential backoff when fetching execution data fails e.g. 10s") - flags.DurationVar(&builder.executionDataConfig.MaxRetryDelay, "execution-data-max-retry-delay", defaultConfig.executionDataConfig.MaxRetryDelay, "maximum delay for exponential backoff when fetching execution data fails e.g. 5m") - }).ValidateFlags(func() error { - if builder.executionDataSyncEnabled { - if builder.executionDataConfig.FetchTimeout <= 0 { - return errors.New("execution-data-fetch-timeout must be greater than 0") - } - if builder.executionDataConfig.RetryDelay <= 0 { - return errors.New("execution-data-retry-delay must be greater than 0") - } - if builder.executionDataConfig.MaxRetryDelay < builder.executionDataConfig.RetryDelay { - return errors.New("execution-data-max-retry-delay must be greater than or equal to execution-data-retry-delay") - } - if builder.executionDataConfig.MaxSearchAhead == 0 { - return errors.New("execution-data-max-search-ahead must be greater than 0") - } - } - return nil }) } @@ -641,9 +482,7 @@ func (builder *ObserverServiceBuilder) initNetwork(nodeID module.Local, topology network.Topology, receiveCache *netcache.ReceiveCache, ) (*p2p.Network, error) { - - // creates network instance - net, err := p2p.NewNetwork(&p2p.NetworkParameters{ + net, err := p2p.NewNetwork(&p2p.NetworkConfig{ Logger: builder.Logger, Codec: cborcodec.NewCodec(), Me: nodeID, @@ -653,6 +492,17 @@ func (builder *ObserverServiceBuilder) initNetwork(nodeID module.Local, Metrics: networkMetrics, IdentityProvider: builder.IdentityProvider, ReceiveCache: receiveCache, + ConduitFactory: conduit.NewDefaultConduitFactory(), + AlspCfg: &alspmgr.MisbehaviorReportManagerConfig{ + Logger: builder.Logger, + SpamRecordCacheSize: builder.FlowConfig.NetworkConfig.AlspConfig.SpamRecordCacheSize, + SpamReportQueueSize: builder.FlowConfig.NetworkConfig.AlspConfig.SpamReportQueueSize, + DisablePenalty: builder.FlowConfig.NetworkConfig.AlspConfig.DisablePenalty, + HeartBeatInterval: builder.FlowConfig.NetworkConfig.AlspConfig.HearBeatInterval, + AlspMetrics: builder.Metrics.Network, + HeroCacheMetricsFactory: builder.HeroCacheMetricsFactory(), + NetworkType: network.PublicNetwork, + }, }) if err != nil { return nil, fmt.Errorf("could not initialize network: %w", err) @@ -743,11 +593,11 @@ func (builder *ObserverServiceBuilder) InitIDProviders() { } builder.IDTranslator = translator.NewHierarchicalIDTranslator(idCache, translator.NewPublicNetworkIDTranslator()) - builder.NodeDisallowListDistributor = cmd.BuildDisallowListNotificationDisseminator(builder.DisallowListNotificationCacheSize, builder.MetricsRegisterer, builder.Logger, builder.MetricsEnabled) - // The following wrapper allows to black-list byzantine nodes via an admin command: // the wrapper overrides the 'Ejected' flag of disallow-listed nodes to true - builder.IdentityProvider, err = cache.NewNodeBlocklistWrapper(idCache, node.DB, builder.NodeDisallowListDistributor) + builder.IdentityProvider, err = cache.NewNodeDisallowListWrapper(idCache, node.DB, func() network.DisallowListNotificationConsumer { + return builder.Middleware + }) if err != nil { return fmt.Errorf("could not initialize NodeBlockListWrapper: %w", err) } @@ -778,11 +628,6 @@ func (builder *ObserverServiceBuilder) InitIDProviders() { return nil }) - - builder.Component("disallow list notification distributor", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - // distributor is returned as a component to be started and stopped. - return builder.NodeDisallowListDistributor, nil - }) } func (builder *ObserverServiceBuilder) Initialize() error { @@ -844,7 +689,7 @@ func (builder *ObserverServiceBuilder) validateParams() error { return nil } -// initPublicLibP2PFactory creates the LibP2P factory function for the given node ID and network key for the observer. +// initPublicLibp2pNode creates a libp2p node for the observer service in the public (unstaked) network. // The factory function is later passed into the initMiddleware function to eventually instantiate the p2p.LibP2PNode instance // The LibP2P host is created with the following options: // * DHT as client and seeded with the given bootstrap peers @@ -853,70 +698,80 @@ func (builder *ObserverServiceBuilder) validateParams() error { // * No connection gater // * No connection manager // * No peer manager -// * Default libp2p pubsub options -func (builder *ObserverServiceBuilder) initPublicLibP2PFactory(networkKey crypto.PrivateKey) p2p.LibP2PFactoryFunc { - return func() (p2p.LibP2PNode, error) { - var pis []peer.AddrInfo - - for _, b := range builder.bootstrapIdentities { - pi, err := utils.PeerAddressInfo(*b) - - if err != nil { - return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err) - } - - pis = append(pis, pi) - } - - meshTracer := tracer.NewGossipSubMeshTracer( - builder.Logger, - builder.Metrics.Network, - builder.IdentityProvider, - builder.GossipSubConfig.LocalMeshLogInterval) - - rpcInspectorBuilder := inspector.NewGossipSubInspectorBuilder(builder.Logger, builder.SporkID, builder.GossipSubRPCInspectorsConfig, builder.GossipSubInspectorNotifDistributor) - rpcInspectors, err := rpcInspectorBuilder. - SetPublicNetwork(p2p.PublicNetworkEnabled). - SetMetrics(builder.Metrics.Network, builder.MetricsRegisterer). - SetMetricsEnabled(builder.MetricsEnabled).Build() +// * Default libp2p pubsub options. +// Args: +// - networkKey: the private key to use for the libp2p node +// Returns: +// - p2p.LibP2PNode: the libp2p node +// - error: if any error occurs. Any error returned is considered irrecoverable. +func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.PrivateKey) (p2p.LibP2PNode, error) { + var pis []peer.AddrInfo + + for _, b := range builder.bootstrapIdentities { + pi, err := utils.PeerAddressInfo(*b) if err != nil { - return nil, fmt.Errorf("failed to create gossipsub rpc inspectors: %w", err) + return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err) } - node, err := p2pbuilder.NewNodeBuilder( - builder.Logger, - builder.Metrics.Network, - builder.BaseConfig.BindAddr, - networkKey, - builder.SporkID, - builder.LibP2PResourceManagerConfig). - SetSubscriptionFilter( - subscription.NewRoleBasedFilter( - subscription.UnstakedRole, builder.IdentityProvider, - ), - ). - SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { - return p2pdht.NewDHT(ctx, h, protocols.FlowPublicDHTProtocolID(builder.SporkID), - builder.Logger, - builder.Metrics.Network, - p2pdht.AsClient(), - dht.BootstrapPeers(pis...), - ) - }). - SetStreamCreationRetryInterval(builder.UnicastCreateStreamRetryDelay). - SetGossipSubTracer(meshTracer). - SetGossipSubScoreTracerInterval(builder.GossipSubConfig.ScoreTracerInterval). - SetGossipSubRPCInspectors(rpcInspectors...). - Build() + pis = append(pis, pi) + } - if err != nil { - return nil, err - } + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: builder.Logger, + Metrics: builder.Metrics.Network, + IDProvider: builder.IdentityProvider, + LoggerInterval: builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, + RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + RpcSentTrackerWorkerQueueCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, + RpcSentTrackerNumOfWorkers: builder.FlowConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, + HeroCacheMetricsFactory: builder.HeroCacheMetricsFactory(), + NetworkingType: network.PublicNetwork, + } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) - builder.LibP2PNode = node + node, err := p2pbuilder.NewNodeBuilder( + builder.Logger, + &p2pconfig.MetricsConfig{ + HeroCacheFactory: builder.HeroCacheMetricsFactory(), + Metrics: builder.Metrics.Network, + }, + network.PublicNetwork, + builder.BaseConfig.BindAddr, + networkKey, + builder.SporkID, + builder.IdentityProvider, + &builder.FlowConfig.NetworkConfig.ResourceManagerConfig, + &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, + p2pconfig.PeerManagerDisableConfig(), // disable peer manager for observer node. + &p2p.DisallowListCacheConfig{ + MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, + Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), + }). + SetSubscriptionFilter( + subscription.NewRoleBasedFilter( + subscription.UnstakedRole, builder.IdentityProvider, + ), + ). + SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { + return p2pdht.NewDHT(ctx, h, protocols.FlowPublicDHTProtocolID(builder.SporkID), + builder.Logger, + builder.Metrics.Network, + p2pdht.AsClient(), + dht.BootstrapPeers(pis...), + ) + }). + SetStreamCreationRetryInterval(builder.FlowConfig.NetworkConfig.UnicastCreateStreamRetryDelay). + SetGossipSubTracer(meshTracer). + SetGossipSubScoreTracerInterval(builder.FlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval). + Build() - return builder.LibP2PNode, nil + if err != nil { + return nil, fmt.Errorf("could not initialize libp2p node for observer: %w", err) } + + builder.LibP2PNode = node + + return builder.LibP2PNode, nil } // initObserverLocal initializes the observer's ID, network key and network address @@ -946,44 +801,35 @@ func (builder *ObserverServiceBuilder) initObserverLocal() func(node *cmd.NodeCo // Currently, the observer only runs the follower engine. func (builder *ObserverServiceBuilder) Build() (cmd.Node, error) { builder.BuildConsensusFollower() - if builder.executionDataSyncEnabled { - builder.BuildExecutionDataRequester() - } return builder.FlowNodeBuilder.Build() } // enqueuePublicNetworkInit enqueues the observer network component initialized for the observer func (builder *ObserverServiceBuilder) enqueuePublicNetworkInit() { - var libp2pNode p2p.LibP2PNode + var publicLibp2pNode p2p.LibP2PNode builder. Component("public libp2p node", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - libP2PFactory := builder.initPublicLibP2PFactory(node.NetworkKey) - var err error - libp2pNode, err = libP2PFactory() + publicLibp2pNode, err = builder.initPublicLibp2pNode(node.NetworkKey) if err != nil { return nil, fmt.Errorf("could not create public libp2p node: %w", err) } - return libp2pNode, nil + return publicLibp2pNode, nil }). Component("public network", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - var heroCacheCollector module.HeroCacheMetrics = metrics.NewNoopCollector() - if builder.HeroCacheMetricsEnable { - heroCacheCollector = metrics.NetworkReceiveCacheMetricsFactory(builder.MetricsRegisterer) - } - receiveCache := netcache.NewHeroReceiveCache(builder.NetworkReceivedMessageCacheSize, + receiveCache := netcache.NewHeroReceiveCache(builder.FlowConfig.NetworkConfig.NetworkReceivedMessageCacheSize, builder.Logger, - heroCacheCollector) + metrics.NetworkReceiveCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork)) - err := node.Metrics.Mempool.Register(metrics.ResourceNetworkingReceiveCache, receiveCache.Size) + err := node.Metrics.Mempool.Register(metrics.PrependPublicPrefix(metrics.ResourceNetworkingReceiveCache), receiveCache.Size) if err != nil { return nil, fmt.Errorf("could not register networking receive cache metric: %w", err) } msgValidators := publicNetworkMsgValidators(node.Logger, node.IdentityProvider, node.NodeID) - builder.initMiddleware(node.NodeID, libp2pNode, msgValidators...) + builder.initMiddleware(node.NodeID, publicLibp2pNode, msgValidators...) // topology is nil since it is automatically managed by libp2p net, err := builder.initNetwork(builder.Me, builder.Metrics.Network, builder.Middleware, nil, receiveCache) @@ -1015,11 +861,73 @@ func (builder *ObserverServiceBuilder) enqueueConnectWithStakedAN() { } func (builder *ObserverServiceBuilder) enqueueRPCServer() { + builder.Module("creating grpc servers", func(node *cmd.NodeConfig) error { + builder.secureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + builder.rpcConf.SecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits, + grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)).Build() + + builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + builder.rpcConf.UnsecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits).Build() + + return nil + }) + builder.Module("rest metrics", func(node *cmd.NodeConfig) error { + m, err := metrics.NewRestCollector(routes.URLToRoute, node.MetricsRegisterer) + if err != nil { + return err + } + builder.RestMetrics = m + return nil + }) + builder.Module("access metrics", func(node *cmd.NodeConfig) error { + builder.AccessMetrics = metrics.NewAccessCollector( + metrics.WithRestMetrics(builder.RestMetrics), + ) + return nil + }) builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - engineBuilder, err := rpc.NewBuilder( - node.Logger, + accessMetrics := builder.AccessMetrics + config := builder.rpcConf + backendConfig := config.BackendConfig + + backendCache, cacheSize, err := backend.NewCache(node.Logger, + accessMetrics, + config.BackendConfig.ConnectionPoolSize) + if err != nil { + return nil, fmt.Errorf("could not initialize backend cache: %w", err) + } + + var connBackendCache *rpcConnection.Cache + if backendCache != nil { + connBackendCache = rpcConnection.NewCache(backendCache, int(cacheSize)) + } + + connFactory := &rpcConnection.ConnectionFactoryImpl{ + CollectionGRPCPort: 0, + ExecutionGRPCPort: 0, + CollectionNodeGRPCTimeout: backendConfig.CollectionClientTimeout, + ExecutionNodeGRPCTimeout: backendConfig.ExecutionClientTimeout, + AccessMetrics: accessMetrics, + Log: node.Logger, + Manager: rpcConnection.NewManager( + connBackendCache, + node.Logger, + accessMetrics, + config.MaxMsgSize, + backendConfig.CircuitBreakerConfig, + ), + } + + accessBackend := backend.New( node.State, - builder.rpcConf, nil, nil, node.Storage.Blocks, @@ -1029,28 +937,56 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { node.Storage.Receipts, node.Storage.Results, node.RootChainID, - nil, - nil, - 0, - 0, + accessMetrics, + connFactory, false, + backendConfig.MaxHeightRange, + backendConfig.PreferredExecutionNodeIDs, + backendConfig.FixedExecutionNodeIDs, + node.Logger, + backend.DefaultSnapshotHistoryLimit, + backendConfig.ArchiveAddressList, + backendConfig.CircuitBreakerConfig.Enabled) + + observerCollector := metrics.NewObserverCollector() + restHandler, err := restapiproxy.NewRestProxyHandler( + accessBackend, + builder.upstreamIdentities, + builder.apiTimeout, + config.MaxMsgSize, + builder.Logger, + observerCollector, + node.RootChainID.Chain()) + if err != nil { + return nil, err + } + + engineBuilder, err := rpc.NewBuilder( + node.Logger, + node.State, + config, + node.RootChainID, + accessMetrics, builder.rpcMetricsEnabled, - builder.apiRatelimits, - builder.apiBurstlimits, + builder.Me, + accessBackend, + restHandler, + builder.secureGrpcServer, + builder.unsecureGrpcServer, ) if err != nil { return nil, err } // upstream access node forwarder - forwarder, err := apiproxy.NewFlowAccessAPIForwarder(builder.upstreamIdentities, builder.apiTimeout, builder.rpcConf.MaxMsgSize) + forwarder, err := apiproxy.NewFlowAccessAPIForwarder(builder.upstreamIdentities, builder.apiTimeout, config.MaxMsgSize) if err != nil { return nil, err } - proxy := &apiproxy.FlowAccessAPIRouter{ + rpcHandler := &apiproxy.FlowAccessAPIRouter{ Logger: builder.Logger, - Metrics: metrics.NewObserverCollector(), + Metrics: observerCollector, Upstream: forwarder, Observer: protocol.NewHandler(protocol.New( node.State, @@ -1062,14 +998,25 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { // build the rpc engine builder.RpcEng, err = engineBuilder. - WithNewHandler(proxy). + WithRpcHandler(rpcHandler). WithLegacy(). Build() if err != nil { return nil, err } + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.RpcEng.OnFinalizedBlock) return builder.RpcEng, nil }) + + // build secure grpc server + builder.Component("secure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + return builder.secureGrpcServer, nil + }) + + // build unsecure grpc server + builder.Component("unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + return builder.unsecureGrpcServer, nil + }) } // initMiddleware creates the network.Middleware implementation with the libp2p factory function, metrics, peer update @@ -1078,19 +1025,18 @@ func (builder *ObserverServiceBuilder) initMiddleware(nodeID flow.Identifier, libp2pNode p2p.LibP2PNode, validators ...network.MessageValidator, ) network.Middleware { - slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network) - mw := middleware.NewMiddleware( - builder.Logger, - libp2pNode, nodeID, - builder.Metrics.Bitswap, - builder.SporkID, - middleware.DefaultUnicastTimeout, - builder.IDTranslator, - builder.CodecFactory(), - slashingViolationsConsumer, + mw := middleware.NewMiddleware(&middleware.Config{ + Logger: builder.Logger, + Libp2pNode: libp2pNode, + FlowId: nodeID, + BitSwapMetrics: builder.Metrics.Bitswap, + RootBlockID: builder.SporkID, + UnicastMessageTimeout: middleware.DefaultUnicastTimeout, + IdTranslator: builder.IDTranslator, + Codec: builder.CodecFactory(), + }, middleware.WithMessageValidators(validators...), // use default identifier provider ) - builder.NodeDisallowListDistributor.AddConsumer(mw) builder.Middleware = mw return builder.Middleware } diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 1a7a4438fce..07e2d63011e 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -5,7 +5,6 @@ import ( "crypto/x509" "errors" "fmt" - "math/rand" "os" "runtime" "strings" @@ -25,6 +24,7 @@ import ( "github.com/onflow/flow-go/admin/commands/common" storageCommands "github.com/onflow/flow-go/admin/commands/storage" "github.com/onflow/flow-go/cmd/build" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/consensus/hotstuff/persister" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/fvm/environment" @@ -44,21 +44,21 @@ import ( "github.com/onflow/flow-go/module/updatable_configs" "github.com/onflow/flow-go/module/util" "github.com/onflow/flow-go/network" + alspmgr "github.com/onflow/flow-go/network/alsp/manager" netcache "github.com/onflow/flow-go/network/cache" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/cache" "github.com/onflow/flow-go/network/p2p/conduit" + "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/dns" - "github.com/onflow/flow-go/network/p2p/inspector/validation" "github.com/onflow/flow-go/network/p2p/middleware" "github.com/onflow/flow-go/network/p2p/p2pbuilder" - "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" "github.com/onflow/flow-go/network/p2p/ping" "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/network/p2p/utils/ratelimiter" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/topology" "github.com/onflow/flow-go/state/protocol" badgerState "github.com/onflow/flow-go/state/protocol/badger" @@ -129,6 +129,14 @@ type FlowNodeBuilder struct { var _ NodeBuilder = (*FlowNodeBuilder)(nil) func (fnb *FlowNodeBuilder) BaseFlags() { + defaultFlowConfig, err := config.DefaultConfig() + if err != nil { + fnb.Logger.Fatal().Err(err).Msg("failed to initialize flow config") + } + + // initialize pflag set for Flow node + config.InitializePFlagSet(fnb.flags, defaultFlowConfig) + defaultConfig := DefaultBaseConfig() // bind configuration parameters @@ -139,8 +147,6 @@ func (fnb *FlowNodeBuilder) BaseFlags() { fnb.flags.StringVar(&fnb.BaseConfig.secretsdir, "secretsdir", defaultConfig.secretsdir, "directory to store private database (secrets)") fnb.flags.StringVarP(&fnb.BaseConfig.level, "loglevel", "l", defaultConfig.level, "level for logging output") fnb.flags.Uint32Var(&fnb.BaseConfig.debugLogLimit, "debug-log-limit", defaultConfig.debugLogLimit, "max number of debug/trace log events per second") - fnb.flags.DurationVar(&fnb.BaseConfig.PeerUpdateInterval, "peerupdate-interval", defaultConfig.PeerUpdateInterval, "how often to refresh the peer connections for the node") - fnb.flags.DurationVar(&fnb.BaseConfig.UnicastMessageTimeout, "unicast-timeout", defaultConfig.UnicastMessageTimeout, "how long a unicast transmission can take to complete") fnb.flags.UintVarP(&fnb.BaseConfig.metricsPort, "metricport", "m", defaultConfig.metricsPort, "port for /metrics endpoint") fnb.flags.BoolVar(&fnb.BaseConfig.profilerConfig.Enabled, "profiler-enabled", defaultConfig.profilerConfig.Enabled, "whether to enable the auto-profiler") fnb.flags.BoolVar(&fnb.BaseConfig.profilerConfig.UploaderEnabled, "profile-uploader-enabled", defaultConfig.profilerConfig.UploaderEnabled, @@ -166,22 +172,6 @@ func (fnb *FlowNodeBuilder) BaseFlags() { fnb.flags.StringVar(&fnb.BaseConfig.AdminClientCAs, "admin-client-certs", defaultConfig.AdminClientCAs, "admin client certs (for mutual TLS)") fnb.flags.UintVar(&fnb.BaseConfig.AdminMaxMsgSize, "admin-max-response-size", defaultConfig.AdminMaxMsgSize, "admin server max response size in bytes") - fnb.flags.Float64Var(&fnb.BaseConfig.LibP2PResourceManagerConfig.FileDescriptorsRatio, "libp2p-fd-ratio", defaultConfig.LibP2PResourceManagerConfig.FileDescriptorsRatio, "ratio of available file descriptors to be used by libp2p (in (0,1])") - fnb.flags.Float64Var(&fnb.BaseConfig.LibP2PResourceManagerConfig.MemoryLimitRatio, "libp2p-memory-limit", defaultConfig.LibP2PResourceManagerConfig.MemoryLimitRatio, "ratio of available memory to be used by libp2p (in (0,1])") - fnb.flags.IntVar(&fnb.BaseConfig.LibP2PResourceManagerConfig.PeerBaseLimitConnsInbound, "libp2p-inbound-conns-limit", defaultConfig.LibP2PResourceManagerConfig.PeerBaseLimitConnsInbound, "the maximum amount of allowed inbound connections per peer") - fnb.flags.IntVar(&fnb.BaseConfig.ConnectionManagerConfig.LowWatermark, "libp2p-connmgr-low", defaultConfig.ConnectionManagerConfig.LowWatermark, "low watermarking for libp2p connection manager") - fnb.flags.IntVar(&fnb.BaseConfig.ConnectionManagerConfig.HighWatermark, "libp2p-connmgr-high", defaultConfig.ConnectionManagerConfig.HighWatermark, "high watermarking for libp2p connection manager") - fnb.flags.DurationVar(&fnb.BaseConfig.ConnectionManagerConfig.GracePeriod, "libp2p-connmgr-grace", defaultConfig.ConnectionManagerConfig.GracePeriod, "grace period for libp2p connection manager") - fnb.flags.DurationVar(&fnb.BaseConfig.ConnectionManagerConfig.SilencePeriod, "libp2p-connmgr-silence", defaultConfig.ConnectionManagerConfig.SilencePeriod, "silence period for libp2p connection manager") - - fnb.flags.DurationVar(&fnb.BaseConfig.DNSCacheTTL, "dns-cache-ttl", defaultConfig.DNSCacheTTL, "time-to-live for dns cache") - fnb.flags.StringSliceVar(&fnb.BaseConfig.PreferredUnicastProtocols, "preferred-unicast-protocols", nil, "preferred unicast protocols in ascending order of preference") - fnb.flags.Uint32Var(&fnb.BaseConfig.NetworkReceivedMessageCacheSize, "networking-receive-cache-size", p2p.DefaultReceiveCacheSize, - "incoming message cache size at networking layer") - fnb.flags.BoolVar(&fnb.BaseConfig.NetworkConnectionPruning, "networking-connection-pruning", defaultConfig.NetworkConnectionPruning, "enabling connection trimming") - fnb.flags.BoolVar(&fnb.BaseConfig.GossipSubConfig.PeerScoring, "peer-scoring-enabled", defaultConfig.GossipSubConfig.PeerScoring, "enabling peer scoring on pubsub network") - fnb.flags.DurationVar(&fnb.BaseConfig.GossipSubConfig.LocalMeshLogInterval, "gossipsub-local-mesh-logging-interval", defaultConfig.GossipSubConfig.LocalMeshLogInterval, "logging interval for local mesh in gossipsub") - fnb.flags.DurationVar(&fnb.BaseConfig.GossipSubConfig.ScoreTracerInterval, "gossipsub-score-tracer-interval", defaultConfig.GossipSubConfig.ScoreTracerInterval, "logging interval for peer score tracer in gossipsub, set to 0 to disable") fnb.flags.UintVar(&fnb.BaseConfig.guaranteesCacheSize, "guarantees-cache-size", bstorage.DefaultCacheSize, "collection guarantees cache size") fnb.flags.UintVar(&fnb.BaseConfig.receiptsCacheSize, "receipts-cache-size", bstorage.DefaultCacheSize, "receipts cache size") @@ -203,29 +193,6 @@ func (fnb *FlowNodeBuilder) BaseFlags() { fnb.flags.UintVar(&fnb.BaseConfig.SyncCoreConfig.MaxRequests, "sync-max-requests", defaultConfig.SyncCoreConfig.MaxRequests, "the maximum number of requests we send during each scanning period") fnb.flags.Uint64Var(&fnb.BaseConfig.ComplianceConfig.SkipNewProposalsThreshold, "compliance-skip-proposals-threshold", defaultConfig.ComplianceConfig.SkipNewProposalsThreshold, "threshold at which new proposals are discarded rather than cached, if their height is this much above local finalized height") - - // unicast stream handler rate limits - fnb.flags.IntVar(&fnb.BaseConfig.UnicastRateLimitersConfig.MessageRateLimit, "unicast-message-rate-limit", defaultConfig.UnicastRateLimitersConfig.MessageRateLimit, "maximum number of unicast messages that a peer can send per second") - fnb.flags.IntVar(&fnb.BaseConfig.UnicastRateLimitersConfig.BandwidthRateLimit, "unicast-bandwidth-rate-limit", defaultConfig.UnicastRateLimitersConfig.BandwidthRateLimit, "bandwidth size in bytes a peer is allowed to send via unicast streams per second") - fnb.flags.IntVar(&fnb.BaseConfig.UnicastRateLimitersConfig.BandwidthBurstLimit, "unicast-bandwidth-burst-limit", defaultConfig.UnicastRateLimitersConfig.BandwidthBurstLimit, "bandwidth size in bytes a peer is allowed to send at one time") - fnb.flags.DurationVar(&fnb.BaseConfig.UnicastRateLimitersConfig.LockoutDuration, "unicast-rate-limit-lockout-duration", defaultConfig.UnicastRateLimitersConfig.LockoutDuration, "the number of seconds a peer will be forced to wait before being allowed to successful reconnect to the node after being rate limited") - fnb.flags.BoolVar(&fnb.BaseConfig.UnicastRateLimitersConfig.DryRun, "unicast-rate-limit-dry-run", defaultConfig.UnicastRateLimitersConfig.DryRun, "disable peer disconnects and connections gating when rate limiting peers") - - // gossipsub RPC control message validation limits used for validation configuration and rate limiting - fnb.flags.IntVar(&fnb.BaseConfig.GossipSubRPCInspectorsConfig.ValidationInspectorConfigs.NumberOfWorkers, "gossipsub-rpc-validation-inspector-workers", defaultConfig.GossipSubRPCInspectorsConfig.ValidationInspectorConfigs.NumberOfWorkers, "number of gossupsub RPC control message validation inspector component workers") - fnb.flags.Uint32Var(&fnb.BaseConfig.GossipSubRPCInspectorsConfig.ValidationInspectorConfigs.CacheSize, "gossipsub-rpc-validation-inspector-cache-size", defaultConfig.GossipSubRPCInspectorsConfig.ValidationInspectorConfigs.CacheSize, "cache size for gossipsub RPC validation inspector events worker pool queue.") - fnb.flags.StringToIntVar(&fnb.BaseConfig.GossipSubRPCInspectorsConfig.ValidationInspectorConfigs.GraftLimits, "gossipsub-rpc-graft-limits", defaultConfig.GossipSubRPCInspectorsConfig.ValidationInspectorConfigs.GraftLimits, fmt.Sprintf("discard threshold, safety and rate limits for gossipsub RPC GRAFT message validation e.g: %s=1000,%s=100,%s=1000", validation.DiscardThresholdMapKey, validation.SafetyThresholdMapKey, validation.RateLimitMapKey)) - fnb.flags.StringToIntVar(&fnb.BaseConfig.GossipSubRPCInspectorsConfig.ValidationInspectorConfigs.PruneLimits, "gossipsub-rpc-prune-limits", defaultConfig.GossipSubRPCInspectorsConfig.ValidationInspectorConfigs.PruneLimits, fmt.Sprintf("discard threshold, safety and rate limits for gossipsub RPC PRUNE message validation e.g: %s=1000,%s=20,%s=1000", validation.DiscardThresholdMapKey, validation.SafetyThresholdMapKey, validation.RateLimitMapKey)) - // gossipsub RPC control message metrics observer inspector configuration - fnb.flags.IntVar(&fnb.BaseConfig.GossipSubRPCInspectorsConfig.MetricsInspectorConfigs.NumberOfWorkers, "gossipsub-rpc-metrics-inspector-workers", defaultConfig.GossipSubRPCInspectorsConfig.MetricsInspectorConfigs.NumberOfWorkers, "cache size for gossipsub RPC metrics inspector events worker pool queue.") - fnb.flags.Uint32Var(&fnb.BaseConfig.GossipSubRPCInspectorsConfig.MetricsInspectorConfigs.CacheSize, "gossipsub-rpc-metrics-inspector-cache-size", defaultConfig.GossipSubRPCInspectorsConfig.MetricsInspectorConfigs.CacheSize, "cache size for gossipsub RPC metrics inspector events worker pool.") - - // networking event notifications - fnb.flags.Uint32Var(&fnb.BaseConfig.GossipSubRPCInspectorsConfig.GossipSubRPCInspectorNotificationCacheSize, "gossipsub-rpc-inspector-notification-cache-size", defaultConfig.GossipSubRPCInspectorsConfig.GossipSubRPCInspectorNotificationCacheSize, "cache size for notification events from gossipsub rpc inspector") - fnb.flags.Uint32Var(&fnb.BaseConfig.DisallowListNotificationCacheSize, "disallow-list-notification-cache-size", defaultConfig.DisallowListNotificationCacheSize, "cache size for notification events from disallow list") - - // unicast manager options - fnb.flags.DurationVar(&fnb.BaseConfig.UnicastCreateStreamRetryDelay, "unicast-manager-create-stream-retry-delay", defaultConfig.NetworkConfig.UnicastCreateStreamRetryDelay, "Initial delay between failing to establish a connection with another node and retrying. This delay increases exponentially (exponential backoff) with the number of subsequent failures to establish a connection.") } func (fnb *FlowNodeBuilder) EnqueuePingService() { @@ -235,7 +202,7 @@ func (fnb *FlowNodeBuilder) EnqueuePingService() { // setup the Ping provider to return the software version and the sealed block height pingInfoProvider := &ping.InfoProvider{ SoftwareVersionFun: func() string { - return build.Semver() + return build.Version() }, SealedBlockHeightFun: func() (uint64, error) { head, err := node.State.Sealed().Head() @@ -292,7 +259,7 @@ func (fnb *FlowNodeBuilder) EnqueueResolver() { node.Logger, fnb.Metrics.Network, cache, - dns.WithTTL(fnb.BaseConfig.DNSCacheTTL)) + dns.WithTTL(fnb.BaseConfig.FlowConfig.NetworkConfig.DNSCacheTTL)) fnb.Resolver = resolver return resolver, nil @@ -309,21 +276,21 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { // setup default rate limiter options unicastRateLimiterOpts := []ratelimit.RateLimitersOption{ - ratelimit.WithDisabledRateLimiting(fnb.BaseConfig.UnicastRateLimitersConfig.DryRun), + ratelimit.WithDisabledRateLimiting(fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.DryRun), ratelimit.WithNotifier(fnb.UnicastRateLimiterDistributor), } // override noop unicast message rate limiter - if fnb.BaseConfig.UnicastRateLimitersConfig.MessageRateLimit > 0 { + if fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.MessageRateLimit > 0 { unicastMessageRateLimiter := ratelimiter.NewRateLimiter( - rate.Limit(fnb.BaseConfig.UnicastRateLimitersConfig.MessageRateLimit), - fnb.BaseConfig.UnicastRateLimitersConfig.MessageRateLimit, - fnb.BaseConfig.UnicastRateLimitersConfig.LockoutDuration, + rate.Limit(fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.MessageRateLimit), + fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.MessageRateLimit, + fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.LockoutDuration, ) unicastRateLimiterOpts = append(unicastRateLimiterOpts, ratelimit.WithMessageRateLimiter(unicastMessageRateLimiter)) // avoid connection gating and pruning during dry run - if !fnb.BaseConfig.UnicastRateLimitersConfig.DryRun { + if !fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.DryRun { f := rateLimiterPeerFilter(unicastMessageRateLimiter) // add IsRateLimited peerFilters to conn gater intercept secure peer and peer manager filters list // don't allow rate limited peers to establishing incoming connections @@ -334,16 +301,16 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { } // override noop unicast bandwidth rate limiter - if fnb.BaseConfig.UnicastRateLimitersConfig.BandwidthRateLimit > 0 && fnb.BaseConfig.UnicastRateLimitersConfig.BandwidthBurstLimit > 0 { + if fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.BandwidthRateLimit > 0 && fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.BandwidthBurstLimit > 0 { unicastBandwidthRateLimiter := ratelimit.NewBandWidthRateLimiter( - rate.Limit(fnb.BaseConfig.UnicastRateLimitersConfig.BandwidthRateLimit), - fnb.BaseConfig.UnicastRateLimitersConfig.BandwidthBurstLimit, - fnb.BaseConfig.UnicastRateLimitersConfig.LockoutDuration, + rate.Limit(fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.BandwidthRateLimit), + fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.BandwidthBurstLimit, + fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.LockoutDuration, ) unicastRateLimiterOpts = append(unicastRateLimiterOpts, ratelimit.WithBandwidthRateLimiter(unicastBandwidthRateLimiter)) // avoid connection gating and pruning during dry run - if !fnb.BaseConfig.UnicastRateLimitersConfig.DryRun { + if !fnb.BaseConfig.FlowConfig.NetworkConfig.UnicastRateLimitersConfig.DryRun { f := rateLimiterPeerFilter(unicastBandwidthRateLimiter) // add IsRateLimited peerFilters to conn gater intercept secure peer and peer manager filters list connGaterInterceptSecureFilters = append(connGaterInterceptSecureFilters, f) @@ -354,19 +321,20 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { // setup unicast rate limiters unicastRateLimiters := ratelimit.NewRateLimiters(unicastRateLimiterOpts...) - uniCfg := &p2pbuilder.UnicastConfig{ - StreamRetryInterval: fnb.UnicastCreateStreamRetryDelay, + uniCfg := &p2pconfig.UnicastConfig{ + StreamRetryInterval: fnb.FlowConfig.NetworkConfig.UnicastCreateStreamRetryDelay, RateLimiterDistributor: fnb.UnicastRateLimiterDistributor, } - connGaterCfg := &p2pbuilder.ConnectionGaterConfig{ + connGaterCfg := &p2pconfig.ConnectionGaterConfig{ InterceptPeerDialFilters: connGaterPeerDialFilters, InterceptSecuredFilters: connGaterInterceptSecureFilters, } - peerManagerCfg := &p2pbuilder.PeerManagerConfig{ - ConnectionPruning: fnb.NetworkConnectionPruning, - UpdateInterval: fnb.PeerUpdateInterval, + peerManagerCfg := &p2pconfig.PeerManagerConfig{ + ConnectionPruning: fnb.FlowConfig.NetworkConfig.NetworkConnectionPruning, + UpdateInterval: fnb.FlowConfig.NetworkConfig.PeerUpdateInterval, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), } fnb.Component(LibP2PNodeComponent, func(node *NodeConfig) (module.ReadyDoneAware, error) { @@ -375,56 +343,50 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { myAddr = fnb.BaseConfig.BindAddr } - fnb.GossipSubInspectorNotifDistributor = BuildGossipsubRPCValidationInspectorNotificationDisseminator(fnb.GossipSubRPCInspectorsConfig.GossipSubRPCInspectorNotificationCacheSize, fnb.MetricsRegisterer, fnb.Logger, fnb.MetricsEnabled) - - rpcInspectorBuilder := inspector.NewGossipSubInspectorBuilder(fnb.Logger, fnb.SporkID, fnb.GossipSubRPCInspectorsConfig, fnb.GossipSubInspectorNotifDistributor) - rpcInspectors, err := rpcInspectorBuilder. - SetPublicNetwork(p2p.PublicNetworkDisabled). - SetMetrics(fnb.Metrics.Network, fnb.MetricsRegisterer). - SetMetricsEnabled(fnb.MetricsEnabled).Build() - if err != nil { - return nil, fmt.Errorf("failed to create gossipsub rpc inspectors: %w", err) - } - - // set rpc inspectors on gossipsub config - fnb.GossipSubConfig.RPCInspectors = rpcInspectors - - libP2PNodeFactory := p2pbuilder.DefaultLibP2PNodeFactory( + builder, err := p2pbuilder.DefaultNodeBuilder( fnb.Logger, myAddr, + network.PrivateNetwork, fnb.NetworkKey, fnb.SporkID, fnb.IdentityProvider, - fnb.Metrics.Network, + &p2pconfig.MetricsConfig{ + Metrics: fnb.Metrics.Network, + HeroCacheFactory: fnb.HeroCacheMetricsFactory(), + }, fnb.Resolver, fnb.BaseConfig.NodeRole, connGaterCfg, peerManagerCfg, - // run peer manager with the specified interval and let it also prune connections - fnb.GossipSubConfig, - fnb.LibP2PResourceManagerConfig, + &fnb.FlowConfig.NetworkConfig.GossipSubConfig, + &fnb.FlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, + &fnb.FlowConfig.NetworkConfig.ResourceManagerConfig, uniCfg, - ) + &fnb.FlowConfig.NetworkConfig.ConnectionManagerConfig, + &p2p.DisallowListCacheConfig{ + MaxSize: fnb.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, + Metrics: metrics.DisallowListCacheMetricsFactory(fnb.HeroCacheMetricsFactory(), network.PrivateNetwork), + }) - libp2pNode, err := libP2PNodeFactory() if err != nil { - return nil, fmt.Errorf("failed to create libp2p node: %w", err) + return nil, fmt.Errorf("could not create libp2p node builder: %w", err) } - fnb.LibP2PNode = libp2pNode - return libp2pNode, nil - }) - fnb.Component("gossipsub inspector notification distributor", func(node *NodeConfig) (module.ReadyDoneAware, error) { - // distributor is returned as a component to be started and stopped. - if fnb.GossipSubInspectorNotifDistributor == nil { - return nil, fmt.Errorf("gossipsub inspector notification distributor has not been set") + libp2pNode, err := builder.Build() + if err != nil { + return nil, fmt.Errorf("could not build libp2p node: %w", err) } - return fnb.GossipSubInspectorNotifDistributor, nil + + fnb.LibP2PNode = libp2pNode + return libp2pNode, nil }) fnb.Component(NetworkComponent, func(node *NodeConfig) (module.ReadyDoneAware, error) { - cf := conduit.NewDefaultConduitFactory() fnb.Logger.Info().Hex("node_id", logging.ID(fnb.NodeID)).Msg("default conduit factory initiated") - return fnb.InitFlowNetworkWithConduitFactory(node, cf, unicastRateLimiters, peerManagerFilters) + return fnb.InitFlowNetworkWithConduitFactory( + node, + conduit.NewDefaultConduitFactory(), + unicastRateLimiters, + peerManagerFilters) }) fnb.Module("middleware dependency", func(node *NodeConfig) error { @@ -439,8 +401,23 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { }, fnb.PeerManagerDependencies) } -func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory(node *NodeConfig, cf network.ConduitFactory, unicastRateLimiters *ratelimit.RateLimiters, peerManagerFilters []p2p.PeerFilter) (network.Network, error) { - var mwOpts []middleware.MiddlewareOption +// HeroCacheMetricsFactory returns a HeroCacheMetricsFactory based on the MetricsEnabled flag. +// If MetricsEnabled is true, it returns a HeroCacheMetricsFactory that will register metrics with the provided MetricsRegisterer. +// If MetricsEnabled is false, it returns a no-op HeroCacheMetricsFactory that will not register any metrics. +func (fnb *FlowNodeBuilder) HeroCacheMetricsFactory() metrics.HeroCacheMetricsFactory { + if fnb.MetricsEnabled { + return metrics.NewHeroCacheMetricsFactory(fnb.MetricsRegisterer) + } + return metrics.NewNoopHeroCacheMetricsFactory() +} + +func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory( + node *NodeConfig, + cf network.ConduitFactory, + unicastRateLimiters *ratelimit.RateLimiters, + peerManagerFilters []p2p.PeerFilter) (network.Network, error) { + + var mwOpts []middleware.OptionFn if len(fnb.MsgValidators) > 0 { mwOpts = append(mwOpts, middleware.WithMessageValidators(fnb.MsgValidators...)) } @@ -450,7 +427,7 @@ func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory(node *NodeConfig, mwOpts = append(mwOpts, middleware.WithUnicastRateLimiters(unicastRateLimiters)) mwOpts = append(mwOpts, - middleware.WithPreferredUnicastProtocols(protocols.ToProtocolNames(fnb.PreferredUnicastProtocols)), + middleware.WithPreferredUnicastProtocols(protocols.ToProtocolNames(fnb.FlowConfig.NetworkConfig.PreferredUnicastProtocols)), ) // peerManagerFilters are used by the peerManager via the middleware to filter peers from the topology. @@ -458,30 +435,25 @@ func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory(node *NodeConfig, mwOpts = append(mwOpts, middleware.WithPeerManagerFilters(peerManagerFilters)) } - slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(fnb.Logger, fnb.Metrics.Network) - mw := middleware.NewMiddleware( - fnb.Logger, - fnb.LibP2PNode, - fnb.Me.NodeID(), - fnb.Metrics.Bitswap, - fnb.SporkID, - fnb.BaseConfig.UnicastMessageTimeout, - fnb.IDTranslator, - fnb.CodecFactory(), - slashingViolationsConsumer, + mw := middleware.NewMiddleware(&middleware.Config{ + Logger: fnb.Logger, + Libp2pNode: fnb.LibP2PNode, + FlowId: fnb.Me.NodeID(), + BitSwapMetrics: fnb.Metrics.Bitswap, + RootBlockID: fnb.SporkID, + UnicastMessageTimeout: fnb.FlowConfig.NetworkConfig.UnicastMessageTimeout, + IdTranslator: fnb.IDTranslator, + Codec: fnb.CodecFactory(), + }, mwOpts...) - fnb.NodeDisallowListDistributor.AddConsumer(mw) + fnb.Middleware = mw subscriptionManager := subscription.NewChannelSubscriptionManager(fnb.Middleware) - var heroCacheCollector module.HeroCacheMetrics = metrics.NewNoopCollector() - if fnb.HeroCacheMetricsEnable { - heroCacheCollector = metrics.NetworkReceiveCacheMetricsFactory(fnb.MetricsRegisterer) - } - receiveCache := netcache.NewHeroReceiveCache(fnb.NetworkReceivedMessageCacheSize, + receiveCache := netcache.NewHeroReceiveCache(fnb.FlowConfig.NetworkConfig.NetworkReceivedMessageCacheSize, fnb.Logger, - heroCacheCollector) + metrics.NetworkReceiveCacheMetricsFactory(fnb.HeroCacheMetricsFactory(), network.PrivateNetwork)) err := node.Metrics.Mempool.Register(metrics.ResourceNetworkingReceiveCache, receiveCache.Size) if err != nil { @@ -489,7 +461,7 @@ func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory(node *NodeConfig, } // creates network instance - net, err := p2p.NewNetwork(&p2p.NetworkParameters{ + net, err := p2p.NewNetwork(&p2p.NetworkConfig{ Logger: fnb.Logger, Codec: fnb.CodecFactory(), Me: fnb.Me, @@ -499,7 +471,17 @@ func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory(node *NodeConfig, Metrics: fnb.Metrics.Network, IdentityProvider: fnb.IdentityProvider, ReceiveCache: receiveCache, - Options: []p2p.NetworkOptFunction{p2p.WithConduitFactory(cf)}, + ConduitFactory: cf, + AlspCfg: &alspmgr.MisbehaviorReportManagerConfig{ + Logger: fnb.Logger, + SpamRecordCacheSize: fnb.FlowConfig.NetworkConfig.AlspConfig.SpamRecordCacheSize, + SpamReportQueueSize: fnb.FlowConfig.NetworkConfig.AlspConfig.SpamReportQueueSize, + DisablePenalty: fnb.FlowConfig.NetworkConfig.AlspConfig.DisablePenalty, + HeartBeatInterval: fnb.FlowConfig.NetworkConfig.AlspConfig.HearBeatInterval, + AlspMetrics: fnb.Metrics.Network, + HeroCacheMetricsFactory: fnb.HeroCacheMetricsFactory(), + NetworkType: network.PrivateNetwork, + }, }) if err != nil { return nil, fmt.Errorf("could not initialize network: %w", err) @@ -592,15 +574,28 @@ func (fnb *FlowNodeBuilder) ParseAndPrintFlags() error { // parse configuration parameters pflag.Parse() - // print all flags - log := fnb.Logger.Info() + configOverride, err := config.BindPFlags(&fnb.BaseConfig.FlowConfig, fnb.flags) + if err != nil { + return err + } - pflag.VisitAll(func(flag *pflag.Flag) { - log = log.Str(flag.Name, flag.Value.String()) - }) + if configOverride { + fnb.Logger.Info().Str("config-file", fnb.FlowConfig.ConfigFile).Msg("configuration file updated") + } - log.Msg("flags loaded") + if err = fnb.BaseConfig.FlowConfig.Validate(); err != nil { + fnb.Logger.Fatal().Err(err).Msg("flow configuration validation failed") + } + info := fnb.Logger.Info() + + noPrint := config.LogConfig(info, fnb.flags) + fnb.flags.VisitAll(func(flag *pflag.Flag) { + if _, ok := noPrint[flag.Name]; !ok { + info.Str(flag.Name, fmt.Sprintf("%v", flag.Value)) + } + }) + info.Msg("configuration loaded") return fnb.extraFlagsValidation() } @@ -615,7 +610,7 @@ func (fnb *FlowNodeBuilder) ValidateFlags(f func() error) NodeBuilder { } func (fnb *FlowNodeBuilder) PrintBuildVersionDetails() { - fnb.Logger.Info().Str("version", build.Semver()).Str("commit", build.Commit()).Msg("build details") + fnb.Logger.Info().Str("version", build.Version()).Str("commit", build.Commit()).Msg("build details") } func (fnb *FlowNodeBuilder) initNodeInfo() error { @@ -738,7 +733,7 @@ func (fnb *FlowNodeBuilder) initMetrics() error { if err != nil { return fmt.Errorf("could not query root snapshoot protocol version: %w", err) } - nodeInfoMetrics.NodeInfo(build.Semver(), build.Commit(), nodeConfig.SporkID.String(), protocolVersion) + nodeInfoMetrics.NodeInfo(build.Version(), build.Commit(), nodeConfig.SporkID.String(), protocolVersion) return nil }) } @@ -766,7 +761,7 @@ func (fnb *FlowNodeBuilder) createGCEProfileUploader(client *gcemd.Client, opts ProjectID: projectID, ChainID: chainID, Role: fnb.NodeConfig.NodeRole, - Version: build.Semver(), + Version: build.Version(), Commit: build.Commit(), Instance: instance, } @@ -987,6 +982,7 @@ func (fnb *FlowNodeBuilder) initStorage() error { epochCommits := bstorage.NewEpochCommits(fnb.Metrics.Cache, fnb.DB) statuses := bstorage.NewEpochStatuses(fnb.Metrics.Cache, fnb.DB) commits := bstorage.NewCommits(fnb.Metrics.Cache, fnb.DB) + versionBeacons := bstorage.NewVersionBeacons(fnb.DB) fnb.Storage = Storage{ Headers: headers, @@ -1002,6 +998,7 @@ func (fnb *FlowNodeBuilder) initStorage() error { Collections: collections, Setups: setups, EpochCommits: epochCommits, + VersionBeacons: versionBeacons, Statuses: statuses, Commits: commits, } @@ -1010,13 +1007,6 @@ func (fnb *FlowNodeBuilder) initStorage() error { } func (fnb *FlowNodeBuilder) InitIDProviders() { - fnb.Component("disallow list notification distributor", func(node *NodeConfig) (module.ReadyDoneAware, error) { - // distributor is returned as a component to be started and stopped. - if fnb.NodeDisallowListDistributor == nil { - return nil, fmt.Errorf("disallow list notification distributor has not been set") - } - return fnb.NodeDisallowListDistributor, nil - }) fnb.Module("id providers", func(node *NodeConfig) error { idCache, err := cache.NewProtocolStateIDCache(node.Logger, node.State, node.ProtocolEvents) if err != nil { @@ -1024,11 +1014,11 @@ func (fnb *FlowNodeBuilder) InitIDProviders() { } node.IDTranslator = idCache - fnb.NodeDisallowListDistributor = BuildDisallowListNotificationDisseminator(fnb.DisallowListNotificationCacheSize, fnb.MetricsRegisterer, fnb.Logger, fnb.MetricsEnabled) - // The following wrapper allows to disallow-list byzantine nodes via an admin command: // the wrapper overrides the 'Ejected' flag of disallow-listed nodes to true - disallowListWrapper, err := cache.NewNodeBlocklistWrapper(idCache, node.DB, fnb.NodeDisallowListDistributor) + disallowListWrapper, err := cache.NewNodeDisallowListWrapper(idCache, node.DB, func() network.DisallowListNotificationConsumer { + return fnb.Middleware + }) if err != nil { return fmt.Errorf("could not initialize NodeBlockListWrapper: %w", err) } @@ -1036,9 +1026,9 @@ func (fnb *FlowNodeBuilder) InitIDProviders() { // register the disallow list wrapper for dynamic configuration via admin command err = node.ConfigManager.RegisterIdentifierListConfig("network-id-provider-blocklist", - disallowListWrapper.GetBlocklist, disallowListWrapper.Update) + disallowListWrapper.GetDisallowList, disallowListWrapper.Update) if err != nil { - return fmt.Errorf("failed to register blocklist with config manager: %w", err) + return fmt.Errorf("failed to register disallow-list wrapper with config manager: %w", err) } node.SyncEngineIdentifierProvider = id.NewIdentityFilterIdentifierProvider( @@ -1074,6 +1064,7 @@ func (fnb *FlowNodeBuilder) initState() error { fnb.Storage.Setups, fnb.Storage.EpochCommits, fnb.Storage.Statuses, + fnb.Storage.VersionBeacons, ) if err != nil { return fmt.Errorf("could not open protocol state: %w", err) @@ -1081,7 +1072,7 @@ func (fnb *FlowNodeBuilder) initState() error { fnb.State = state // set root snapshot field - rootBlock, err := state.Params().Root() + rootBlock, err := state.Params().FinalizedRoot() if err != nil { return fmt.Errorf("could not get root block from protocol state: %w", err) } @@ -1125,6 +1116,7 @@ func (fnb *FlowNodeBuilder) initState() error { fnb.Storage.Setups, fnb.Storage.EpochCommits, fnb.Storage.Statuses, + fnb.Storage.VersionBeacons, fnb.RootSnapshot, options..., ) @@ -1135,8 +1127,10 @@ func (fnb *FlowNodeBuilder) initState() error { fnb.Logger.Info(). Hex("root_result_id", logging.Entity(fnb.RootResult)). Hex("root_state_commitment", fnb.RootSeal.FinalState[:]). - Hex("root_block_id", logging.Entity(fnb.RootBlock)). - Uint64("root_block_height", fnb.RootBlock.Header.Height). + Hex("finalized_root_block_id", logging.Entity(fnb.FinalizedRootBlock)). + Uint64("finalized_root_block_height", fnb.FinalizedRootBlock.Header.Height). + Hex("sealed_root_block_id", logging.Entity(fnb.SealedRootBlock)). + Uint64("sealed_root_block_height", fnb.SealedRootBlock.Header.Height). Msg("protocol state bootstrapped") } @@ -1151,12 +1145,22 @@ func (fnb *FlowNodeBuilder) initState() error { if err != nil { return fmt.Errorf("could not get last finalized block header: %w", err) } + fnb.NodeConfig.LastFinalizedHeader = lastFinalized + + lastSealed, err := fnb.State.Sealed().Head() + if err != nil { + return fmt.Errorf("could not get last sealed block header: %w", err) + } fnb.Logger.Info(). - Hex("root_block_id", logging.Entity(fnb.RootBlock)). - Uint64("root_block_height", fnb.RootBlock.Header.Height). - Hex("finalized_block_id", logging.Entity(lastFinalized)). - Uint64("finalized_block_height", lastFinalized.Height). + Hex("last_finalized_block_id", logging.Entity(lastFinalized)). + Uint64("last_finalized_block_height", lastFinalized.Height). + Hex("last_sealed_block_id", logging.Entity(lastSealed)). + Uint64("last_sealed_block_height", lastSealed.Height). + Hex("finalized_root_block_id", logging.Entity(fnb.FinalizedRootBlock)). + Uint64("finalized_root_block_height", fnb.FinalizedRootBlock.Header.Height). + Hex("sealed_root_block_id", logging.Entity(fnb.SealedRootBlock)). + Uint64("sealed_root_block_height", fnb.SealedRootBlock.Header.Height). Msg("successfully opened protocol state") return nil @@ -1192,13 +1196,14 @@ func (fnb *FlowNodeBuilder) setRootSnapshot(rootSnapshot protocol.Snapshot) erro return fmt.Errorf("failed to read root sealing segment: %w", err) } - fnb.RootBlock = sealingSegment.Highest() + fnb.FinalizedRootBlock = sealingSegment.Highest() + fnb.SealedRootBlock = sealingSegment.Sealed() fnb.RootQC, err = fnb.RootSnapshot.QuorumCertificate() if err != nil { return fmt.Errorf("failed to read root QC: %w", err) } - fnb.RootChainID = fnb.RootBlock.Header.ChainID + fnb.RootChainID = fnb.FinalizedRootBlock.Header.ChainID fnb.SporkID, err = fnb.RootSnapshot.Params().SporkID() if err != nil { return fmt.Errorf("failed to read spork ID: %w", err) @@ -1226,7 +1231,7 @@ func (fnb *FlowNodeBuilder) initLocal() error { // We enforce this strictly for MainNet. For other networks (e.g. TestNet or BenchNet), we // are lenient, to allow ghost node to run as any role. if self.Role.String() != fnb.BaseConfig.NodeRole { - rootBlockHeader, err := fnb.State.Params().Root() + rootBlockHeader, err := fnb.State.Params().FinalizedRoot() if err != nil { return fmt.Errorf("could not get root block from protocol state: %w", err) } @@ -1739,6 +1744,8 @@ func (fnb *FlowNodeBuilder) RegisterDefaultAdminCommands() { return common.NewListConfigCommand(config.ConfigManager) }).AdminCommand("read-blocks", func(config *NodeConfig) commands.AdminCommand { return storageCommands.NewReadBlocksCommand(config.State, config.Storage.Blocks) + }).AdminCommand("read-range-blocks", func(conf *NodeConfig) commands.AdminCommand { + return storageCommands.NewReadRangeBlocksCommand(conf.Storage.Blocks) }).AdminCommand("read-results", func(config *NodeConfig) commands.AdminCommand { return storageCommands.NewReadResultsCommand(config.State, config.Storage.Results) }).AdminCommand("read-seals", func(config *NodeConfig) commands.AdminCommand { @@ -1765,10 +1772,6 @@ func (fnb *FlowNodeBuilder) Build() (Node, error) { } func (fnb *FlowNodeBuilder) onStart() error { - - // seed random generator - rand.Seed(time.Now().UnixNano()) - // init nodeinfo by reading the private bootstrap file if not already set if fnb.NodeID == flow.ZeroID { if err := fnb.initNodeInfo(); err != nil { diff --git a/cmd/testclient/go.mod b/cmd/testclient/go.mod index 0a02e69ad42..dbe66a78fb5 100644 --- a/cmd/testclient/go.mod +++ b/cmd/testclient/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/cmd/testclient -go 1.19 +go 1.20 require ( github.com/onflow/flow-go-sdk v0.4.1 diff --git a/cmd/util/cmd/common/state.go b/cmd/util/cmd/common/state.go index 17f448c6a51..16d5295a729 100644 --- a/cmd/util/cmd/common/state.go +++ b/cmd/util/cmd/common/state.go @@ -25,6 +25,7 @@ func InitProtocolState(db *badger.DB, storages *storage.All) (protocol.State, er storages.Setups, storages.EpochCommits, storages.Statuses, + storages.VersionBeacons, ) if err != nil { diff --git a/cmd/util/cmd/epochs/cmd/flags.go b/cmd/util/cmd/epochs/cmd/flags.go index 13d3f712fe5..f818542f99d 100644 --- a/cmd/util/cmd/epochs/cmd/flags.go +++ b/cmd/util/cmd/epochs/cmd/flags.go @@ -3,7 +3,6 @@ package cmd var ( flagBootDir string - flagPayout string flagBucketNetworkName string flagFlowSupplyIncreasePercentage string diff --git a/cmd/util/cmd/epochs/cmd/reset.go b/cmd/util/cmd/epochs/cmd/reset.go index 48a49e32e49..2a1469dab35 100644 --- a/cmd/util/cmd/epochs/cmd/reset.go +++ b/cmd/util/cmd/epochs/cmd/reset.go @@ -7,7 +7,6 @@ import ( "net/http" "os" "path/filepath" - "strings" "github.com/spf13/cobra" @@ -44,7 +43,6 @@ func init() { } func addResetCmdFlags() { - resetCmd.Flags().StringVar(&flagPayout, "payout", "", "the payout eg. 10000.0") resetCmd.Flags().StringVar(&flagBucketNetworkName, "bucket-network-name", "", "when retrieving the root snapshot from a GCP bucket, the network name portion of the URL (eg. \"mainnet-13\")") } @@ -132,7 +130,7 @@ func extractResetEpochArgs(snapshot *inmem.Snapshot) []cadence.Value { log.Fatal().Err(err).Msg("could not get final view from epoch") } - return convertResetEpochArgs(epochCounter, randomSource, flagPayout, firstView, stakingEndView, finalView) + return convertResetEpochArgs(epochCounter, randomSource, firstView, stakingEndView, finalView) } // getStakingAuctionEndView determines the staking auction end view from the @@ -169,7 +167,7 @@ func getStakingAuctionEndView(epoch protocol.Epoch) (uint64, error) { // convertResetEpochArgs converts the arguments required by `resetEpoch` to cadence representations // Contract Method: https://github.com/onflow/flow-core-contracts/blob/master/contracts/epochs/FlowEpoch.cdc#L413 // Transaction: https://github.com/onflow/flow-core-contracts/blob/master/transactions/epoch/admin/reset_epoch.cdc -func convertResetEpochArgs(epochCounter uint64, randomSource []byte, payout string, firstView, stakingEndView, finalView uint64) []cadence.Value { +func convertResetEpochArgs(epochCounter uint64, randomSource []byte, firstView, stakingEndView, finalView uint64) []cadence.Value { args := make([]cadence.Value, 0) @@ -183,23 +181,6 @@ func convertResetEpochArgs(epochCounter uint64, randomSource []byte, payout stri } args = append(args, cdcRandomSource) - // add payout - var cdcPayout cadence.Value - if payout != "" { - index := strings.Index(payout, ".") - if index == -1 { - log.Fatal().Msg("invalid --payout, eg: 10000.0") - } - - cdcPayout, err = cadence.NewUFix64(payout) - if err != nil { - log.Fatal().Err(err).Msg("could not convert payout to cadence type") - } - } else { - cdcPayout = cadence.NewOptional(nil) - } - args = append(args, cdcPayout) - // add first view args = append(args, cadence.NewUInt64(firstView)) diff --git a/cmd/util/cmd/epochs/cmd/reset_test.go b/cmd/util/cmd/epochs/cmd/reset_test.go index 680e9eb9e0f..25983e5cf61 100644 --- a/cmd/util/cmd/epochs/cmd/reset_test.go +++ b/cmd/util/cmd/epochs/cmd/reset_test.go @@ -37,39 +37,6 @@ func TestReset_LocalSnapshot(t *testing.T) { // set initial flag values flagBootDir = bootDir - flagPayout = "" - - // run command with overwritten stdout - stdout := bytes.NewBuffer(nil) - resetCmd.SetOut(stdout) - resetRun(resetCmd, nil) - - // read output from stdout - var outputTxArgs []interface{} - err = json.NewDecoder(stdout).Decode(&outputTxArgs) - require.NoError(t, err) - - // compare to expected values - expectedArgs := extractResetEpochArgs(rootSnapshot) - verifyArguments(t, expectedArgs, outputTxArgs) - }) - }) - - // tests that given the root snapshot file and payout, the command - // writes the expected arguments to stdout. - t.Run("with payout flag set", func(t *testing.T) { - unittest.RunWithTempDir(t, func(bootDir string) { - - // create a root snapshot - rootSnapshot := unittest.RootSnapshotFixture(unittest.IdentityListFixture(10, unittest.WithAllRoles())) - - // write snapshot to correct path in bootDir - err := writeRootSnapshot(bootDir, rootSnapshot) - require.NoError(t, err) - - // set initial flag values - flagBootDir = bootDir - flagPayout = "10.0" // run command with overwritten stdout stdout := bytes.NewBuffer(nil) @@ -97,7 +64,6 @@ func TestReset_LocalSnapshot(t *testing.T) { // set initial flag values flagBootDir = bootDir - flagPayout = "" // run command resetRun(resetCmd, nil) @@ -117,7 +83,6 @@ func TestReset_BucketSnapshot(t *testing.T) { t.Run("happy path", func(t *testing.T) { // set initial flag values flagBucketNetworkName = "mainnet-13" - flagPayout = "" // run command with overwritten stdout stdout := bytes.NewBuffer(nil) @@ -140,7 +105,6 @@ func TestReset_BucketSnapshot(t *testing.T) { t.Run("happy path - with payout", func(t *testing.T) { // set initial flag values flagBucketNetworkName = "mainnet-13" - flagPayout = "10.0" // run command with overwritten stdout stdout := bytes.NewBuffer(nil) @@ -167,7 +131,6 @@ func TestReset_BucketSnapshot(t *testing.T) { // set initial flag values flagBucketNetworkName = "not-a-real-network-name" - flagPayout = "" // run command resetRun(resetCmd, nil) diff --git a/cmd/util/cmd/exec-data-json-export/delta_snapshot_exporter.go b/cmd/util/cmd/exec-data-json-export/delta_snapshot_exporter.go index 6afec2a3945..68fbc9f4070 100644 --- a/cmd/util/cmd/exec-data-json-export/delta_snapshot_exporter.go +++ b/cmd/util/cmd/exec-data-json-export/delta_snapshot_exporter.go @@ -8,7 +8,7 @@ import ( "path/filepath" "github.com/onflow/flow-go/cmd/util/cmd/common" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/storage/badger" @@ -49,7 +49,7 @@ func ExportDeltaSnapshots(blockID flow.Identifier, dbPath string, outputPath str return nil } - var snap []*state.ExecutionSnapshot + var snap []*snapshot.ExecutionSnapshot err = db.View(operation.RetrieveExecutionStateInteractions(activeBlockID, &snap)) if err != nil { return fmt.Errorf("could not load delta snapshot: %w", err) diff --git a/cmd/util/cmd/execution-data-blobstore/cmd/get.go b/cmd/util/cmd/execution-data-blobstore/cmd/get.go index 0a1c7f70e4c..e18e9476d6b 100644 --- a/cmd/util/cmd/execution-data-blobstore/cmd/get.go +++ b/cmd/util/cmd/execution-data-blobstore/cmd/get.go @@ -45,7 +45,7 @@ func run(*cobra.Command, []string) { edID := flow.HashToID(b) - ed, err := eds.GetExecutionData(context.Background(), edID) + ed, err := eds.Get(context.Background(), edID) if err != nil { logger.Fatal().Err(err).Msg("failed to get execution data") } diff --git a/cmd/util/cmd/execution-state-extract/cmd.go b/cmd/util/cmd/execution-state-extract/cmd.go index c8519b015ad..919cca15d28 100644 --- a/cmd/util/cmd/execution-state-extract/cmd.go +++ b/cmd/util/cmd/execution-state-extract/cmd.go @@ -2,7 +2,6 @@ package extract import ( "encoding/hex" - "fmt" "path" "github.com/rs/zerolog/log" @@ -27,16 +26,6 @@ var ( flagNWorker int ) -func getChain(chainName string) (chain flow.Chain, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("invalid chain: %s", r) - } - }() - chain = flow.ChainID(chainName).Chain() - return -} - var Cmd = &cobra.Command{ Use: "execution-state-extract", Short: "Reads WAL files and generates the checkpoint containing state commitment for given block hash", @@ -135,19 +124,23 @@ func run(*cobra.Command, []string) { // log.Fatal().Err(err).Msgf("cannot ensure checkpoint file exist in folder %v", flagExecutionStateDir) // } - chain, err := getChain(flagChain) - if err != nil { - log.Fatal().Err(err).Msgf("invalid chain name") + if len(flagChain) > 0 { + log.Warn().Msgf("--chain flag is deprecated") + } + + if flagNoReport { + log.Warn().Msgf("--no-report flag is deprecated") + } + + if flagNoMigration { + log.Warn().Msgf("--no-migration flag is deprecated") } - err = extractExecutionState( + err := extractExecutionState( flagExecutionStateDir, stateCommitment, flagOutputDir, log.Logger, - chain, - !flagNoMigration, - !flagNoReport, flagNWorker, ) diff --git a/cmd/util/cmd/execution-state-extract/execution_state_extract.go b/cmd/util/cmd/execution-state-extract/execution_state_extract.go index 613e34c2326..dbcb82d53e5 100644 --- a/cmd/util/cmd/execution-state-extract/execution_state_extract.go +++ b/cmd/util/cmd/execution-state-extract/execution_state_extract.go @@ -1,16 +1,20 @@ package extract import ( + "encoding/json" "fmt" "math" + "os" "github.com/rs/zerolog" "go.uber.org/atomic" "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/hash" "github.com/onflow/flow-go/ledger/common/pathfinder" "github.com/onflow/flow-go/ledger/complete" + "github.com/onflow/flow-go/ledger/complete/mtrie/trie" "github.com/onflow/flow-go/ledger/complete/wal" "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/flow" @@ -27,9 +31,6 @@ func extractExecutionState( targetHash flow.StateCommitment, outputDir string, log zerolog.Logger, - chain flow.Chain, - migrate bool, - report bool, nWorker int, // number of concurrent worker to migation payloads ) error { @@ -82,50 +83,33 @@ func extractExecutionState( }() var migrations []ledger.Migration - var preCheckpointReporters, postCheckpointReporters []ledger.Reporter newState := ledger.State(targetHash) - if migrate { - // add migration here - migrations = []ledger.Migration{ - // the following migration calculate the storage usage and update the storage for each account - // mig.MigrateAccountUsage, - } - } - // generating reports at the end, so that the checkpoint file can be used - // for sporking as soon as it's generated. - if report { - log.Info().Msgf("preparing reporter files") - reportFileWriterFactory := reporters.NewReportFileWriterFactory(outputDir, log) - - preCheckpointReporters = []ledger.Reporter{ - // report epoch counter which is needed for finalizing root block - reporters.NewExportReporter(log, - chain, - func() flow.StateCommitment { return targetHash }, - ), - } - - postCheckpointReporters = []ledger.Reporter{ - &reporters.AccountReporter{ - Log: log, - Chain: chain, - RWF: reportFileWriterFactory, - }, - reporters.NewFungibleTokenTracker(log, reportFileWriterFactory, chain, []string{reporters.FlowTokenTypeID(chain)}), - &reporters.AtreeReporter{ - Log: log, - RWF: reportFileWriterFactory, - }, - } - } - - migratedState, err := led.ExportCheckpointAt( + // migrate the trie if there migrations + newTrie, err := led.MigrateAt( newState, migrations, - preCheckpointReporters, - postCheckpointReporters, complete.DefaultPathFinderVersion, + ) + + if err != nil { + return err + } + + // create reporter + reporter := reporters.NewExportReporter(log, + func() flow.StateCommitment { return targetHash }, + ) + + newMigratedState := ledger.State(newTrie.RootHash()) + err = reporter.Report(nil, newMigratedState) + if err != nil { + log.Error().Err(err).Msgf("can not generate report for migrated state: %v", newMigratedState) + } + + migratedState, err := createCheckpoint( + newTrie, + log, outputDir, bootstrap.FilenameWALRootCheckpoint, ) @@ -141,3 +125,42 @@ func extractExecutionState( return nil } + +func createCheckpoint( + newTrie *trie.MTrie, + log zerolog.Logger, + outputDir, + outputFile string, +) (ledger.State, error) { + statecommitment := ledger.State(newTrie.RootHash()) + + log.Info().Msgf("successfully built new trie. NEW ROOT STATECOMMIEMENT: %v", statecommitment.String()) + + err := os.MkdirAll(outputDir, os.ModePerm) + if err != nil { + return ledger.State(hash.DummyHash), fmt.Errorf("could not create output dir %s: %w", outputDir, err) + } + + err = wal.StoreCheckpointV6Concurrently([]*trie.MTrie{newTrie}, outputDir, outputFile, &log) + + // Writing the checkpoint takes time to write and copy. + // Without relying on an exit code or stdout, we need to know when the copy is complete. + writeStatusFileErr := writeStatusFile("checkpoint_status.json", err) + if writeStatusFileErr != nil { + return ledger.State(hash.DummyHash), fmt.Errorf("failed to write checkpoint status file: %w", writeStatusFileErr) + } + + if err != nil { + return ledger.State(hash.DummyHash), fmt.Errorf("failed to store the checkpoint: %w", err) + } + + log.Info().Msgf("checkpoint file successfully stored at: %v %v", outputDir, outputFile) + return statecommitment, nil +} + +func writeStatusFile(fileName string, e error) error { + checkpointStatus := map[string]bool{"succeeded": e == nil} + checkpointStatusJson, _ := json.MarshalIndent(checkpointStatus, "", " ") + err := os.WriteFile(fileName, checkpointStatusJson, 0644) + return err +} diff --git a/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go b/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go index 1f770f12426..018c5474c66 100644 --- a/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go +++ b/cmd/util/cmd/execution-state-extract/execution_state_extract_test.go @@ -64,9 +64,6 @@ func TestExtractExecutionState(t *testing.T) { unittest.StateCommitmentFixture(), outdir, zerolog.Nop(), - flow.Emulator.Chain(), - false, - false, 10, ) require.Error(t, err) diff --git a/cmd/util/cmd/read-execution-state/list-accounts/cmd.go b/cmd/util/cmd/read-execution-state/list-accounts/cmd.go index dbc47a3891f..4a4ba7adbbf 100644 --- a/cmd/util/cmd/read-execution-state/list-accounts/cmd.go +++ b/cmd/util/cmd/read-execution-state/list-accounts/cmd.go @@ -11,9 +11,9 @@ import ( "github.com/spf13/cobra" executionState "github.com/onflow/flow-go/engine/execution/state" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/pathfinder" "github.com/onflow/flow-go/ledger/complete" @@ -75,7 +75,7 @@ func run(*cobra.Command, []string) { log.Fatal().Err(err).Msgf("invalid chain name") } - ldg := delta.NewDeltaView(state.NewReadFuncStorageSnapshot( + ldg := snapshot.NewReadFuncStorageSnapshot( func(id flow.RegisterID) (flow.RegisterValue, error) { ledgerKey := executionState.RegisterIDToKey(id) @@ -99,7 +99,7 @@ func run(*cobra.Command, []string) { } return values[0], nil - })) + }) txnState := state.NewTransactionState(ldg, state.DefaultParameters()) accounts := environment.NewAccounts(txnState) diff --git a/cmd/util/cmd/read-hotstuff/cmd/get_liveness.go b/cmd/util/cmd/read-hotstuff/cmd/get_liveness.go index c6eb12e2c43..e5c68d0dfc6 100644 --- a/cmd/util/cmd/read-hotstuff/cmd/get_liveness.go +++ b/cmd/util/cmd/read-hotstuff/cmd/get_liveness.go @@ -27,7 +27,7 @@ func runGetLivenessData(*cobra.Command, []string) { log.Fatal().Err(err).Msg("could not init protocol state") } - rootBlock, err := state.Params().Root() + rootBlock, err := state.Params().FinalizedRoot() if err != nil { log.Fatal().Err(err).Msgf("could not get root block") } diff --git a/cmd/util/cmd/read-hotstuff/cmd/get_safety.go b/cmd/util/cmd/read-hotstuff/cmd/get_safety.go index bd0281990c7..a9e4e6c0bc6 100644 --- a/cmd/util/cmd/read-hotstuff/cmd/get_safety.go +++ b/cmd/util/cmd/read-hotstuff/cmd/get_safety.go @@ -27,7 +27,7 @@ func runGetSafetyData(*cobra.Command, []string) { log.Fatal().Err(err).Msg("could not init protocol state") } - rootBlock, err := state.Params().Root() + rootBlock, err := state.Params().FinalizedRoot() if err != nil { log.Fatal().Err(err).Msgf("could not get root block") } diff --git a/cmd/util/cmd/read-light-block/read_light_block.go b/cmd/util/cmd/read-light-block/read_light_block.go new file mode 100644 index 00000000000..7e61a3348be --- /dev/null +++ b/cmd/util/cmd/read-light-block/read_light_block.go @@ -0,0 +1,73 @@ +package read + +import ( + "errors" + "fmt" + + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" +) + +type ClusterLightBlock struct { + ID flow.Identifier + Height uint64 + CollectionID flow.Identifier + Transactions []flow.Identifier +} + +func ClusterBlockToLight(clusterBlock *cluster.Block) *ClusterLightBlock { + return &ClusterLightBlock{ + ID: clusterBlock.ID(), + Height: clusterBlock.Header.Height, + CollectionID: clusterBlock.Payload.Collection.ID(), + Transactions: clusterBlock.Payload.Collection.Light().Transactions, + } +} + +func ReadClusterLightBlockByHeightRange(clusterBlocks storage.ClusterBlocks, startHeight uint64, endHeight uint64) ([]*ClusterLightBlock, error) { + blocks := make([]*ClusterLightBlock, 0) + for height := startHeight; height <= endHeight; height++ { + block, err := clusterBlocks.ByHeight(height) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + break + } + return nil, fmt.Errorf("could not get cluster block by height %v: %w", height, err) + } + light := ClusterBlockToLight(block) + blocks = append(blocks, light) + } + return blocks, nil +} + +type LightBlock struct { + ID flow.Identifier + Height uint64 + Collections []flow.Identifier +} + +func BlockToLight(block *flow.Block) *LightBlock { + return &LightBlock{ + ID: block.ID(), + Height: block.Header.Height, + Collections: flow.EntitiesToIDs(block.Payload.Guarantees), + } +} + +func ReadLightBlockByHeightRange(blocks storage.Blocks, startHeight uint64, endHeight uint64) ([]*LightBlock, error) { + bs := make([]*LightBlock, 0) + for height := startHeight; height <= endHeight; height++ { + block, err := blocks.ByHeight(height) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + break + } + + return nil, fmt.Errorf("could not get cluster block by height %v: %w", height, err) + } + light := BlockToLight(block) + bs = append(bs, light) + } + return bs, nil +} diff --git a/cmd/util/cmd/read-light-block/read_light_block_test.go b/cmd/util/cmd/read-light-block/read_light_block_test.go new file mode 100644 index 00000000000..78a84d60823 --- /dev/null +++ b/cmd/util/cmd/read-light-block/read_light_block_test.go @@ -0,0 +1,56 @@ +package read + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + badgerstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestReadClusterRange(t *testing.T) { + + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + chain := unittest.ClusterBlockChainFixture(5) + parent, blocks := chain[0], chain[1:] + + // add parent as boundary + err := db.Update(operation.IndexClusterBlockHeight(parent.Header.ChainID, parent.Header.Height, parent.ID())) + require.NoError(t, err) + + err = db.Update(operation.InsertClusterFinalizedHeight(parent.Header.ChainID, parent.Header.Height)) + require.NoError(t, err) + + // add blocks + for _, block := range blocks { + err := db.Update(procedure.InsertClusterBlock(&block)) + require.NoError(t, err) + + err = db.Update(procedure.FinalizeClusterBlock(block.Header.ID())) + require.NoError(t, err) + } + + clusterBlocks := badgerstorage.NewClusterBlocks( + db, + blocks[0].Header.ChainID, + badgerstorage.NewHeaders(metrics.NewNoopCollector(), db), + badgerstorage.NewClusterPayloads(metrics.NewNoopCollector(), db), + ) + + startHeight := blocks[0].Header.Height + endHeight := startHeight + 10 // if end height is exceeded the last finalized height, only return up to the last finalized + lights, err := ReadClusterLightBlockByHeightRange(clusterBlocks, startHeight, endHeight) + require.NoError(t, err) + + for i, light := range lights { + require.Equal(t, light.ID, blocks[i].ID()) + } + + require.Equal(t, len(blocks), len(lights)) + }) +} diff --git a/cmd/util/cmd/read-protocol-state/cmd/snapshot.go b/cmd/util/cmd/read-protocol-state/cmd/snapshot.go new file mode 100644 index 00000000000..13386195ab3 --- /dev/null +++ b/cmd/util/cmd/read-protocol-state/cmd/snapshot.go @@ -0,0 +1,79 @@ +package cmd + +import ( + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/onflow/flow-go/cmd/util/cmd/common" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/inmem" +) + +var SnapshotCmd = &cobra.Command{ + Use: "snapshot", + Short: "Read snapshot from protocol state", + Run: runSnapshot, +} + +func init() { + rootCmd.AddCommand(SnapshotCmd) + + SnapshotCmd.Flags().Uint64Var(&flagHeight, "height", 0, + "Block height") + + SnapshotCmd.Flags().BoolVar(&flagFinal, "final", false, + "get finalized block") + + SnapshotCmd.Flags().BoolVar(&flagSealed, "sealed", false, + "get sealed block") +} + +func runSnapshot(*cobra.Command, []string) { + db := common.InitStorage(flagDatadir) + defer db.Close() + + storages := common.InitStorages(db) + state, err := common.InitProtocolState(db, storages) + if err != nil { + log.Fatal().Err(err).Msg("could not init protocol state") + } + + var snapshot protocol.Snapshot + + if flagHeight > 0 { + log.Info().Msgf("get snapshot by height: %v", flagHeight) + snapshot = state.AtHeight(flagHeight) + } else if flagFinal { + log.Info().Msgf("get last finalized snapshot") + snapshot = state.Final() + } else if flagSealed { + log.Info().Msgf("get last sealed snapshot") + snapshot = state.Sealed() + } + + head, err := snapshot.Head() + if err != nil { + log.Fatal().Err(err).Msg("fail to get block of snapshot") + } + + log.Info().Msgf("creating snapshot for block height %v, id %v", head.Height, head.ID()) + + serializable, err := inmem.FromSnapshot(snapshot) + if err != nil { + log.Fatal().Err(err).Msg("fail to serialize snapshot") + } + + sealingSegment, err := serializable.SealingSegment() + if err != nil { + log.Fatal().Err(err).Msg("could not get sealing segment") + } + + log.Info().Msgf("snapshot created, sealed height %v, id %v", + sealingSegment.Sealed().Header.Height, sealingSegment.Sealed().Header.ID()) + + log.Info().Msgf("highest finalized height %v, id %v", + sealingSegment.Highest().Header.Height, sealingSegment.Highest().Header.ID()) + + encoded := serializable.Encodable() + common.PrettyPrint(encoded) +} diff --git a/cmd/util/cmd/reindex/cmd/results.go b/cmd/util/cmd/reindex/cmd/results.go index aee5b711d5f..8b62d618755 100644 --- a/cmd/util/cmd/reindex/cmd/results.go +++ b/cmd/util/cmd/reindex/cmd/results.go @@ -26,7 +26,7 @@ var resultsCmd = &cobra.Command{ results := storages.Results blocks := storages.Blocks - root, err := state.Params().Root() + root, err := state.Params().FinalizedRoot() if err != nil { log.Fatal().Err(err).Msg("could not get root header from protocol state") } diff --git a/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height.go b/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height.go index 0ffe2d702fd..83ef43f79de 100644 --- a/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height.go +++ b/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height.go @@ -121,7 +121,7 @@ func removeExecutionResultsFromHeight( fromHeight uint64) error { log.Info().Msgf("removing results for blocks from height: %v", fromHeight) - root, err := protoState.Params().Root() + root, err := protoState.Params().FinalizedRoot() if err != nil { return fmt.Errorf("could not get root: %w", err) } @@ -224,12 +224,6 @@ func removeForBlockID( return fmt.Errorf("could not remove chunk id %v for block id %v: %w", chunkID, blockID, err) } - // remove chunkID-blockID index - err = headers.BatchRemoveChunkBlockIndexByChunkID(chunkID, writeBatch) - - if err != nil { - return fmt.Errorf("could not remove chunk block index for chunk %v block id %v: %w", chunkID, blockID, err) - } } // remove commits diff --git a/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height_test.go b/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height_test.go index 77bdf983cbc..ef2f9ae6284 100644 --- a/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height_test.go +++ b/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height_test.go @@ -7,10 +7,9 @@ import ( "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/execution" "github.com/onflow/flow-go/engine/execution/state" "github.com/onflow/flow-go/engine/execution/state/bootstrap" - "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/trace" bstorage "github.com/onflow/flow-go/storage/badger" @@ -25,7 +24,8 @@ func TestReExecuteBlock(t *testing.T) { // bootstrap to init highest executed height bootstrapper := bootstrap.NewBootstrapper(unittest.Logger()) genesis := unittest.BlockHeaderFixture() - err := bootstrapper.BootstrapExecutionDatabase(db, unittest.StateCommitmentFixture(), genesis) + rootSeal := unittest.Seal.Fixture(unittest.Seal.WithBlock(genesis)) + err := bootstrapper.BootstrapExecutionDatabase(db, rootSeal) require.NoError(t, err) // create all modules @@ -64,37 +64,12 @@ func TestReExecuteBlock(t *testing.T) { ) require.NotNil(t, es) - // prepare data - executableBlock := unittest.ExecutableBlockFixtureWithParent( - nil, - genesis) // make sure the height is higher than genesis - header := executableBlock.Block.Header - executionReceipt := unittest.ExecutionReceiptFixture() - executionReceipt.ExecutionResult.BlockID = header.ID() - cdp := make([]*flow.ChunkDataPack, 0, len(executionReceipt.ExecutionResult.Chunks)) - for _, chunk := range executionReceipt.ExecutionResult.Chunks { - cdp = append(cdp, unittest.ChunkDataPackFixture(chunk.ID())) - } - endState, err := executionReceipt.ExecutionResult.FinalStateCommitment() - require.NoError(t, err) - blockEvents := unittest.BlockEventsFixture(header, 3) - // se := unittest.ServiceEventsFixture(2) - se := unittest.BlockEventsFixture(header, 8) - tes := unittest.TransactionResultsFixture(4) + computationResult := testutil.ComputationResultFixture(t) + header := computationResult.Block.Header err = headers.Store(header) require.NoError(t, err) - computationResult := &execution.ComputationResult{ - ExecutableBlock: executableBlock, - EndState: endState, - ChunkDataPacks: cdp, - Events: []flow.EventsList{blockEvents.Events}, - ServiceEvents: se.Events, - TransactionResults: tes, - ExecutionReceipt: executionReceipt, - } - // save execution results err = es.SaveExecutionResults(context.Background(), computationResult) require.NoError(t, err) @@ -170,7 +145,9 @@ func TestReExecuteBlockWithDifferentResult(t *testing.T) { // bootstrap to init highest executed height bootstrapper := bootstrap.NewBootstrapper(unittest.Logger()) genesis := unittest.BlockHeaderFixture() - err := bootstrapper.BootstrapExecutionDatabase(db, unittest.StateCommitmentFixture(), genesis) + rootSeal := unittest.Seal.Fixture() + unittest.Seal.WithBlock(genesis)(rootSeal) + err := bootstrapper.BootstrapExecutionDatabase(db, rootSeal) require.NoError(t, err) // create all modules @@ -209,36 +186,18 @@ func TestReExecuteBlockWithDifferentResult(t *testing.T) { ) require.NotNil(t, es) - // prepare data executableBlock := unittest.ExecutableBlockFixtureWithParent( nil, - genesis) // make sure the height is higher than genesis + genesis, + &unittest.GenesisStateCommitment) header := executableBlock.Block.Header - executionReceipt := unittest.ExecutionReceiptFixture() - executionReceipt.ExecutionResult.BlockID = header.ID() - cdp := make([]*flow.ChunkDataPack, 0, len(executionReceipt.ExecutionResult.Chunks)) - for _, chunk := range executionReceipt.ExecutionResult.Chunks { - cdp = append(cdp, unittest.ChunkDataPackFixture(chunk.ID())) - } - endState, err := executionReceipt.ExecutionResult.FinalStateCommitment() - require.NoError(t, err) - blockEvents := unittest.BlockEventsFixture(header, 3) - // se := unittest.ServiceEventsFixture(2) - se := unittest.BlockEventsFixture(header, 8) - tes := unittest.TransactionResultsFixture(4) err = headers.Store(header) require.NoError(t, err) - computationResult := &execution.ComputationResult{ - ExecutableBlock: executableBlock, - EndState: endState, - ChunkDataPacks: cdp, - Events: []flow.EventsList{blockEvents.Events}, - ServiceEvents: se.Events, - TransactionResults: tes, - ExecutionReceipt: executionReceipt, - } + computationResult := testutil.ComputationResultFixture(t) + computationResult.ExecutableBlock = executableBlock + computationResult.ExecutionReceipt.ExecutionResult.BlockID = header.ID() // save execution results err = es.SaveExecutionResults(context.Background(), computationResult) @@ -286,24 +245,9 @@ func TestReExecuteBlockWithDifferentResult(t *testing.T) { require.NoError(t, err) require.NoError(t, err2) - executionReceipt2 := unittest.ExecutionReceiptFixture() - executionReceipt2.ExecutionResult.BlockID = header.ID() - cdp2 := make([]*flow.ChunkDataPack, 0, len(executionReceipt2.ExecutionResult.Chunks)) - for _, chunk := range executionReceipt.ExecutionResult.Chunks { - cdp2 = append(cdp2, unittest.ChunkDataPackFixture(chunk.ID())) - } - endState2, err := executionReceipt2.ExecutionResult.FinalStateCommitment() - require.NoError(t, err) - - computationResult2 := &execution.ComputationResult{ - ExecutableBlock: executableBlock, - EndState: endState2, - ChunkDataPacks: cdp2, - Events: []flow.EventsList{blockEvents.Events}, - ServiceEvents: se.Events, - TransactionResults: tes, - ExecutionReceipt: executionReceipt2, - } + computationResult2 := testutil.ComputationResultFixture(t) + computationResult2.ExecutableBlock = executableBlock + computationResult2.ExecutionResult.BlockID = header.ID() // re execute result err = es.SaveExecutionResults(context.Background(), computationResult2) diff --git a/cmd/util/ledger/reporters/account_reporter.go b/cmd/util/ledger/reporters/account_reporter.go index df2ceca91da..aed287c0298 100644 --- a/cmd/util/ledger/reporters/account_reporter.go +++ b/cmd/util/ledger/reporters/account_reporter.go @@ -12,10 +12,10 @@ import ( jsoncdc "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/runtime/common" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" ) @@ -91,7 +91,7 @@ func (r *AccountReporter) Report(payload []ledger.Payload, commit ledger.State) } txnState := state.NewTransactionState( - delta.NewDeltaView(snapshot), + snapshot, state.DefaultParameters()) gen := environment.NewAddressGenerator(txnState, r.Chain) addressCount := gen.AddressCount() @@ -124,7 +124,7 @@ func (r *AccountReporter) Report(payload []ledger.Payload, commit ledger.State) type balanceProcessor struct { vm fvm.VM ctx fvm.Context - storageSnapshot state.StorageSnapshot + storageSnapshot snapshot.StorageSnapshot env environment.Environment balanceScript []byte momentsScript []byte @@ -138,7 +138,7 @@ type balanceProcessor struct { func NewBalanceReporter( chain flow.Chain, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) *balanceProcessor { vm := fvm.NewVirtualMachine() ctx := fvm.NewContext( @@ -163,7 +163,7 @@ func newAccountDataProcessor( rwc ReportWriter, rwm ReportWriter, chain flow.Chain, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) *balanceProcessor { bp := NewBalanceReporter(chain, snapshot) @@ -399,7 +399,7 @@ func (c *balanceProcessor) ReadStored(address flow.Address, domain common.PathDo receiver, err := rt.ReadStored( addr, cadence.Path{ - Domain: domain.Identifier(), + Domain: domain, Identifier: id, }, ) diff --git a/cmd/util/ledger/reporters/export_reporter.go b/cmd/util/ledger/reporters/export_reporter.go index 9c69ebf7218..460c0ebe0dd 100644 --- a/cmd/util/ledger/reporters/export_reporter.go +++ b/cmd/util/ledger/reporters/export_reporter.go @@ -24,18 +24,15 @@ type ExportReport struct { // ExportReporter writes data that can be leveraged outside of extraction type ExportReporter struct { logger zerolog.Logger - chain flow.Chain getBeforeMigrationSCFunc GetStateCommitmentFunc } func NewExportReporter( logger zerolog.Logger, - chain flow.Chain, getBeforeMigrationSCFunc GetStateCommitmentFunc, ) *ExportReporter { return &ExportReporter{ logger: logger, - chain: chain, getBeforeMigrationSCFunc: getBeforeMigrationSCFunc, } } diff --git a/cmd/util/ledger/reporters/fungible_token_tracker.go b/cmd/util/ledger/reporters/fungible_token_tracker.go index d981f041259..0bb1db764bd 100644 --- a/cmd/util/ledger/reporters/fungible_token_tracker.go +++ b/cmd/util/ledger/reporters/fungible_token_tracker.go @@ -14,10 +14,9 @@ import ( "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/flow-go/cmd/util/ledger/migrations" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" ) @@ -142,8 +141,9 @@ func (r *FungibleTokenTracker) worker( wg *sync.WaitGroup) { for j := range jobs { - view := delta.NewDeltaView(NewStorageSnapshotFromPayload(j.payloads)) - txnState := state.NewTransactionState(view, state.DefaultParameters()) + txnState := state.NewTransactionState( + NewStorageSnapshotFromPayload(j.payloads), + state.DefaultParameters()) accounts := environment.NewAccounts(txnState) storage := cadenceRuntime.NewStorage( &migrations.AccountsAtreeLedger{Accounts: accounts}, @@ -165,7 +165,8 @@ func (r *FungibleTokenTracker) worker( itr := storageMap.Iterator(inter) key, value := itr.Next() for value != nil { - r.iterateChildren(append([]string{domain}, key), j.owner, value) + identifier := string(key.(interpreter.StringAtreeValue)) + r.iterateChildren(append([]string{domain}, identifier), j.owner, value) key, value = itr.Next() } } diff --git a/cmd/util/ledger/reporters/fungible_token_tracker_test.go b/cmd/util/ledger/reporters/fungible_token_tracker_test.go index 3149d64d351..60a3988299c 100644 --- a/cmd/util/ledger/reporters/fungible_token_tracker_test.go +++ b/cmd/util/ledger/reporters/fungible_token_tracker_test.go @@ -13,8 +13,8 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/cmd/util/ledger/reporters" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -44,8 +44,9 @@ func TestFungibleTokenTracker(t *testing.T) { // bootstrap ledger payloads := []ledger.Payload{} chain := flow.Testnet.Chain() - view := delta.NewDeltaView( - reporters.NewStorageSnapshotFromPayload(payloads)) + view := state.NewExecutionState( + reporters.NewStorageSnapshotFromPayload(payloads), + state.DefaultParameters()) vm := fvm.NewVirtualMachine() opts := []fvm.Option{ diff --git a/cmd/util/ledger/reporters/storage_snapshot.go b/cmd/util/ledger/reporters/storage_snapshot.go index ade68abc7f6..b9ca42c1fe5 100644 --- a/cmd/util/ledger/reporters/storage_snapshot.go +++ b/cmd/util/ledger/reporters/storage_snapshot.go @@ -1,7 +1,7 @@ package reporters import ( - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" ) @@ -10,8 +10,8 @@ import ( // entries loaded from payloads (should only be used for migration) func NewStorageSnapshotFromPayload( payloads []ledger.Payload, -) state.MapStorageSnapshot { - snapshot := make(state.MapStorageSnapshot, len(payloads)) +) snapshot.MapStorageSnapshot { + snapshot := make(snapshot.MapStorageSnapshot, len(payloads)) for _, entry := range payloads { key, err := entry.Key() if err != nil { diff --git a/cmd/utils.go b/cmd/utils.go index 6e4b02118b8..05763933ebc 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -6,15 +6,10 @@ import ( "path/filepath" "github.com/libp2p/go-libp2p/core/peer" - "github.com/prometheus/client_golang/prometheus" - "github.com/rs/zerolog" "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/mempool/queue" - "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/distributor" "github.com/onflow/flow-go/state/protocol/inmem" "github.com/onflow/flow-go/utils/io" ) @@ -68,23 +63,3 @@ func rateLimiterPeerFilter(rateLimiter p2p.RateLimiter) p2p.PeerFilter { return nil } } - -// BuildDisallowListNotificationDisseminator builds the disallow list notification distributor. -func BuildDisallowListNotificationDisseminator(size uint32, metricsRegistry prometheus.Registerer, logger zerolog.Logger, metricsEnabled bool) p2p.DisallowListNotificationDistributor { - heroStoreOpts := []queue.HeroStoreConfigOption{queue.WithHeroStoreSizeLimit(size)} - if metricsEnabled { - collector := metrics.DisallowListNotificationQueueMetricFactory(metricsRegistry) - heroStoreOpts = append(heroStoreOpts, queue.WithHeroStoreCollector(collector)) - } - return distributor.DefaultDisallowListNotificationDistributor(logger, heroStoreOpts...) -} - -// BuildGossipsubRPCValidationInspectorNotificationDisseminator builds the gossipsub rpc validation inspector notification distributor. -func BuildGossipsubRPCValidationInspectorNotificationDisseminator(size uint32, metricsRegistry prometheus.Registerer, logger zerolog.Logger, metricsEnabled bool) p2p.GossipSubInspectorNotificationDistributor { - heroStoreOpts := []queue.HeroStoreConfigOption{queue.WithHeroStoreSizeLimit(size)} - if metricsEnabled { - collector := metrics.RpcInspectorNotificationQueueMetricFactory(metricsRegistry) - heroStoreOpts = append(heroStoreOpts, queue.WithHeroStoreCollector(collector)) - } - return distributor.DefaultGossipSubInspectorNotificationDistributor(logger, heroStoreOpts...) -} diff --git a/cmd/verification_builder.go b/cmd/verification_builder.go index 52e0438d8b5..948857fb196 100644 --- a/cmd/verification_builder.go +++ b/cmd/verification_builder.go @@ -29,7 +29,6 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/chainsync" "github.com/onflow/flow-go/module/chunks" - modulecompliance "github.com/onflow/flow-go/module/compliance" finalizer "github.com/onflow/flow-go/module/finalizer/consensus" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/mempool/stdmap" @@ -94,15 +93,14 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { processedBlockHeight *badger.ConsumerProgress // used in block consumer chunkQueue *badger.ChunksQueue // used in chunk consumer - syncCore *chainsync.Core // used in follower engine - assignerEngine *assigner.Engine // the assigner engine - fetcherEngine *fetcher.Engine // the fetcher engine - requesterEngine *requester.Engine // the requester engine - verifierEng *verifier.Engine // the verifier engine - chunkConsumer *chunkconsumer.ChunkConsumer - blockConsumer *blockconsumer.BlockConsumer - finalizationDistributor *pubsub.FinalizationDistributor - finalizedHeader *commonsync.FinalizedHeaderCache + syncCore *chainsync.Core // used in follower engine + assignerEngine *assigner.Engine // the assigner engine + fetcherEngine *fetcher.Engine // the fetcher engine + requesterEngine *requester.Engine // the requester engine + verifierEng *verifier.Engine // the verifier engine + chunkConsumer *chunkconsumer.ChunkConsumer + blockConsumer *blockconsumer.BlockConsumer + followerDistributor *pubsub.FollowerDistributor committee *committees.Consensus followerCore *hotstuff.FollowerLoop // follower hotstuff logic @@ -177,9 +175,9 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { return nil }). - Module("finalization distributor", func(node *NodeConfig) error { - finalizationDistributor = pubsub.NewFinalizationDistributor() - finalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) + Module("follower distributor", func(node *NodeConfig) error { + followerDistributor = pubsub.NewFollowerDistributor() + followerDistributor.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(node.Logger)) return nil }). Module("sync core", func(node *NodeConfig) error { @@ -313,15 +311,6 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { return blockConsumer, nil }). - Component("finalized snapshot", func(node *NodeConfig) (module.ReadyDoneAware, error) { - var err error - finalizedHeader, err = commonsync.NewFinalizedHeaderCache(node.Logger, node.State, finalizationDistributor) - if err != nil { - return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) - } - - return finalizedHeader, nil - }). Component("consensus committee", func(node *NodeConfig) (module.ReadyDoneAware, error) { // initialize consensus committee's membership state // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS Committee @@ -332,32 +321,26 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { return committee, err }). Component("follower core", func(node *NodeConfig) (module.ReadyDoneAware, error) { - // create a finalizer that handles updating the protocol // state when the follower detects newly finalized blocks final := finalizer.NewFinalizer(node.DB, node.Storage.Headers, followerState, node.Tracer) - packer := hotsignature.NewConsensusSigDataPacker(committee) - // initialize the verifier for the protocol consensus - verifier := verification.NewCombinedVerifier(committee, packer) - finalized, pending, err := recoveryprotocol.FindLatest(node.State, node.Storage.Headers) if err != nil { return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - finalizationDistributor.AddConsumer(blockConsumer) + followerDistributor.AddOnBlockFinalizedConsumer(blockConsumer.OnFinalizedBlock) // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block followerCore, err = flowconsensus.NewFollower( node.Logger, - committee, + node.Metrics.Mempool, node.Storage.Headers, final, - verifier, - finalizationDistributor, - node.RootBlock.Header, + followerDistributor, + node.FinalizedRootBlock.Header, node.RootQC, finalized, pending, @@ -383,7 +366,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { node.Logger, node.Metrics.Mempool, heroCacheCollector, - finalizationDistributor, + followerDistributor, followerState, followerCore, validator, @@ -400,13 +383,14 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { node.Me, node.Metrics.Engine, node.Storage.Headers, - finalizedHeader.Get(), + node.LastFinalizedHeader, core, - followereng.WithComplianceConfigOpt(modulecompliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold)), + node.ComplianceConfig, ) if err != nil { return nil, fmt.Errorf("could not create follower engine: %w", err) } + followerDistributor.AddOnBlockFinalizedConsumer(followerEng.OnFinalizedBlock) return followerEng, nil }). @@ -416,15 +400,17 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { node.Metrics.Engine, node.Network, node.Me, + node.State, node.Storage.Blocks, followerEng, syncCore, - finalizedHeader, node.SyncEngineIdentifierProvider, ) if err != nil { return nil, fmt.Errorf("could not create synchronization engine: %w", err) } + followerDistributor.AddFinalizationConsumer(sync) + return sync, nil }) } diff --git a/config/README.md b/config/README.md new file mode 100644 index 00000000000..f8a31bda478 --- /dev/null +++ b/config/README.md @@ -0,0 +1,92 @@ +## config +config is a package to hold all configuration values for each Flow component. This package centralizes configuration management providing access +to the entire FlowConfig and utilities to add a new config value, corresponding CLI flag, and validation. + +### Package structure +The root config package contains the FlowConfig struct and the default config file [default-config.yml](https://github.com/onflow/flow-go/blob/master/config/default-config.yml). The `default-config.yml` file is the default configuration that is loaded when the config package is initialize. +The `default-config.yml` is a snapshot of all the configuration values defined for Flow. +Each subpackage contains configuration structs and utilities for components and their related subcomponents. These packages also contain the CLI flags for each configuration value. The [network](https://github.com/onflow/flow-go/tree/master/config/network) package +is a good example of this pattern. The network component is a large component made of many other large components and subcomponents. Each configuration +struct is defined for all of these network related components in the network subpackage and CLI flags. + +### Overriding default values +The entire default config can be overridden using the `--config-file` CLI flag. When set the config package will attempt to parse the specified config file and override all the values +defined. A single default value can be overridden by setting the CLI flag for that specific config. For example `--network-connection-pruning=false` will override the default network connection pruning +config to false. +Override entire config file. +```shell +go build -tags relic -o flow-access-node ./cmd/access +./flow-access-node --config-file=config/config.yml +``` +Override a single configuration value. +```shell +go build -tags relic -o flow-access-node ./cmd/access +./flow-access-node --network-connection-pruning=false +``` +### Adding a new config value +Adding a new config to the FlowConfig can be done in a few easy steps. + +1. Create a new subpackage in the config package for the new configuration structs to live. Although it is encouraged to put all configuration sub-packages in the config package +so that configuration can be updated in one place these sub-packages can live anywhere. This package will define the configuration structs and CLI flags for overriding. + ```shell + mkdir example_config + ``` +2. Add a new CLI flag for the config value. + ```go + const workersCLIFlag = "app-workers" + flags.String(workersCLIFlag, 1, "number of app workers") + ``` + The network package can be used as a good example of how to structure CLI flag initialization. All flags are initialized in a single function [InitializeNetworkFlags](https://github.com/onflow/flow-go/blob/master/config/network/flags.go#L80), this function is then used during flag initialization + of the [config package](https://github.com/onflow/flow-go/blob/master/config/base_flags.go#L22). +3. Add the config as a new field to an existing configuration struct or create a new one. Each configuration struct must be a field on the FlowConfig struct so that it is unmarshalled during configuration initialization. + Each field on a configuration struct must contain the following field tags. + 1. `validate` - validate tag is used to perform validation on field structs using the [validator](https://github.com/go-playground/validator) package. In the example below you will notice + the `validate:"gt=0"` tag, this will ensure that the value of `AppWorkers` is greater than 0. The top level `FlowConfig` struct has a Validate method that performs struct validation. This + validation is done with the validator package, each validate tag on ever struct field and sub struct field will be validated and validation errors are returned. + 2. `mapstructure` - mapstructure tag is used for unmarshalling and must match the CLI flag name defined in step or else the field will not be set when the config is unmarshalled. + ```go + type MyComponentConfig struct { + AppWorkers int `validate:"gt=0" mapstructure:"app-workers"` + } + ``` + It's important to make sure that the CLI flag name matches the mapstructure field tag to avoid parsing errors. +4. Add the new config and a default value to the `default-config.yml` file. Ensure that the new property added matches the configuration struct structure for the subpackage the config belongs to. + ```yaml + config-file: "./default-config.yml" + network-config: + ... + my-component: + app-workers: 1 + ``` +5. Finally, if a new struct was created add it as a new field to the FlowConfig. In the previous steps we added a new config struct and added a new property to the default-config.yml for this struct `my-component`. This property name + must match the mapstructure field tag for the struct when added to the FlowConfig. + ```go + // FlowConfig Flow configuration. + type FlowConfig struct { + ConfigFile string `validate:"filepath" mapstructure:"config-file"` + NetworkConfig *network.Config `mapstructure:"network-config"` + MyComponentConfig *mypackage.MyComponentConfig `mapstructure:"my-component"` + } + ``` + +### Nested structs +In an effort to keep the configuration yaml structure readable some configuration will be in nested properties. When this is the case the mapstructure `squash` tag can be used so that the corresponding nested struct will be +flattened before the configuration is unmarshalled. This is used in the network package which is a collection of configuration structs nested on the network.Config struct. +```go +type Config struct { + // UnicastRateLimitersConfig configuration for all unicast rate limiters. + UnicastRateLimitersConfig `mapstructure:",squash"` + ... +} +``` +`UnicastRateLimitersConfig` is a nested struct that defines configuration for unicast rate limiter component. In our configuration yaml structure you will see that all network configs are defined under the `network-config` property. + +### Setting Aliases +Most configs will not be defined on the top layer FlowConfig but instead be defined on nested structs and in nested properties of the configuration yaml. When the default config is initially loaded the underlying config [viper](https://github.com/spf13/viper) store will store +each configuration with a key that is prefixed with each parent property. For example, because `network-connection-pruning` is found on the `network-config` property of the configuration yaml, the key used by the config store to +store this config value will be prefixed with `network` e.g. +```network.network-connection-pruning``` + +Later in the config process we bind the underlying config store with our pflag set, this allows us to override default values using CLI flags. +At this time the underlying config store would have 2 separate keys `network-connection-pruning` and `network.network-connection-pruning` for the same configuration value. This is because we don't use the network prefix for the CLI flags +used to override network configs. As a result, an alias must be set from `network.network-connection-pruning` -> `network-connection-pruning` so that they both point to the value loaded from the CLI flag. See [SetAliases](https://github.com/onflow/flow-go/blob/master/config/network/config.go#L84) in the network package for a reference. diff --git a/config/base_flags.go b/config/base_flags.go new file mode 100644 index 00000000000..360c4af89b6 --- /dev/null +++ b/config/base_flags.go @@ -0,0 +1,23 @@ +package config + +import ( + "github.com/spf13/pflag" + + "github.com/onflow/flow-go/network/netconf" +) + +const ( + configFileFlagName = "config-file" +) + +// InitializePFlagSet initializes all CLI flags for the Flow node base configuration on the provided pflag set. +// Args: +// +// *pflag.FlagSet: the pflag set of the Flow node. +// *FlowConfig: the config used to set default values on the flags +// +// Note: in subsequent PR's all flag initialization for Flow node should be moved to this func. +func InitializePFlagSet(flags *pflag.FlagSet, config *FlowConfig) { + flags.String(configFileFlagName, "", "provide a path to a Flow configuration file that will be used to set configuration values") + netconf.InitializeNetworkFlags(flags, config.NetworkConfig) +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 00000000000..c6b15190563 --- /dev/null +++ b/config/config.go @@ -0,0 +1,252 @@ +package config + +import ( + "bytes" + _ "embed" + "errors" + "fmt" + "path/filepath" + "regexp" + "strings" + + "github.com/go-playground/validator/v10" + "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/onflow/flow-go/network/netconf" +) + +var ( + conf = viper.New() + validate *validator.Validate + //go:embed default-config.yml + configFile string + + errPflagsNotParsed = errors.New("failed to bind flags to configuration values, pflags must be parsed before binding") +) + +func init() { + initialize() +} + +// FlowConfig Flow configuration. +type FlowConfig struct { + // ConfigFile used to set a path to a config.yml file used to override the default-config.yml file. + ConfigFile string `validate:"filepath" mapstructure:"config-file"` + NetworkConfig *netconf.Config `mapstructure:"network-config"` +} + +// Validate checks validity of the Flow config. Errors indicate that either the configuration is broken, +// incompatible with the node's internal state, or that the node's internal state is corrupted. In all +// cases, continuation is impossible. +func (fc *FlowConfig) Validate() error { + err := validate.Struct(fc) + if err != nil { + if validationErrors, ok := err.(validator.ValidationErrors); ok { + return fmt.Errorf("failed to validate flow configuration: %w", validationErrors) + } + return fmt.Errorf("unexpeceted error encountered while validating flow configuration: %w", err) + } + return nil +} + +// DefaultConfig initializes the flow configuration. All default values for the Flow +// configuration are stored in the default-config.yml file. These values can be overridden +// by node operators by setting the corresponding cli flag. DefaultConfig should be called +// before any pflags are parsed, this will allow the configuration to initialize with defaults +// from default-config.yml. +// Returns: +// +// *FlowConfig: an instance of the network configuration fully initialized to the default values set in the config file +// error: if there is any error encountered while initializing the configuration, all errors are considered irrecoverable. +func DefaultConfig() (*FlowConfig, error) { + var flowConfig FlowConfig + err := Unmarshall(&flowConfig) + if err != nil { + return nil, fmt.Errorf("failed to unmarshall the Flow config: %w", err) + } + return &flowConfig, nil +} + +// BindPFlags binds the configuration to the cli pflag set. This should be called +// after all pflags have been parsed. If the --config-file flag has been set the config will +// be loaded from the specified config file. +// Args: +// +// c: The Flow configuration that will be used to unmarshall the configuration values into after binding pflags. +// This needs to be done because pflags may override a configuration value. +// +// Returns: +// +// error: if there is any error encountered binding pflags or unmarshalling the config struct, all errors are considered irrecoverable. +// bool: true if --config-file flag was set and config file was loaded, false otherwise. +// +// Note: As configuration management is improved, this func should accept the entire Flow config as the arg to unmarshall new config values into. +func BindPFlags(c *FlowConfig, flags *pflag.FlagSet) (bool, error) { + if !flags.Parsed() { + return false, errPflagsNotParsed + } + + // update the config store values from config file if --config-file flag is set + // if config file provided we will use values from the file and skip binding pflags + overridden, err := overrideConfigFile(flags) + if err != nil { + return false, err + } + + if !overridden { + err = conf.BindPFlags(flags) + if err != nil { + return false, fmt.Errorf("failed to bind pflag set: %w", err) + } + setAliases() + } + + err = Unmarshall(c) + if err != nil { + return false, fmt.Errorf("failed to unmarshall the Flow config: %w", err) + } + + return overridden, nil +} + +// Unmarshall unmarshalls the Flow configuration into the provided FlowConfig struct. +// Args: +// +// flowConfig: the flow config struct used for unmarshalling. +// +// Returns: +// +// error: if there is any error encountered unmarshalling the configuration, all errors are considered irrecoverable. +func Unmarshall(flowConfig *FlowConfig) error { + err := conf.Unmarshal(flowConfig, func(decoderConfig *mapstructure.DecoderConfig) { + // enforce all fields are set on the FlowConfig struct + decoderConfig.ErrorUnset = true + // currently the entire flow configuration has not been moved to this package + // for now we allow key's in the config which are unused. + decoderConfig.ErrorUnused = false + }) + if err != nil { + return fmt.Errorf("failed to unmarshal network config: %w", err) + } + return nil +} + +// LogConfig logs configuration keys and values if they were overridden with a config file. +// It also returns a map of keys for which the values were set by a config file. +// +// Parameters: +// - logger: *zerolog.Event to which the configuration keys and values will be logged. +// - flags: *pflag.FlagSet containing the set flags. +// +// Returns: +// - map[string]struct{}: map of keys for which the values were set by a config file. +func LogConfig(logger *zerolog.Event, flags *pflag.FlagSet) map[string]struct{} { + keysToAvoid := make(map[string]struct{}) + + if flags.Lookup(configFileFlagName).Changed { + for _, key := range conf.AllKeys() { + logger.Str(key, fmt.Sprint(conf.Get(key))) + parts := strings.Split(key, ".") + if len(parts) == 2 { + keysToAvoid[parts[1]] = struct{}{} + } else { + keysToAvoid[key] = struct{}{} + } + } + } + + return keysToAvoid +} + +// setAliases sets aliases for config sub packages. This should be done directly after pflags are bound to the configuration store. +// Upon initialization the conf will be loaded with the default config values, those values are then used as the default values for +// all the CLI flags, the CLI flags are then bound to the configuration store and at this point all aliases should be set if configuration +// keys do not match the CLI flags 1:1. ie: networking-connection-pruning -> network-config.networking-connection-pruning. After aliases +// are set the conf store will override values with any CLI flag values that are set as expected. +func setAliases() { + err := netconf.SetAliases(conf) + if err != nil { + panic(fmt.Errorf("failed to set network aliases: %w", err)) + } +} + +// overrideConfigFile overrides the default config file by reading in the config file at the path set +// by the --config-file flag in our viper config store. +// +// Returns: +// +// error: if there is any error encountered while reading new config file, all errors are considered irrecoverable. +// bool: true if the config was overridden by the new config file, false otherwise or if an error is encountered reading the new config file. +func overrideConfigFile(flags *pflag.FlagSet) (bool, error) { + configFileFlag := flags.Lookup(configFileFlagName) + if configFileFlag.Changed { + p := configFileFlag.Value.String() + dirPath, fileName := splitConfigPath(p) + conf.AddConfigPath(dirPath) + conf.SetConfigName(fileName) + err := conf.ReadInConfig() + if err != nil { + return false, fmt.Errorf("failed to read config file %s: %w", p, err) + } + if len(conf.AllKeys()) == 0 { + return false, fmt.Errorf("failed to read in config file no config values found") + } + return true, nil + } + return false, nil +} + +// splitConfigPath returns the directory and base name (without extension) of the config file from the provided path string. +// If the file name does not match the expected pattern, the function panics. +// +// The expected pattern for file names is that they must consist of alphanumeric characters, hyphens, or underscores, +// followed by a single dot and then the extension. +// +// Legitimate Inputs: +// - /path/to/my_config.yaml +// - /path/to/my-config123.yaml +// - my-config.yaml (when in the current directory) +// +// Illegitimate Inputs: +// - /path/to/my.config.yaml (contains multiple dots) +// - /path/to/my config.yaml (contains spaces) +// - /path/to/.config.yaml (does not have a file name before the dot) +// +// Args: +// - path: The file path string to be split into directory and base name. +// +// Returns: +// - The directory and base name without extension. +// +// Panics: +// - If the file name does not match the expected pattern. +func splitConfigPath(path string) (string, string) { + // Regex to match filenames like 'my_config.yaml' or 'my-config.yaml' but not 'my.config.yaml' + validFileNamePattern := regexp.MustCompile(`^[a-zA-Z0-9_-]+\.[a-zA-Z0-9]+$`) + + dir, name := filepath.Split(path) + + // Panic if the file name does not match the expected pattern + if !validFileNamePattern.MatchString(name) { + panic(fmt.Errorf("Invalid config file name '%s'. Expected pattern: alphanumeric, hyphens, or underscores followed by a single dot and extension", name)) + } + + // Extracting the base name without extension + baseName := strings.Split(name, ".")[0] + return dir, baseName +} + +func initialize() { + buf := bytes.NewBufferString(configFile) + conf.SetConfigType("yaml") + if err := conf.ReadConfig(buf); err != nil { + panic(fmt.Errorf("failed to initialize flow config failed to read in config file: %w", err)) + } + + // create validator, at this point you can register custom validation funcs + // struct tag translation etc. + validate = validator.New() +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 00000000000..034a0ed6efd --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,141 @@ +package config + +import ( + "errors" + "fmt" + "os" + "strings" + "testing" + + "github.com/go-playground/validator/v10" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/utils/unittest" +) + +// TestBindPFlags ensures configuration is bound to the pflag set as expected and configuration values are overridden when set with CLI flags. +func TestBindPFlags(t *testing.T) { + t.Run("should override config values when any flag is set", func(t *testing.T) { + c := defaultConfig(t) + flags := testFlagSet(c) + err := flags.Set("networking-connection-pruning", "false") + require.NoError(t, err) + require.NoError(t, flags.Parse(nil)) + + configFileUsed, err := BindPFlags(c, flags) + require.NoError(t, err) + require.False(t, configFileUsed) + require.False(t, c.NetworkConfig.NetworkConnectionPruning) + }) + t.Run("should return an error if flags are not parsed", func(t *testing.T) { + c := defaultConfig(t) + flags := testFlagSet(c) + configFileUsed, err := BindPFlags(&FlowConfig{}, flags) + require.False(t, configFileUsed) + require.Error(t, err) + require.True(t, errors.Is(err, errPflagsNotParsed)) + }) +} + +// TestDefaultConfig ensures the default Flow config is created and returned without errors. +func TestDefaultConfig(t *testing.T) { + c := defaultConfig(t) + require.Equalf(t, "./default-config.yml", c.ConfigFile, "expected default config file to be used") + require.NoErrorf(t, c.Validate(), "unexpected error encountered validating default config") + unittest.IdentifierFixture() +} + +// TestFlowConfig_Validate ensures the Flow validate returns the expected number of validator.ValidationErrors when incorrect +// fields are set. +func TestFlowConfig_Validate(t *testing.T) { + c := defaultConfig(t) + // set invalid config values + c.NetworkConfig.UnicastRateLimitersConfig.MessageRateLimit = -100 + c.NetworkConfig.UnicastRateLimitersConfig.BandwidthRateLimit = -100 + err := c.Validate() + require.Error(t, err) + errs, ok := errors.Unwrap(err).(validator.ValidationErrors) + require.True(t, ok) + require.Len(t, errs, 2) +} + +// TestUnmarshall_UnsetFields ensures that if the config store has any missing config values an error is returned when the config is decoded into a Flow config. +func TestUnmarshall_UnsetFields(t *testing.T) { + conf = viper.New() + c := &FlowConfig{} + err := Unmarshall(c) + require.True(t, strings.Contains(err.Error(), "has unset fields")) +} + +// Test_overrideConfigFile ensures configuration values can be overridden via the --config-file flag. +func Test_overrideConfigFile(t *testing.T) { + t.Run("should override the default config if --config-file is set", func(t *testing.T) { + file, err := os.CreateTemp("", "config-*.yml") + require.NoError(t, err) + defer os.Remove(file.Name()) + + var data = fmt.Sprintf(`config-file: "%s" +network-config: + networking-connection-pruning: false +`, file.Name()) + _, err = file.Write([]byte(data)) + require.NoError(t, err) + c := defaultConfig(t) + flags := testFlagSet(c) + err = flags.Set(configFileFlagName, file.Name()) + + require.NoError(t, err) + overridden, err := overrideConfigFile(flags) + require.NoError(t, err) + require.True(t, overridden) + + // ensure config values overridden with values from our inline config + require.Equal(t, conf.GetString(configFileFlagName), file.Name()) + require.False(t, conf.GetBool("networking-connection-pruning")) + }) + t.Run("should return an error for missing --config file", func(t *testing.T) { + c := defaultConfig(t) + flags := testFlagSet(c) + err := flags.Set(configFileFlagName, "./missing-config.yml") + require.NoError(t, err) + overridden, err := overrideConfigFile(flags) + require.Error(t, err) + require.False(t, overridden) + }) + t.Run("should not attempt to override config if --config-file is not set", func(t *testing.T) { + c := defaultConfig(t) + flags := testFlagSet(c) + overridden, err := overrideConfigFile(flags) + require.NoError(t, err) + require.False(t, overridden) + }) + t.Run("should return an error for file types other than .yml", func(t *testing.T) { + file, err := os.CreateTemp("", "config-*.json") + require.NoError(t, err) + defer os.Remove(file.Name()) + c := defaultConfig(t) + flags := testFlagSet(c) + err = flags.Set(configFileFlagName, file.Name()) + require.NoError(t, err) + overridden, err := overrideConfigFile(flags) + require.Error(t, err) + require.False(t, overridden) + }) +} + +// defaultConfig resets the config store gets the default Flow config. +func defaultConfig(t *testing.T) *FlowConfig { + initialize() + c, err := DefaultConfig() + require.NoError(t, err) + return c +} + +func testFlagSet(c *FlowConfig) *pflag.FlagSet { + flags := pflag.NewFlagSet("test", pflag.PanicOnError) + // initialize default flags + InitializePFlagSet(flags, c) + return flags +} diff --git a/config/default-config.yml b/config/default-config.yml new file mode 100644 index 00000000000..c14f7e13c0b --- /dev/null +++ b/config/default-config.yml @@ -0,0 +1,118 @@ +config-file: "./default-config.yml" +network-config: + # Network Configuration + # Connection pruning determines whether connections to nodes + # that are not part of protocol state should be trimmed + networking-connection-pruning: true + # Preferred unicasts protocols list of unicast protocols in preferred order + preferred-unicast-protocols: [ ] + received-message-cache-size: 10e4 + peerupdate-interval: 10m + unicast-message-timeout: 5s + # Unicast create stream retry delay is initial delay used in the exponential backoff for create stream retries + unicast-create-stream-retry-delay: 1s + dns-cache-ttl: 5m + # The size of the queue for notifications about new peers in the disallow list. + disallow-list-notification-cache-size: 100 + # unicast rate limiters config + # Setting this to true will disable connection disconnects and gating when unicast rate limiters are configured + unicast-dry-run: true + # The number of seconds a peer will be forced to wait before being allowed to successfully reconnect to the node after being rate limited + unicast-lockout-duration: 10s + # Amount of unicast messages that can be sent by a peer per second + unicast-message-rate-limit: 0 + # Bandwidth size in bytes a peer is allowed to send via unicast streams per second + unicast-bandwidth-rate-limit: 0 + # Bandwidth size in bytes a peer is allowed to send via unicast streams at once + unicast-bandwidth-burst-limit: 1e9 + # Resource manager config + # Maximum allowed fraction of file descriptors to be allocated by the libp2p resources in (0,1] + libp2p-memory-limit-ratio: 0.5 # flow default + # Maximum allowed fraction of memory to be allocated by the libp2p resources in (0,1] + libp2p-file-descriptors-ratio: 0.2 # libp2p default + # The default value for libp2p PeerBaseLimitConnsInbound. This limit + # restricts the amount of inbound connections from a peer to 1, forcing libp2p to reuse the connection. + # Without this limit peers can end up in a state where there exists n number of connections per peer which + # can lead to resource exhaustion of the libp2p node. + libp2p-peer-base-limits-conns-inbound: 1 + # Connection manager config + # HighWatermark and LowWatermark govern the number of connections are maintained by the ConnManager. + # When the peer count exceeds the HighWatermark, as many peers will be pruned (and + # their connections terminated) until LowWatermark peers remain. In other words, whenever the + # peer count is x > HighWatermark, the ConnManager will prune x - LowWatermark peers. + # The pruning algorithm is as follows: + # 1. The ConnManager will not prune any peers that have been connected for less than GracePeriod. + # 2. The ConnManager will not prune any peers that are protected. + # 3. The ConnManager will sort the peers based on their number of streams and direction of connections, and + # prunes the peers with the least number of streams. If there are ties, the peer with the incoming connection + # will be pruned. If both peers have incoming connections, and there are still ties, one of the peers will be + # pruned at random. + # Algorithm implementation is in https://github.com/libp2p/go-libp2p/blob/master/p2p/net/connmgr/connmgr.go#L262-L318 + libp2p-high-watermark: 500 + libp2p-low-watermark: 450 + # The time to wait before pruning a new connection + libp2p-silence-period: 10s + # The time to wait before start pruning connections + libp2p-grace-period: 1m + # Gossipsub config + # The default interval at which the mesh tracer logs the mesh topology. This is used for debugging and forensics purposes. + # Note that we purposefully choose this logging interval high enough to avoid spamming the logs. Moreover, the + # mesh updates will be logged individually and separately. The logging interval is only used to log the mesh + # topology as a whole specially when there are no updates to the mesh topology for a long time. + gossipsub-local-mesh-logging-interval: 1m + # The default interval at which the gossipsub score tracer logs the peer scores. This is used for debugging and forensics purposes. + # Note that we purposefully choose this logging interval high enough to avoid spamming the logs. + gossipsub-score-tracer-interval: 1m + # The default RPC sent tracker cache size. The RPC sent tracker is used to track RPC control messages sent from the local node. + # Note: this cache size must be large enough to keep a history of sent messages in a reasonable time window of past history. + gossipsub-rpc-sent-tracker-cache-size: 1_000_000 + # Cache size of the rpc sent tracker queue used for async tracking. + gossipsub-rpc-sent-tracker-queue-cache-size: 100_000 + # Number of workers for rpc sent tracker worker pool. + gossipsub-rpc-sent-tracker-workers: 5 + # Peer scoring is the default value for enabling peer scoring + gossipsub-peer-scoring-enabled: true + # Gossipsub rpc inspectors configs + # The size of the queue for notifications about invalid RPC messages + gossipsub-rpc-inspector-notification-cache-size: 10000 + # RPC control message validation inspector configs + # Rpc validation inspector number of pool workers + gossipsub-rpc-validation-inspector-workers: 5 + # The size of the queue used by worker pool for the control message validation inspector + gossipsub-rpc-validation-inspector-queue-cache-size: 100 + # Cluster prefixed control message validation configs + # The size of the cache used to track the amount of cluster prefixed topics received by peers + gossipsub-cluster-prefix-tracker-cache-size: 100 + # The decay val used for the geometric decay of cache counters used to keep track of cluster prefixed topics received by peers + gossipsub-cluster-prefix-tracker-cache-decay: 0.99 + # The upper bound on the amount of cluster prefixed control messages that will be processed + gossipsub-rpc-cluster-prefixed-hard-threshold: 100 + # GRAFT libp2p control message validation limits + gossipsub-rpc-graft-hard-threshold: 30 + gossipsub-rpc-graft-safety-threshold: 15 + gossipsub-rpc-graft-rate-limit: 30 + # PRUNE libp2p control message validation limits + gossipsub-rpc-prune-hard-threshold: 30 + gossipsub-rpc-prune-safety-threshold: 15 + gossipsub-rpc-prune-rate-limit: 30 + # IHAVE libp2p control message validation limits + gossipsub-rpc-ihave-hard-threshold: 100 + gossipsub-rpc-ihave-safety-threshold: 50 + # Rate limiting is disabled for ihave control messages + gossipsub-rpc-ihave-rate-limit: 0 + # Percentage of ihaves to use as the sample size for synchronous inspection 25% + ihave-sync-inspection-sample-size-percentage: .25 + # Percentage of ihaves to use as the sample size for asynchronous inspection 10% + ihave-async-inspection-sample-size-percentage: .10 + # Max number of ihave messages in a sample to be inspected + ihave-max-sample-size: 100 + # RPC metrics observer inspector configs + # The number of metrics inspector pool workers + gossipsub-rpc-metrics-inspector-workers: 1 + # The size of the queue used by worker pool for the control message metrics inspector + gossipsub-rpc-metrics-inspector-cache-size: 100 + # Application layer spam prevention + alsp-spam-record-cache-size: 10e3 + alsp-spam-report-queue-size: 10e4 + alsp-disable-penalty: false + alsp-heart-beat-interval: 1s diff --git a/consensus/aggregators.go b/consensus/aggregators.go index 10bf86083c8..853bec81798 100644 --- a/consensus/aggregators.go +++ b/consensus/aggregators.go @@ -23,9 +23,9 @@ func NewVoteAggregator( engineMetrics module.EngineMetrics, mempoolMetrics module.MempoolMetrics, lowestRetainedView uint64, - notifier hotstuff.Consumer, + notifier hotstuff.VoteAggregationConsumer, voteProcessorFactory hotstuff.VoteProcessorFactory, - distributor *pubsub.FinalizationDistributor, + distributor *pubsub.FollowerDistributor, ) (hotstuff.VoteAggregator, error) { createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) @@ -57,12 +57,12 @@ func NewTimeoutAggregator(log zerolog.Logger, mempoolMetrics module.MempoolMetrics, notifier *pubsub.Distributor, timeoutProcessorFactory hotstuff.TimeoutProcessorFactory, - distributor *pubsub.TimeoutCollectorDistributor, + distributor *pubsub.TimeoutAggregationDistributor, lowestRetainedView uint64, ) (hotstuff.TimeoutAggregator, error) { - timeoutCollectorFactory := timeoutcollector.NewTimeoutCollectorFactory(log, notifier, distributor, timeoutProcessorFactory) - collectors := timeoutaggregator.NewTimeoutCollectors(log, lowestRetainedView, timeoutCollectorFactory) + timeoutCollectorFactory := timeoutcollector.NewTimeoutCollectorFactory(log, distributor, timeoutProcessorFactory) + collectors := timeoutaggregator.NewTimeoutCollectors(log, hotstuffMetrics, lowestRetainedView, timeoutCollectorFactory) // initialize the timeout aggregator aggregator, err := timeoutaggregator.NewTimeoutAggregator( @@ -70,7 +70,6 @@ func NewTimeoutAggregator(log zerolog.Logger, hotstuffMetrics, engineMetrics, mempoolMetrics, - notifier, lowestRetainedView, collectors, ) diff --git a/consensus/config.go b/consensus/config.go index 8862ffd366e..bb4c40d930b 100644 --- a/consensus/config.go +++ b/consensus/config.go @@ -5,35 +5,33 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" + "github.com/onflow/flow-go/consensus/hotstuff/pacemaker" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" - "github.com/onflow/flow-go/module/updatable_configs" ) // HotstuffModules is a helper structure to encapsulate dependencies to create // a hotStuff participant. type HotstuffModules struct { - Committee hotstuff.DynamicCommittee // consensus committee - Signer hotstuff.Signer // signer of proposal & votes - Persist hotstuff.Persister // last state of consensus participant - Notifier *pubsub.Distributor // observer for hotstuff events - FinalizationDistributor *pubsub.FinalizationDistributor // observer for finalization events, used by compliance engine - QCCreatedDistributor *pubsub.QCCreatedDistributor // observer for qc created event, used by leader - TimeoutCollectorDistributor *pubsub.TimeoutCollectorDistributor - Forks hotstuff.Forks // information about multiple forks - Validator hotstuff.Validator // validator of proposals & votes - VoteAggregator hotstuff.VoteAggregator // aggregator of votes, used by leader - TimeoutAggregator hotstuff.TimeoutAggregator // aggregator of `TimeoutObject`s, used by every replica + Committee hotstuff.DynamicCommittee // consensus committee + Signer hotstuff.Signer // signer of proposal & votes + Persist hotstuff.Persister // last state of consensus participant + Notifier *pubsub.Distributor // observer for hotstuff events + VoteCollectorDistributor *pubsub.VoteCollectorDistributor // observer for vote aggregation events, used by leader + TimeoutCollectorDistributor *pubsub.TimeoutCollectorDistributor // observer for timeout aggregation events + Forks hotstuff.Forks // information about multiple forks + Validator hotstuff.Validator // validator of proposals & votes + VoteAggregator hotstuff.VoteAggregator // aggregator of votes, used by leader + TimeoutAggregator hotstuff.TimeoutAggregator // aggregator of `TimeoutObject`s, used by every replica } type ParticipantConfig struct { - StartupTime time.Time // the time when consensus participant enters first view - TimeoutMinimum time.Duration // the minimum timeout for the pacemaker - TimeoutMaximum time.Duration // the maximum timeout for the pacemaker - TimeoutAdjustmentFactor float64 // the factor at which the timeout duration is adjusted - HappyPathMaxRoundFailures uint64 // number of failed rounds before first timeout increase - BlockRateDelay time.Duration // a delay to broadcast block proposal in order to control the block production rate - MaxTimeoutObjectRebroadcastInterval time.Duration // maximum interval for timeout object rebroadcast - Registrar updatable_configs.Registrar // optional: for registering HotStuff configs as dynamically configurable + StartupTime time.Time // the time when consensus participant enters first view + TimeoutMinimum time.Duration // the minimum timeout for the pacemaker + TimeoutMaximum time.Duration // the maximum timeout for the pacemaker + TimeoutAdjustmentFactor float64 // the factor at which the timeout duration is adjusted + HappyPathMaxRoundFailures uint64 // number of failed rounds before first timeout increase + MaxTimeoutObjectRebroadcastInterval time.Duration // maximum interval for timeout object rebroadcast + ProposalDurationProvider hotstuff.ProposalDurationProvider // a delay to broadcast block proposal in order to control the block production rate } func DefaultParticipantConfig() ParticipantConfig { @@ -43,9 +41,8 @@ func DefaultParticipantConfig() ParticipantConfig { TimeoutMaximum: time.Duration(defTimeout.MaxReplicaTimeout) * time.Millisecond, TimeoutAdjustmentFactor: defTimeout.TimeoutAdjustmentFactor, HappyPathMaxRoundFailures: defTimeout.HappyPathMaxRoundFailures, - BlockRateDelay: defTimeout.GetBlockRateDelay(), MaxTimeoutObjectRebroadcastInterval: time.Duration(defTimeout.MaxTimeoutObjectRebroadcastInterval) * time.Millisecond, - Registrar: nil, + ProposalDurationProvider: pacemaker.NoProposalDelay(), } return cfg } @@ -76,14 +73,14 @@ func WithHappyPathMaxRoundFailures(happyPathMaxRoundFailures uint64) Option { } } -func WithBlockRateDelay(delay time.Duration) Option { +func WithProposalDurationProvider(provider hotstuff.ProposalDurationProvider) Option { return func(cfg *ParticipantConfig) { - cfg.BlockRateDelay = delay + cfg.ProposalDurationProvider = provider } } -func WithConfigRegistrar(reg updatable_configs.Registrar) Option { +func WithStaticProposalDuration(dur time.Duration) Option { return func(cfg *ParticipantConfig) { - cfg.Registrar = reg + cfg.ProposalDurationProvider = pacemaker.NewStaticProposalDurationProvider(dur) } } diff --git a/consensus/follower.go b/consensus/follower.go index c366d2d8881..24d272ce0ff 100644 --- a/consensus/follower.go +++ b/consensus/follower.go @@ -6,8 +6,6 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/follower" - "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/recovery" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -16,33 +14,37 @@ import ( // TODO: this needs to be integrated with proper configuration and bootstrapping. -func NewFollower(log zerolog.Logger, committee hotstuff.DynamicCommittee, headers storage.Headers, updater module.Finalizer, - verifier hotstuff.Verifier, notifier hotstuff.FinalizationConsumer, rootHeader *flow.Header, - rootQC *flow.QuorumCertificate, finalized *flow.Header, pending []*flow.Header, +// NewFollower instantiates the consensus follower and recovers its in-memory state of pending blocks. +// It receives the list `pending` containing _all_ blocks that +// - have passed the compliance layer and stored in the protocol state +// - descend from the latest finalized block +// - are listed in ancestor-first order (i.e. for any block B ∈ pending, B's parent must +// be listed before B, unless B's parent is the latest finalized block) +// +// CAUTION: all pending blocks are required to be valid (guaranteed if the block passed the compliance layer) +func NewFollower(log zerolog.Logger, + mempoolMetrics module.MempoolMetrics, + headers storage.Headers, + updater module.Finalizer, + notifier hotstuff.FollowerConsumer, + rootHeader *flow.Header, + rootQC *flow.QuorumCertificate, + finalized *flow.Header, + pending []*flow.Header, ) (*hotstuff.FollowerLoop, error) { - forks, err := NewForks(finalized, headers, updater, notifier, rootHeader, rootQC) if err != nil { return nil, fmt.Errorf("could not initialize forks: %w", err) } - // initialize the Validator - validator := validator.New(committee, verifier) - - // recover the HotStuff follower's internal state (inserts all pending blocks into Forks) - err = recovery.Follower(log, forks, validator, pending) + // recover forks internal state (inserts all pending blocks) + err = recovery.Recover(log, pending, recovery.ForksState(forks)) if err != nil { return nil, fmt.Errorf("could not recover hotstuff follower state: %w", err) } - // initialize the follower logic - logic, err := follower.New(log, validator, forks) - if err != nil { - return nil, fmt.Errorf("could not create follower logic: %w", err) - } - // initialize the follower loop - loop, err := hotstuff.NewFollowerLoop(log, logic) + loop, err := hotstuff.NewFollowerLoop(log, mempoolMetrics, forks) if err != nil { return nil, fmt.Errorf("could not create follower loop: %w", err) } diff --git a/consensus/follower_test.go b/consensus/follower_test.go index 26a61c88ae5..06e81b70f25 100644 --- a/consensus/follower_test.go +++ b/consensus/follower_test.go @@ -6,8 +6,6 @@ import ( "testing" "time" - "github.com/onflow/flow-go/module/signature" - "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -15,16 +13,23 @@ import ( "github.com/onflow/flow-go/consensus" "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/committees" mockhotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" mockmodule "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/module/signature" mockstorage "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/unittest" ) +/***************************************************************************** + * NOTATION: * + * A block is denoted as [◄() ]. * + * For example, [◄(1) 2] means: a block of view 2 that has a QC for view 1. * + *****************************************************************************/ + // TestHotStuffFollower is a test suite for the HotStuff Follower. // The main focus of this test suite is to test that the follower generates the expected callbacks to // module.Finalizer and hotstuff.FinalizationConsumer. In this context, note that the Follower internally @@ -52,11 +57,9 @@ func TestHotStuffFollower(t *testing.T) { type HotStuffFollowerSuite struct { suite.Suite - committee *mockhotstuff.DynamicCommittee headers *mockstorage.Headers finalizer *mockmodule.Finalizer - verifier *mockhotstuff.Verifier - notifier *mockhotstuff.FinalizationConsumer + notifier *mockhotstuff.FollowerConsumer rootHeader *flow.Header rootQC *flow.QuorumCertificate finalized *flow.Header @@ -75,38 +78,14 @@ func (s *HotStuffFollowerSuite) SetupTest() { identities := unittest.IdentityListFixture(4, unittest.WithRole(flow.RoleConsensus)) s.mockConsensus = &MockConsensus{identities: identities} - // mock consensus committee - s.committee = &mockhotstuff.DynamicCommittee{} - s.committee.On("IdentitiesByEpoch", mock.Anything).Return( - func(_ uint64) flow.IdentityList { - return identities - }, - nil, - ) - for _, identity := range identities { - s.committee.On("IdentityByEpoch", mock.Anything, identity.NodeID).Return(identity, nil) - s.committee.On("IdentityByBlock", mock.Anything, identity.NodeID).Return(identity, nil) - } - s.committee.On("LeaderForView", mock.Anything).Return( - func(view uint64) flow.Identifier { return identities[int(view)%len(identities)].NodeID }, - nil, - ) - s.committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(identities.TotalWeight()), nil) - // mock storage headers s.headers = &mockstorage.Headers{} // mock finalization finalizer s.finalizer = mockmodule.NewFinalizer(s.T()) - // mock finalization finalizer - s.verifier = mockhotstuff.NewVerifier(s.T()) - s.verifier.On("VerifyVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - s.verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - s.verifier.On("VerifyTC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - // mock consumer for finalization notifications - s.notifier = mockhotstuff.NewFinalizationConsumer(s.T()) + s.notifier = mockhotstuff.NewFollowerConsumer(s.T()) // root block and QC parentID, err := flow.HexStringToIdentifier("aa7693d498e9a087b1cadf5bfe9a1ff07829badc1915c210e482f369f9a00a70") @@ -138,10 +117,9 @@ func (s *HotStuffFollowerSuite) BeforeTest(suiteName, testName string) { var err error s.follower, err = consensus.NewFollower( zerolog.New(os.Stderr), - s.committee, + metrics.NewNoopCollector(), s.headers, s.finalizer, - s.verifier, s.notifier, s.rootHeader, s.rootQC, @@ -159,6 +137,7 @@ func (s *HotStuffFollowerSuite) BeforeTest(suiteName, testName string) { func (s *HotStuffFollowerSuite) AfterTest(suiteName, testName string) { s.cancel() unittest.RequireCloseBefore(s.T(), s.follower.Done(), time.Second, "follower failed to stop") + select { case err := <-s.errs: require.NoError(s.T(), err) @@ -171,72 +150,106 @@ func (s *HotStuffFollowerSuite) TestInitialization() { // we expect no additional calls to s.finalizer or s.notifier besides what is already specified in BeforeTest } -// TestSubmitProposal verifies that when submitting a single valid block (child's root block), +// TestOnBlockIncorporated verifies that when submitting a single valid block, // the Follower reacts with callbacks to s.notifier.OnBlockIncorporated with this new block -func (s *HotStuffFollowerSuite) TestSubmitProposal() { +// We simulate the following consensus Fork: +// +// [ 52078] <-- [◄(52078) 52078+2] <-- [◄(52078+2) 52078+3] +// ╰─────────────────────────────────╯ +// certified child of root block +// +// with: +// - [ 52078] is the root block with view 52078 +// - The child block [◄(52078) 52078+2] was produced 2 views later. This +// is an _indirect_ 1 chain and therefore does not advance finalization. +// - the certified child is given by [◄(52078) 52078+2] ◄(52078+2) +func (s *HotStuffFollowerSuite) TestOnBlockIncorporated() { rootBlockView := s.rootHeader.View - nextBlock := s.mockConsensus.extendBlock(rootBlockView+1, s.rootHeader) + child := s.mockConsensus.extendBlock(rootBlockView+2, s.rootHeader) + grandChild := s.mockConsensus.extendBlock(child.View+2, child) - s.notifier.On("OnBlockIncorporated", blockWithID(nextBlock.ID())).Return().Once() - s.submitProposal(nextBlock) + certifiedChild := toCertifiedBlock(s.T(), child, grandChild.QuorumCertificate()) + blockIngested := make(chan struct{}) // close when child was ingested + s.notifier.On("OnBlockIncorporated", blockWithID(child.ID())).Run(func(_ mock.Arguments) { + close(blockIngested) + }).Return().Once() + + s.follower.AddCertifiedBlock(certifiedChild) + unittest.RequireCloseBefore(s.T(), blockIngested, time.Second, "expect `OnBlockIncorporated` notification before timeout") } -// TestFollowerFinalizedBlock verifies that when submitting 2 extra blocks +// TestFollowerFinalizedBlock verifies that when submitting a certified block that advances +// finality, the follower detects this and emits a finalization `OnFinalizedBlock` // the Follower reacts with callbacks to s.notifier.OnBlockIncorporated // for all the added blocks. Furthermore, the follower should finalize the first submitted block, // i.e. call s.finalizer.MakeFinal and s.notifier.OnFinalizedBlock +// +// TestFollowerFinalizedBlock verifies that when submitting a certified block that, +// the Follower reacts with callbacks to s.notifier.OnBlockIncorporated with this new block +// We simulate the following consensus Fork: +// +// block b (view 52078+2) +// ╭─────────^────────╮ +// [ 52078] <-- [◄(52078) 52078+2] <-- [◄(52078+2) 52078+3] <-- [◄(52078+3) 52078+5] +// ╰─────────────────────────────────────╯ +// certified child of b +// +// with: +// - [ 52078] is the root block with view 52078 +// - The block b = [◄(52078) 52078+2] was produced 2 views later (no finalization advancement). +// - Block b has a certified child: [◄(52078+2) 52078+3] ◄(52078+3) +// The child's view 52078+3 is exactly one bigger than B's view. Hence it proves finalization of b. func (s *HotStuffFollowerSuite) TestFollowerFinalizedBlock() { - expectedFinalized := s.mockConsensus.extendBlock(s.rootHeader.View+1, s.rootHeader) - s.notifier.On("OnBlockIncorporated", blockWithID(expectedFinalized.ID())).Return().Once() - s.submitProposal(expectedFinalized) - - // direct 1-chain on top of expectedFinalized - nextBlock := s.mockConsensus.extendBlock(expectedFinalized.View+1, expectedFinalized) - s.notifier.On("OnBlockIncorporated", blockWithID(nextBlock.ID())).Return().Once() - s.submitProposal(nextBlock) - - done := make(chan struct{}) - - // indirect 2-chain on top of expectedFinalized - lastBlock := nextBlock - nextBlock = s.mockConsensus.extendBlock(lastBlock.View+5, lastBlock) - s.notifier.On("OnBlockIncorporated", blockWithID(nextBlock.ID())).Return().Once() - s.notifier.On("OnFinalizedBlock", blockWithID(expectedFinalized.ID())).Return().Once() - s.finalizer.On("MakeFinal", blockID(expectedFinalized.ID())).Run(func(_ mock.Arguments) { - close(done) - }).Return(nil).Once() - s.submitProposal(nextBlock) - unittest.RequireCloseBefore(s.T(), done, time.Second, "expect to close before timeout") + b := s.mockConsensus.extendBlock(s.rootHeader.View+2, s.rootHeader) + c := s.mockConsensus.extendBlock(b.View+1, b) + d := s.mockConsensus.extendBlock(c.View+1, c) + + // adding b should not advance finality + bCertified := toCertifiedBlock(s.T(), b, c.QuorumCertificate()) + s.notifier.On("OnBlockIncorporated", blockWithID(b.ID())).Return().Once() + s.follower.AddCertifiedBlock(bCertified) + + // adding the certified child of b should advance finality to b + finalityAdvanced := make(chan struct{}) // close when finality has advanced to b + certifiedChild := toCertifiedBlock(s.T(), c, d.QuorumCertificate()) + s.notifier.On("OnBlockIncorporated", blockWithID(certifiedChild.ID())).Return().Once() + s.finalizer.On("MakeFinal", blockID(b.ID())).Return(nil).Once() + s.notifier.On("OnFinalizedBlock", blockWithID(b.ID())).Run(func(_ mock.Arguments) { + close(finalityAdvanced) + }).Return().Once() + + s.follower.AddCertifiedBlock(certifiedChild) + unittest.RequireCloseBefore(s.T(), finalityAdvanced, time.Second, "expect finality progress before timeout") } // TestOutOfOrderBlocks verifies that when submitting a variety of blocks with view numbers // OUT OF ORDER, the Follower reacts with callbacks to s.notifier.OnBlockIncorporated // for all the added blocks. Furthermore, we construct the test such that the follower should finalize // eventually a bunch of blocks in one go. -// The following illustrates the tree of submitted blocks, with notation +// The following illustrates the tree of submitted blocks: // -// [52078+14, 52078+20] (should finalize this fork) -// | -// | -// [52078+13, 52078+14] -// | -// | -// [52078+11, 52078+17] [52078+ 9, 52078+13] [52078+ 9, 52078+10] -// | | / -// | | / -// [52078+ 7, 52078+ 8] [52078+ 7, 52078+11] [52078+ 5, 52078+ 9] [52078+ 5, 52078+ 6] +// [◄(52078+14) 52078+20] (should finalize this fork) +// | +// | +// [◄(52078+13) 52078+14] +// | +// | +// [◄(52078+11) 52078+17] [◄(52078+9) 52078+13] [◄(52078+9) 52078+10] +// | | / +// | |/ +// [◄(52078+7) 52078+ 8] [◄(52078+7) 52078+11] [◄(52078+5) 52078+9] [◄(52078+5) 52078+6] // \ | | / -// \| | / -// [52078+ 3, 52078+ 4] [52078+ 3, 52078+ 7] [52078+ 1, 52078+ 5] [52078+ 1, 52078+ 2] +// \| |/ +// [◄(52078+3) 52078+4] [◄(52078+3) 52078+7] [◄(52078+1) 52078+5] [◄(52078+1) 52078+2] // \ | | / -// \| | / -// [52078+ 0, 52078+ 3] [52078+ 0, 52078+ 1] +// \| |/ +// [◄(52078+0) 52078+3] [◄(52078+0) 52078+1] // \ / // \ / -// [52078+ 0, x] (root block; no qc to parent) +// [◄(52078+0) x] (root block; no qc to parent) func (s *HotStuffFollowerSuite) TestOutOfOrderBlocks() { // in the following, we reference the block's by their view minus the view of the - // root block (52078). E.g. block [52078+ 9, 52078+10] would be referenced as `block10` + // root block (52078). E.g. block [◄(52078+ 9) 52078+10] would be referenced as `block10` rootView := s.rootHeader.View // constructing blocks bottom up, line by line, left to right @@ -260,30 +273,22 @@ func (s *HotStuffFollowerSuite) TestOutOfOrderBlocks() { block14 := s.mockConsensus.extendBlock(rootView+14, block13) block20 := s.mockConsensus.extendBlock(rootView+20, block14) - for _, b := range []*flow.Header{block01, block02, block03, block04, block05, block06, block07, block08, block09, block10, block11, block13, block14, block17, block20} { + for _, b := range []*flow.Header{block01, block03, block05, block07, block09, block11, block13, block14} { s.notifier.On("OnBlockIncorporated", blockWithID(b.ID())).Return().Once() } // now we feed the blocks in some wild view order into the Follower // (Caution: we still have to make sure the parent is known, before we give its child to the Follower) - s.submitProposal(block03) - s.submitProposal(block07) - s.submitProposal(block11) - s.submitProposal(block01) - s.submitProposal(block05) - s.submitProposal(block17) - s.submitProposal(block09) - s.submitProposal(block06) - s.submitProposal(block10) - s.submitProposal(block04) - s.submitProposal(block13) - s.submitProposal(block14) - s.submitProposal(block08) - s.submitProposal(block02) - - done := make(chan struct{}) + s.follower.AddCertifiedBlock(toCertifiedBlock(s.T(), block03, block04.QuorumCertificate())) + s.follower.AddCertifiedBlock(toCertifiedBlock(s.T(), block07, block08.QuorumCertificate())) + s.follower.AddCertifiedBlock(toCertifiedBlock(s.T(), block11, block17.QuorumCertificate())) + s.follower.AddCertifiedBlock(toCertifiedBlock(s.T(), block01, block02.QuorumCertificate())) + s.follower.AddCertifiedBlock(toCertifiedBlock(s.T(), block05, block06.QuorumCertificate())) + s.follower.AddCertifiedBlock(toCertifiedBlock(s.T(), block09, block10.QuorumCertificate())) + s.follower.AddCertifiedBlock(toCertifiedBlock(s.T(), block13, block14.QuorumCertificate())) // Block 20 should now finalize the fork up to and including block13 + finalityAdvanced := make(chan struct{}) // close when finality has advanced to b s.notifier.On("OnFinalizedBlock", blockWithID(block01.ID())).Return().Once() s.finalizer.On("MakeFinal", blockID(block01.ID())).Return(nil).Once() s.notifier.On("OnFinalizedBlock", blockWithID(block05.ID())).Return().Once() @@ -292,10 +297,11 @@ func (s *HotStuffFollowerSuite) TestOutOfOrderBlocks() { s.finalizer.On("MakeFinal", blockID(block09.ID())).Return(nil).Once() s.notifier.On("OnFinalizedBlock", blockWithID(block13.ID())).Return().Once() s.finalizer.On("MakeFinal", blockID(block13.ID())).Run(func(_ mock.Arguments) { - close(done) + close(finalityAdvanced) }).Return(nil).Once() - s.submitProposal(block20) - unittest.RequireCloseBefore(s.T(), done, time.Second, "expect to close before timeout") + + s.follower.AddCertifiedBlock(toCertifiedBlock(s.T(), block14, block20.QuorumCertificate())) + unittest.RequireCloseBefore(s.T(), finalityAdvanced, time.Second, "expect finality progress before timeout") } // blockWithID returns a testify `argumentMatcher` that only accepts blocks with the given ID @@ -308,9 +314,11 @@ func blockID(expectedBlockID flow.Identifier) interface{} { return mock.MatchedBy(func(blockID flow.Identifier) bool { return expectedBlockID == blockID }) } -// submitProposal submits the given (proposal, parentView) pair to the Follower. -func (s *HotStuffFollowerSuite) submitProposal(proposal *flow.Header) { - s.follower.SubmitProposal(model.ProposalFromFlow(proposal)) +func toCertifiedBlock(t *testing.T, block *flow.Header, qc *flow.QuorumCertificate) *model.CertifiedBlock { + // adding b should not advance finality + certifiedBlock, err := model.NewCertifiedBlock(model.BlockFromFlow(block), qc) + require.NoError(t, err) + return &certifiedBlock } // MockConsensus is used to generate Blocks for a mocked consensus committee diff --git a/consensus/hotstuff/committees/cluster_committee_test.go b/consensus/hotstuff/committees/cluster_committee_test.go index 83903d23c3d..e6c36aea044 100644 --- a/consensus/hotstuff/committees/cluster_committee_test.go +++ b/consensus/hotstuff/committees/cluster_committee_test.go @@ -12,7 +12,7 @@ import ( clusterstate "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/state/protocol" protocolmock "github.com/onflow/flow-go/state/protocol/mock" - "github.com/onflow/flow-go/state/protocol/seed" + "github.com/onflow/flow-go/state/protocol/prg" storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -56,7 +56,7 @@ func (suite *ClusterSuite) SetupTest() { suite.cluster.On("Members").Return(suite.members) suite.cluster.On("RootBlock").Return(suite.root) suite.epoch.On("Counter").Return(counter, nil) - suite.epoch.On("RandomSource").Return(unittest.SeedFixture(seed.RandomSourceLength), nil) + suite.epoch.On("RandomSource").Return(unittest.SeedFixture(prg.RandomSourceLength), nil) var err error suite.com, err = NewClusterCommittee( diff --git a/consensus/hotstuff/committees/consensus_committee.go b/consensus/hotstuff/committees/consensus_committee.go index cc29265e464..2c81adc78f3 100644 --- a/consensus/hotstuff/committees/consensus_committee.go +++ b/consensus/hotstuff/committees/consensus_committee.go @@ -15,7 +15,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/events" - "github.com/onflow/flow-go/state/protocol/seed" + "github.com/onflow/flow-go/state/protocol/prg" ) // staticEpochInfo contains leader selection and the initial committee for one epoch. @@ -85,7 +85,7 @@ func newStaticEpochInfo(epoch protocol.Epoch) (*staticEpochInfo, error) { // * has the same static committee as the last committed epoch func newEmergencyFallbackEpoch(lastCommittedEpoch *staticEpochInfo) (*staticEpochInfo, error) { - rng, err := seed.PRGFromRandomSource(lastCommittedEpoch.randomSource, seed.ProtocolConsensusLeaderSelection) + rng, err := prg.New(lastCommittedEpoch.randomSource, prg.ConsensusLeaderSelection, nil) if err != nil { return nil, fmt.Errorf("could not create rng from seed: %w", err) } diff --git a/consensus/hotstuff/committees/consensus_committee_test.go b/consensus/hotstuff/committees/consensus_committee_test.go index b8d1f5bc415..61012ee51a9 100644 --- a/consensus/hotstuff/committees/consensus_committee_test.go +++ b/consensus/hotstuff/committees/consensus_committee_test.go @@ -18,7 +18,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" protocolmock "github.com/onflow/flow-go/state/protocol/mock" - "github.com/onflow/flow-go/state/protocol/seed" + "github.com/onflow/flow-go/state/protocol/prg" "github.com/onflow/flow-go/utils/unittest" "github.com/onflow/flow-go/utils/unittest/mocks" ) @@ -269,7 +269,7 @@ func (suite *ConsensusSuite) TestIdentitiesByBlock() { blockID := unittest.IdentifierFixture() // create a mock epoch for leader selection setup in constructor - currEpoch := newMockEpoch(1, unittest.IdentityListFixture(10), 1, 100, unittest.SeedFixture(seed.RandomSourceLength), true) + currEpoch := newMockEpoch(1, unittest.IdentityListFixture(10), 1, 100, unittest.SeedFixture(prg.RandomSourceLength), true) suite.epochs.Add(currEpoch) suite.state.On("AtBlockID", blockID).Return(suite.snapshot) @@ -337,9 +337,9 @@ func (suite *ConsensusSuite) TestIdentitiesByEpoch() { epoch2Identities := flow.IdentityList{epoch2Identity} // create a mock epoch for leader selection setup in constructor - epoch1 := newMockEpoch(suite.currentEpochCounter, epoch1Identities, 1, 100, unittest.SeedFixture(seed.RandomSourceLength), true) + epoch1 := newMockEpoch(suite.currentEpochCounter, epoch1Identities, 1, 100, unittest.SeedFixture(prg.RandomSourceLength), true) // initially epoch 2 is not committed - epoch2 := newMockEpoch(suite.currentEpochCounter+1, epoch2Identities, 101, 200, unittest.SeedFixture(seed.RandomSourceLength), true) + epoch2 := newMockEpoch(suite.currentEpochCounter+1, epoch2Identities, 101, 200, unittest.SeedFixture(prg.RandomSourceLength), true) suite.epochs.Add(epoch1) suite.CreateAndStartCommittee() @@ -428,7 +428,7 @@ func (suite *ConsensusSuite) TestThresholds() { identities := unittest.IdentityListFixture(10) - prevEpoch := newMockEpoch(suite.currentEpochCounter-1, identities.Map(mapfunc.WithWeight(100)), 1, 100, unittest.SeedFixture(seed.RandomSourceLength), true) + prevEpoch := newMockEpoch(suite.currentEpochCounter-1, identities.Map(mapfunc.WithWeight(100)), 1, 100, unittest.SeedFixture(prg.RandomSourceLength), true) currEpoch := newMockEpoch(suite.currentEpochCounter, identities.Map(mapfunc.WithWeight(200)), 101, 200, unittest.SeedFixture(32), true) suite.epochs.Add(prevEpoch) suite.epochs.Add(currEpoch) @@ -466,7 +466,7 @@ func (suite *ConsensusSuite) TestThresholds() { }) // now, add a valid next epoch - nextEpoch := newMockEpoch(suite.currentEpochCounter+1, identities.Map(mapfunc.WithWeight(300)), 201, 300, unittest.SeedFixture(seed.RandomSourceLength), true) + nextEpoch := newMockEpoch(suite.currentEpochCounter+1, identities.Map(mapfunc.WithWeight(300)), 201, 300, unittest.SeedFixture(prg.RandomSourceLength), true) suite.CommitEpoch(nextEpoch) t.Run("next epoch ready", func(t *testing.T) { @@ -517,7 +517,7 @@ func (suite *ConsensusSuite) TestLeaderForView() { identities := unittest.IdentityListFixture(10) - prevEpoch := newMockEpoch(suite.currentEpochCounter-1, identities, 1, 100, unittest.SeedFixture(seed.RandomSourceLength), true) + prevEpoch := newMockEpoch(suite.currentEpochCounter-1, identities, 1, 100, unittest.SeedFixture(prg.RandomSourceLength), true) currEpoch := newMockEpoch(suite.currentEpochCounter, identities, 101, 200, unittest.SeedFixture(32), true) suite.epochs.Add(currEpoch) suite.epochs.Add(prevEpoch) @@ -550,7 +550,7 @@ func (suite *ConsensusSuite) TestLeaderForView() { }) // now, add a valid next epoch - nextEpoch := newMockEpoch(suite.currentEpochCounter+1, identities, 201, 300, unittest.SeedFixture(seed.RandomSourceLength), true) + nextEpoch := newMockEpoch(suite.currentEpochCounter+1, identities, 201, 300, unittest.SeedFixture(prg.RandomSourceLength), true) suite.CommitEpoch(nextEpoch) t.Run("next epoch ready", func(t *testing.T) { @@ -597,7 +597,7 @@ func TestRemoveOldEpochs(t *testing.T) { currentEpochCounter := firstEpochCounter epochFinalView := uint64(100) - epoch1 := newMockEpoch(currentEpochCounter, identities, 1, epochFinalView, unittest.SeedFixture(seed.RandomSourceLength), true) + epoch1 := newMockEpoch(currentEpochCounter, identities, 1, epochFinalView, unittest.SeedFixture(prg.RandomSourceLength), true) // create mocks state := new(protocolmock.State) @@ -634,7 +634,7 @@ func TestRemoveOldEpochs(t *testing.T) { firstView := epochFinalView + 1 epochFinalView = epochFinalView + 100 currentEpochCounter++ - nextEpoch := newMockEpoch(currentEpochCounter, identities, firstView, epochFinalView, unittest.SeedFixture(seed.RandomSourceLength), true) + nextEpoch := newMockEpoch(currentEpochCounter, identities, firstView, epochFinalView, unittest.SeedFixture(prg.RandomSourceLength), true) epochQuery.Add(nextEpoch) currentEpochPhase = flow.EpochPhaseCommitted diff --git a/consensus/hotstuff/committees/leader/cluster.go b/consensus/hotstuff/committees/leader/cluster.go index 2de6899d8d4..b1a2af13be2 100644 --- a/consensus/hotstuff/committees/leader/cluster.go +++ b/consensus/hotstuff/committees/leader/cluster.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/state/protocol/seed" + "github.com/onflow/flow-go/state/protocol/prg" ) // SelectionForCluster pre-computes and returns leaders for the given cluster @@ -27,7 +27,7 @@ func SelectionForCluster(cluster protocol.Cluster, epoch protocol.Epoch) (*Leade return nil, fmt.Errorf("could not get leader selection seed for cluster (index: %v) at epoch: %v: %w", cluster.Index(), counter, err) } // create random number generator from the seed and customizer - rng, err := seed.PRGFromRandomSource(randomSeed, seed.ProtocolCollectorClusterLeaderSelection(cluster.Index())) + rng, err := prg.New(randomSeed, prg.CollectorClusterLeaderSelection(cluster.Index()), nil) if err != nil { return nil, fmt.Errorf("could not create rng: %w", err) } diff --git a/consensus/hotstuff/committees/leader/consensus.go b/consensus/hotstuff/committees/leader/consensus.go index c9ea12eeece..17f8c108069 100644 --- a/consensus/hotstuff/committees/leader/consensus.go +++ b/consensus/hotstuff/committees/leader/consensus.go @@ -5,7 +5,7 @@ import ( "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/state/protocol/seed" + "github.com/onflow/flow-go/state/protocol/prg" ) // SelectionForConsensus pre-computes and returns leaders for the consensus committee @@ -26,7 +26,7 @@ func SelectionForConsensus(epoch protocol.Epoch) (*LeaderSelection, error) { return nil, fmt.Errorf("could not get epoch seed: %w", err) } // create random number generator from the seed and customizer - rng, err := seed.PRGFromRandomSource(randomSeed, seed.ProtocolConsensusLeaderSelection) + rng, err := prg.New(randomSeed, prg.ConsensusLeaderSelection, nil) if err != nil { return nil, fmt.Errorf("could not create rng: %w", err) } diff --git a/consensus/hotstuff/committees/leader/leader_selection_test.go b/consensus/hotstuff/committees/leader/leader_selection_test.go index 7d580c76a6a..ecf13e4aa83 100644 --- a/consensus/hotstuff/committees/leader/leader_selection_test.go +++ b/consensus/hotstuff/committees/leader/leader_selection_test.go @@ -24,7 +24,7 @@ var someSeed = []uint8{0x6A, 0x23, 0x41, 0xB7, 0x80, 0xE1, 0x64, 0x59, func TestSingleConsensusNode(t *testing.T) { identity := unittest.IdentityFixture(unittest.WithWeight(8)) - rng := prg(t, someSeed) + rng := getPRG(t, someSeed) selection, err := ComputeLeaderSelection(0, rng, 10, []*flow.Identity{identity}) require.NoError(t, err) for i := uint64(0); i < 10; i++ { @@ -114,7 +114,7 @@ func bruteSearch(value uint64, arr []uint64) (int, error) { return 0, fmt.Errorf("not found") } -func prg(t testing.TB, seed []byte) random.Rand { +func getPRG(t testing.TB, seed []byte) random.Rand { rng, err := random.NewChacha20PRG(seed, []byte("random")) require.NoError(t, err) return rng @@ -130,12 +130,12 @@ func TestDeterministic(t *testing.T) { for i, identity := range identities { identity.Weight = uint64(i + 1) } - rng := prg(t, someSeed) + rng := getPRG(t, someSeed) leaders1, err := ComputeLeaderSelection(0, rng, N_VIEWS, identities) require.NoError(t, err) - rng = prg(t, someSeed) + rng = getPRG(t, someSeed) leaders2, err := ComputeLeaderSelection(0, rng, N_VIEWS, identities) require.NoError(t, err) @@ -153,7 +153,7 @@ func TestDeterministic(t *testing.T) { func TestInputValidation(t *testing.T) { - rng := prg(t, someSeed) + rng := getPRG(t, someSeed) // should return an error if we request to compute leader selection for <1 views t.Run("epoch containing no views", func(t *testing.T) { @@ -176,7 +176,7 @@ func TestInputValidation(t *testing.T) { // test that requesting a view outside the given range returns an error func TestViewOutOfRange(t *testing.T) { - rng := prg(t, someSeed) + rng := getPRG(t, someSeed) firstView := uint64(100) finalView := uint64(200) @@ -203,7 +203,7 @@ func TestViewOutOfRange(t *testing.T) { _, err = leaders.LeaderForView(before) assert.Error(t, err) - before = rand.Uint64() % firstView // random view before first view + before = uint64(rand.Intn(int(firstView))) // random view before first view _, err = leaders.LeaderForView(before) assert.Error(t, err) }) @@ -230,11 +230,11 @@ func TestDifferentSeedWillProduceDifferentSelection(t *testing.T) { identity.Weight = uint64(i) } - rng1 := prg(t, someSeed) + rng1 := getPRG(t, someSeed) seed2 := make([]byte, 32) seed2[0] = 8 - rng2 := prg(t, seed2) + rng2 := getPRG(t, seed2) leaders1, err := ComputeLeaderSelection(0, rng1, N_VIEWS, identities) require.NoError(t, err) @@ -262,7 +262,7 @@ func TestDifferentSeedWillProduceDifferentSelection(t *testing.T) { // The number of time being selected as leader might not exactly match their weight, but also // won't go too far from that. func TestLeaderSelectionAreWeighted(t *testing.T) { - rng := prg(t, someSeed) + rng := getPRG(t, someSeed) const N_VIEWS = 100000 const N_NODES = 4 @@ -311,7 +311,7 @@ func BenchmarkLeaderSelection(b *testing.B) { for i := 0; i < N_NODES; i++ { identities = append(identities, unittest.IdentityFixture(unittest.WithWeight(uint64(i)))) } - rng := prg(b, someSeed) + rng := getPRG(b, someSeed) for n := 0; n < b.N; n++ { _, err := ComputeLeaderSelection(0, rng, N_VIEWS, identities) @@ -321,7 +321,7 @@ func BenchmarkLeaderSelection(b *testing.B) { } func TestInvalidTotalWeight(t *testing.T) { - rng := prg(t, someSeed) + rng := getPRG(t, someSeed) identities := unittest.IdentityListFixture(4, unittest.WithWeight(0)) _, err := ComputeLeaderSelection(0, rng, 10, identities) require.Error(t, err) @@ -330,8 +330,8 @@ func TestInvalidTotalWeight(t *testing.T) { func TestZeroWeightNodeWillNotBeSelected(t *testing.T) { // create 2 RNGs from the same seed - rng := prg(t, someSeed) - rng_copy := prg(t, someSeed) + rng := getPRG(t, someSeed) + rng_copy := getPRG(t, someSeed) // check that if there is some node with 0 weight, the selections for each view should be the same as // with no zero-weight nodes. @@ -365,7 +365,7 @@ func TestZeroWeightNodeWillNotBeSelected(t *testing.T) { }) t.Run("fuzzy set", func(t *testing.T) { - toolRng := prg(t, someSeed) + toolRng := getPRG(t, someSeed) // create 1002 nodes with all 0 weight identities := unittest.IdentityListFixture(1002, unittest.WithWeight(0)) @@ -399,7 +399,7 @@ func TestZeroWeightNodeWillNotBeSelected(t *testing.T) { } t.Run("if there is only 1 node has weight, then it will be always be the leader and the only leader", func(t *testing.T) { - toolRng := prg(t, someSeed) + toolRng := getPRG(t, someSeed) identities := unittest.IdentityListFixture(1000, unittest.WithWeight(0)) diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index 5eb592b9912..1a5bfb175af 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -7,52 +7,119 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// FinalizationConsumer consumes outbound notifications produced by the finalization logic. -// Notifications represent finalization-specific state changes which are potentially relevant -// to the larger node. The notifications are emitted in the order in which the -// finalization algorithm makes the respective steps. +// ProposalViolationConsumer consumes outbound notifications about HotStuff-protocol violations. +// Such notifications are produced by the active consensus participants and consensus follower. // // Implementations must: // - be concurrency safe // - be non-blocking // - handle repetition of the same events (with some processing overhead). -type FinalizationConsumer interface { +type ProposalViolationConsumer interface { + // OnInvalidBlockDetected notifications are produced by components that have detected + // that a block proposal is invalid and need to report it. + // Most of the time such block can be detected by calling Validator.ValidateProposal. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnInvalidBlockDetected(err flow.Slashable[model.InvalidProposalError]) - // OnBlockIncorporated notifications are produced by the Finalization Logic - // whenever a block is incorporated into the consensus state. + // OnDoubleProposeDetected notifications are produced by the Finalization Logic + // whenever a double block proposal (equivocation) was detected. + // Equivocation occurs when the same leader proposes two different blocks for the same view. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnBlockIncorporated(*model.Block) + OnDoubleProposeDetected(*model.Block, *model.Block) +} - // OnFinalizedBlock notifications are produced by the Finalization Logic whenever - // a block has been finalized. They are emitted in the order the blocks are finalized. +// VoteAggregationViolationConsumer consumes outbound notifications about HotStuff-protocol violations specifically +// invalid votes during processing. +// Such notifications are produced by the Vote Aggregation logic. +// +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type VoteAggregationViolationConsumer interface { + // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic + // whenever a double voting (same voter voting for different blocks at the same view) was detected. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnFinalizedBlock(*model.Block) + OnDoubleVotingDetected(*model.Vote, *model.Vote) - // OnDoubleProposeDetected notifications are produced by the Finalization Logic - // whenever a double block proposal (equivocation) was detected. + // OnInvalidVoteDetected notifications are produced by the Vote Aggregation logic + // whenever an invalid vote was detected. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnDoubleProposeDetected(*model.Block, *model.Block) + OnInvalidVoteDetected(err model.InvalidVoteError) + + // OnVoteForInvalidBlockDetected notifications are produced by the Vote Aggregation logic + // whenever vote for invalid proposal was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) } -// Consumer consumes outbound notifications produced by HotStuff and its components. -// Notifications are consensus-internal state changes which are potentially relevant to -// the larger node in which HotStuff is running. The notifications are emitted -// in the order in which the HotStuff algorithm makes the respective steps. +// TimeoutAggregationViolationConsumer consumes outbound notifications about Active Pacemaker violations specifically +// invalid timeouts during processing. +// Such notifications are produced by the Timeout Aggregation logic. // // Implementations must: // - be concurrency safe // - be non-blocking // - handle repetition of the same events (with some processing overhead). -type Consumer interface { - FinalizationConsumer - CommunicatorConsumer +type TimeoutAggregationViolationConsumer interface { + // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic + // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) + + // OnInvalidTimeoutDetected notifications are produced by the Timeout Aggregation logic + // whenever an invalid timeout was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnInvalidTimeoutDetected(err model.InvalidTimeoutError) +} + +// FinalizationConsumer consumes outbound notifications produced by the logic tracking +// forks and finalization. Such notifications are produced by the active consensus +// participants, and generally potentially relevant to the larger node. The notifications +// are emitted in the order in which the finalization algorithm makes the respective steps. +// +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type FinalizationConsumer interface { + // OnBlockIncorporated notifications are produced by the Finalization Logic + // whenever a block is incorporated into the consensus state. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnBlockIncorporated(*model.Block) + // OnFinalizedBlock notifications are produced by the Finalization Logic whenever + // a block has been finalized. They are emitted in the order the blocks are finalized. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnFinalizedBlock(*model.Block) +} + +// ParticipantConsumer consumes outbound notifications produced by consensus participants +// actively proposing blocks, voting, collecting & aggregating votes to QCs, and participating in +// the pacemaker (sending timeouts, collecting & aggregating timeouts to TCs). +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type ParticipantConsumer interface { // OnEventProcessed notifications are produced by the EventHandler when it is done processing // and hands control back to the EventLoop to wait for the next event. // Prerequisites: @@ -133,20 +200,6 @@ type Consumer interface { // and must handle repetition of the same events (with some processing overhead). OnStartingTimeout(model.TimerInfo) - // OnVoteProcessed notifications are produced by the Vote Aggregation logic, each time - // we successfully ingest a valid vote. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnVoteProcessed(vote *model.Vote) - - // OnTimeoutProcessed notifications are produced by the Timeout Aggregation logic, - // each time we successfully ingest a valid timeout. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnTimeoutProcessed(timeout *model.TimeoutObject) - // OnCurrentViewDetails notifications are produced by the EventHandler during the course of a view with auxiliary information. // These notifications are generally not produced for all views (for example skipped views). // These notifications are guaranteed to be produced for all views we enter after fully processing a message. @@ -162,59 +215,30 @@ type Consumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) - - // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic - // whenever a double voting (same voter voting for different blocks at the same view) was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnDoubleVotingDetected(*model.Vote, *model.Vote) - - // OnInvalidVoteDetected notifications are produced by the Vote Aggregation logic - // whenever an invalid vote was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnInvalidVoteDetected(err model.InvalidVoteError) - - // OnVoteForInvalidBlockDetected notifications are produced by the Vote Aggregation logic - // whenever vote for invalid proposal was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) - - // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic - // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) - - // OnInvalidTimeoutDetected notifications are produced by the Timeout Aggregation logic - // whenever an invalid timeout was detected. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnInvalidTimeoutDetected(err model.InvalidTimeoutError) } -// QCCreatedConsumer consumes outbound notifications produced by HotStuff and its components. -// Notifications are consensus-internal state changes which are potentially relevant to -// the larger node in which HotStuff is running. The notifications are emitted -// in the order in which the HotStuff algorithm makes the respective steps. +// VoteCollectorConsumer consumes outbound notifications produced by HotStuff's vote aggregation +// component. These events are primarily intended for the HotStuff-internal state machine (EventHandler), +// but might also be relevant to the larger node in which HotStuff is running. // // Implementations must: // - be concurrency safe // - be non-blocking // - handle repetition of the same events (with some processing overhead). -type QCCreatedConsumer interface { +type VoteCollectorConsumer interface { // OnQcConstructedFromVotes notifications are produced by the VoteAggregator // component, whenever it constructs a QC from votes. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnQcConstructedFromVotes(*flow.QuorumCertificate) + + // OnVoteProcessed notifications are produced by the Vote Aggregation logic, each time + // we successfully ingest a valid vote. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnVoteProcessed(vote *model.Vote) } // TimeoutCollectorConsumer consumes outbound notifications produced by HotStuff's timeout aggregation @@ -263,6 +287,13 @@ type TimeoutCollectorConsumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnNewTcDiscovered(certificate *flow.TimeoutCertificate) + + // OnTimeoutProcessed notifications are produced by the Timeout Aggregation logic, + // each time we successfully ingest a valid timeout. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnTimeoutProcessed(timeout *model.TimeoutObject) } // CommunicatorConsumer consumes outbound notifications produced by HotStuff and it's components. @@ -292,3 +323,51 @@ type CommunicatorConsumer interface { // and must handle repetition of the same events (with some processing overhead). OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) } + +// FollowerConsumer consumes outbound notifications produced by consensus followers. +// It is a subset of the notifications produced by consensus participants. +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type FollowerConsumer interface { + ProposalViolationConsumer + FinalizationConsumer +} + +// Consumer consumes outbound notifications produced by consensus participants. +// Notifications are consensus-internal state changes which are potentially relevant to +// the larger node in which HotStuff is running. The notifications are emitted +// in the order in which the HotStuff algorithm makes the respective steps. +// +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type Consumer interface { + FollowerConsumer + CommunicatorConsumer + ParticipantConsumer +} + +// VoteAggregationConsumer consumes outbound notifications produced by Vote Aggregation logic. +// It is a subset of the notifications produced by consensus participants. +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type VoteAggregationConsumer interface { + VoteAggregationViolationConsumer + VoteCollectorConsumer +} + +// TimeoutAggregationConsumer consumes outbound notifications produced by Vote Aggregation logic. +// It is a subset of the notifications produced by consensus participants. +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type TimeoutAggregationConsumer interface { + TimeoutAggregationViolationConsumer + TimeoutCollectorConsumer +} diff --git a/consensus/hotstuff/cruisectl/Readme.md b/consensus/hotstuff/cruisectl/Readme.md new file mode 100644 index 00000000000..1ae65560e31 --- /dev/null +++ b/consensus/hotstuff/cruisectl/Readme.md @@ -0,0 +1,294 @@ +# Cruise Control: Automated Block Time Adjustment for Precise Epoch Switchover Timing + +# Overview + +## Context + +Epochs have a fixed length, measured in views. +The actual view rate of the network varies depending on network conditions, e.g. load, number of offline replicas, etc. +We would like for consensus nodes to observe the actual view rate of the committee, and adjust how quickly they proceed +through views accordingly, to target a desired weekly epoch switchover time. + +## High-Level Design + +The `BlockTimeController` observes the current view rate and adjusts the timing when the proposal should be released. +It is a [PID controller](https://en.wikipedia.org/wiki/PID_controller). The essential idea is to take into account the +current error, the rate of change of the error, and the cumulative error, when determining how much compensation to apply. +The compensation function $u[v]$ has three terms: + +- $P[v]$ compensates proportionally to the magnitude of the instantaneous error +- $I[v]$ compensates proportionally to the magnitude of the error and how long it has persisted +- $D[v]$ compensates proportionally to the rate of change of the error + + +📚 This document uses ideas from: + +- the paper [Fast self-tuning PID controller specially suited for mini robots](https://www.frba.utn.edu.ar/wp-content/uploads/2021/02/EWMA_PID_7-1.pdf) +- the ‘Leaky Integrator’ [[forum discussion](https://engineering.stackexchange.com/questions/29833/limiting-the-integral-to-a-time-window-in-pid-controller), [technical background](https://www.music.mcgill.ca/~gary/307/week2/node4.html)] + + +### Choice of Process Variable: Targeted Epoch Switchover Time + +The process variable is the variable which: + +- has a target desired value, or setpoint ($SP$) +- is successively measured by the controller to compute the error $e$ + +--- +👉 The `BlockTimeController` controls the progression through views, such that the epoch switchover happens at the intended point in time. We define: + +- $\gamma = k\cdot \tau_0$ is the remaining epoch duration of a hypothetical ideal system, where *all* remaining $k$ views of the epoch progress with the ideal view time $\tau_0$. +- $\gamma = k\cdot \tau_0$ is the remaining epoch duration of a hypothetical ideal system, where *all* remaining $k$ views of the epoch progress with the ideal view time $\tau_0$. +- The parameter $\tau_0$ is computed solely based on the Epoch configuration as + $\tau_0 := \frac{<{\rm total\ epoch\ time}>}{<{\rm total\ views\ in\ epoch}>}$ (for mainnet 22, Epoch 75, we have $\tau_0 \simeq$ 1250ms). +- $\Gamma$ is the *actual* time remaining until the desired epoch switchover. + +The error, which the controller should drive towards zero, is defined as: + +```math +e := \gamma - \Gamma +``` +--- + + +From our definition it follows that: + +- $e > 0$ implies that the estimated epoch switchover (assuming ideal system behaviour) happens too late. Therefore, to hit the desired epoch switchover time, the time we spend in views has to be *smaller* than $\tau_0$. +- For $e < 0$ means that we estimate the epoch switchover to be too early. Therefore, we should be slowing down and spend more than $\tau_0$ in the following views. + +**Reasoning:** + +The desired idealized system behaviour would a constant view duration $\tau_0$ throughout the entire epoch. + +However, in the real-world system we have disturbances (varying message relay times, slow or offline nodes, etc) and measurement uncertainty (node can only observe its local view times, but not the committee’s collective swarm behaviour). + +![](/docs/CruiseControl_BlockTimeController/PID_controller_for_block-rate-delay.png) + +After a disturbance, we want the controller to drive the system back to a state, where it can closely follow the ideal behaviour from there on. + +- Simulations have shown that this approach produces *very* stable controller with the intended behaviour. + + **Controller driving $e := \gamma - \Gamma \rightarrow 0$** + - setting the differential term $K_d=0$, the controller responds as expected with damped oscillatory behaviour + to a singular strong disturbance. Setting $K_d=3$ suppresses oscillations and the controller's performance improves as it responds more effectively. + + ![](/docs/CruiseControl_BlockTimeController/EpochSimulation_029.png) + ![](/docs/CruiseControl_BlockTimeController/EpochSimulation_030.png) + + - controller very quickly compensates for moderate disturbances and observational noise in a well-behaved system: + + ![](/docs/CruiseControl_BlockTimeController/EpochSimulation_028.png) + + - controller compensates massive anomaly (100s network partition) effectively: + + ![](/docs/CruiseControl_BlockTimeController/EpochSimulation_000.png) + + - controller effectively stabilizes system with continued larger disturbances (20% of offline consensus participants) and notable observational noise: + + ![](/docs/CruiseControl_BlockTimeController/EpochSimulation_005-0.png) + + **References:** + + - statistical model for happy-path view durations: [ID controller for ``block-rate-delay``](https://www.notion.so/ID-controller-for-block-rate-delay-cc9c2d9785ac4708a37bb952557b5ef4?pvs=21) + - For Python implementation with additional disturbances (offline nodes) and observational noise, see GitHub repo: [flow-internal/analyses/pacemaker_timing/2023-05_Blocktime_PID-controller](https://github.com/dapperlabs/flow-internal/tree/master/analyses/pacemaker_timing/2023-05_Blocktime_PID-controller) → [controller_tuning_v01.py](https://github.com/dapperlabs/flow-internal/blob/master/analyses/pacemaker_timing/2023-05_Blocktime_PID-controller/controller_tuning_v01.py) + +# Detailed PID controller specification + +Each consensus participant runs a local instance of the controller described below. Hence, all the quantities are based on the node’s local observation. + +## Definitions + +**Observables** (quantities provided to the node or directly measurable by the node): + +- $v$ is the node’s current view +- ideal view time $\tau_0$ is computed solely based on the Epoch configuration: +$\tau_0 := \frac{<{\rm total\ epoch\ time}>}{<{\rm total\ views\ in\ epoch}>}$ (for mainnet 22, Epoch 75, we have $\tau_0 \simeq$ 1250ms). +- $t[v]$ is the time the node entered view $v$ +- $F[v]$ is the final view of the current epoch +- $T[v]$ is the target end time of the current epoch + +**Derived quantities** + +- remaining views of the epoch $k[v] := F[v] +1 - v$ +- time remaining until the desired epoch switchover $\Gamma[v] := T[v]-t[v]$ +- error $e[v] := \underbrace{k\cdot\tau_0}_{\gamma[v]} - \Gamma[v] = t[v] + k\cdot\tau_0 - T[v]$ + +### Precise convention of View Timing + +Upon observing block `B` with view $v$, the controller updates its internal state. + +Note the '+1' term in the computation of the remaining views $k[v] := F[v] +1 - v$ . This is related to our convention that the epoch begins (happy path) when observing the first block of the epoch. Only by observing this block, the nodes transition to the first view of the epoch. Up to that point, the consensus replicas remain in the last view of the previous epoch, in the state of `having processed the last block of the old epoch and voted for it` (happy path). Replicas remain in this state until they see a confirmation of the view (either QC or TC for the last view of the previous epoch). + +![](/docs/CruiseControl_BlockTimeController/ViewDurationConvention.png) + +In accordance with this convention, observing the proposal for the last view of an epoch, marks the start of the last view. By observing the proposal, nodes enter the last view, verify the block, vote for it, the primary aggregates the votes, constructs the child (for first view of new epoch). The last view of the epoch ends, when the child proposal is published. + +### Controller + +The goal of the controller is to drive the system towards an error of zero, i.e. $e[v] \rightarrow 0$. For a [PID controller](https://en.wikipedia.org/wiki/PID_controller), the output $u$ for view $v$ has the form: + +```math +u[v] = K_p \cdot e[v]+K_i \cdot \mathcal{I}[v] + K_d \cdot \Delta[v] +``` + +With error terms (computed from observations) + +- $e[v]$ representing the *instantaneous* error as of view $v$ +(commonly referred to as ‘proportional term’) +- $\mathcal{I} [v] = \sum_v e[v]$ the sum of the errors +(commonly referred to as ‘integral term’) +- $\Delta[v]=e[v]-e[v-1]$ the rate of change of the error +(commonly referred to as ‘derivative term’) + +and controller parameters (values derived from controller tuning): + +- $K_p$ be the proportional coefficient +- $K_i$ be the integral coefficient +- $K_d$ be the derivative coefficient + +## Measuring view duration + +Each consensus participant observes the error $e[v]$ based on its local view evolution. As the following figure illustrates, the view duration is highly variable on small time scales. + +![](/docs/CruiseControl_BlockTimeController/ViewRate.png) + +Therefore, we expect $e[v]$ to be very variable. Furthermore, note that a node uses its local view transition times as an estimator for the collective behaviour of the entire committee. Therefore, there is also observational noise obfuscating the underlying collective behaviour. Hence, we expect notable noise. + +## Managing noise + +Noisy values for $e[v]$ also impact the derivative term $\Delta[v]$ and integral term $\mathcal{I}[v]$. This can impact the controller’s performance. + +### **Managing noise in the proportional term** + +An established approach for managing noise in observables is to use [exponentially weighted moving average [EWMA]](https://en.wikipedia.org/wiki/Moving_average) instead of the instantaneous values. Specifically, let $\bar{e}[v]$ denote the EWMA of the instantaneous error, which is computed as follows: + +```math +\eqalign{ +\textnormal{initialization: }\quad \bar{e} :&= 0 \\ +\textnormal{update with instantaneous error\ } e[v]:\quad \bar{e}[v] &= \alpha \cdot e[v] + (1-\alpha)\cdot \bar{e}[v-1] +} +``` + +The parameter $\alpha$ relates to the averaging time window. Let $\alpha \equiv \frac{1}{N_\textnormal{ewma}}$ and consider that the input changes from $x_\textnormal{old}$ to $x_\textnormal{new}$ as a step function. Then $N_\textnormal{ewma}$ is the number of samples required to move the output average about 2/3 of the way from $x_\textnormal{old}$ to $x_\textnormal{new}$. + +see also [Python `Ewma` implementation](https://github.com/dapperlabs/flow-internal/blob/423d927421c073e4c3f66165d8f51b829925278f/analyses/pacemaker_timing/2023-05_Blocktime_PID-controller/controller_tuning_v01.py#L405-L431) + +### **Managing noise in the integral term** + +In particular systematic observation bias are a problem, as it leads to a diverging integral term. The commonly adopted approach is to use a ‘leaky integrator’ [[1](https://www.music.mcgill.ca/~gary/307/week2/node4.html), [2](https://engineering.stackexchange.com/questions/29833/limiting-the-integral-to-a-time-window-in-pid-controller)], which we denote as $\bar{\mathcal{I}}[v]$. + +```math +\eqalign{ +\textnormal{initialization: }\quad \bar{\mathcal{I}} :&= 0 \\ +\textnormal{update with instantaneous error\ } e[v]:\quad \bar{\mathcal{I}}[v] &= e[v] + (1-\beta)\cdot\bar{\mathcal{I}}[v-1] +} +``` + +Intuitively, the loss factor $\beta$ relates to the time window of the integrator. A factor of 0 means an infinite time horizon, while $\beta =1$ makes the integrator only memorize the last input. Let $\beta \equiv \frac{1}{N_\textnormal{itg}}$ and consider a constant input value $x$. Then $N_\textnormal{itg}$ relates to the number of past samples that the integrator remembers: + +- the integrators output will saturate at $x\cdot N_\textnormal{itg}$ +- an integrator initialized with 0, reaches 2/3 of the saturation value $x\cdot N_\textnormal{itg}$ after consuming $N_\textnormal{itg}$ inputs + +see also [Python `LeakyIntegrator` implementation](https://github.com/dapperlabs/flow-internal/blob/423d927421c073e4c3f66165d8f51b829925278f/analyses/pacemaker_timing/2023-05_Blocktime_PID-controller/controller_tuning_v01.py#L444-L468) + +### **Managing noise in the derivative term** + +Similarly to the proportional term, we apply an EWMA to the differential term and denote the averaged value as $\bar{\Delta}[v]$: + +```math +\eqalign{ +\textnormal{initialization: }\quad \bar{\Delta} :&= 0 \\ +\textnormal{update with instantaneous error\ } e[v]:\quad \bar{\Delta}[v] &= \bar{e}[v] - \bar{e}[v-1] +} +``` + +## Final formula for PID controller + +We have used a statistical model of the view duration extracted from mainnet 22 (Epoch 75) and manually added disturbances and observational noise and systemic observational bias. + +The following parameters have proven to generate stable controller behaviour over a large variety of network conditions: + +--- +👉 The controller is given by + +```math +u[v] = K_p \cdot \bar{e}[v]+K_i \cdot \bar{\mathcal{I}}[v] + K_d \cdot \bar{\Delta}[v] +``` + +with parameters: + +- $K_p = 2.0$ +- $K_i = 0.6$ +- $K_d = 3.0$ +- $N_\textnormal{ewma} = 5$, i.e. $\alpha = \frac{1}{N_\textnormal{ewma}} = 0.2$ +- $N_\textnormal{itg} = 50$, i.e. $\beta = \frac{1}{N_\textnormal{itg}} = 0.02$ + +The controller output $u[v]$ represents the amount of time by which the controller wishes to deviate from the ideal view duration $\tau_0$. In other words, the duration of view $v$ that the controller wants to set is +```math +\widehat{\tau}[v] = \tau_0 - u[v] +``` +--- + + +For further details about + +- the statistical model of the view duration, see [ID controller for ``block-rate-delay``](https://www.notion.so/ID-controller-for-block-rate-delay-cc9c2d9785ac4708a37bb952557b5ef4?pvs=21) +- the simulation and controller tuning, see [flow-internal/analyses/pacemaker_timing/2023-05_Blocktime_PID-controller](https://github.com/dapperlabs/flow-internal/tree/master/analyses/pacemaker_timing/2023-05_Blocktime_PID-controller) → [controller_tuning_v01.py](https://github.com/dapperlabs/flow-internal/blob/master/analyses/pacemaker_timing/2023-05_Blocktime_PID-controller/controller_tuning_v01.py) + +### Limits of authority + +In general, there is no bound on the output of the controller output $u$. However, it is important to limit the controller’s influence to keep $u$ within a sensible range. + +- upper bound on view duration $\widehat{\tau}[v]$ that we allow the controller to set: + + The current timeout threshold is set to 2.5s. Therefore, the largest view duration we want to allow the controller to set is 1.6s. + Thereby, approx. 900ms remain for message propagation, voting and constructing the child block, which will prevent the controller to drive the node into timeout with high probability. + +- lower bound on the view duration: + + Let $t_\textnormal{p}[v]$ denote the time when the primary for view $v$ has constructed its block proposal. + The time difference $t_\textnormal{p}[v] - t[v]$ between the primary entering the view and having its proposal + ready is the minimally required time to execute the protocol. The controller can only *delay* broadcasting the block, + but it cannot release the block before $t_\textnormal{p}[v]$ simply because the proposal isn’t ready any earlier. + + + +👉 Let $\hat{t}[v]$ denote the time when the primary for view $v$ *broadcasts* its proposal. We assign: + +```math +\hat{t}[v] := \max\big(t[v] +\min(\widehat{\tau}[v],\ 2\textnormal{s}),\ t_\textnormal{p}[v]\big) +``` + + + +## Edge Cases + +### A node is catching up + +When a node is catching up, it processes blocks more quickly than when it is up-to-date, and therefore observes a faster view rate. This would cause the node’s `BlockRateManager` to compensate by increasing the block rate delay. + +As long as delay function is responsive, it doesn’t have a practical impact, because nodes catching up don’t propose anyway. + +To the extent the delay function is not responsive, this would cause the block rate to slow down slightly, when the node is caught up. + +**Assumption:** as we assume that only a smaller fraction of nodes go offline, the effect is expected to be small and easily compensated for by the supermajority of online nodes. + +### A node has a misconfigured clock + +Cap the maximum deviation from the default delay (limits the general impact of error introduced by the `BlockTimeController`). The node with misconfigured clock will contribute to the error in a limited way, but as long as the majority of nodes have an accurate clock, they will offset this error. + +**Assumption:** few enough nodes will have a misconfigured clock, that the effect will be small enough to be easily compensated for by the supermajority of correct nodes. + +### Near epoch boundaries + +We might incorrectly compute high error in the target view rate, if local current view and current epoch are not exactly synchronized. By default, they would not be, because `EpochTransition` events occur upon finalization, and current view is updated as soon as QC/TC is available. + +**Solution:** determine epoch locally based on view only, do not use `EpochTransition` event. + +### EECC + +We need to detect EECC and revert to a default block-rate-delay (stop adjusting). + +## Testing + +[Cruise Control: Benchnet Testing Notes](https://www.notion.so/Cruise-Control-Benchnet-Testing-Notes-ea08f49ba9d24ce2a158fca9358966df?pvs=21) diff --git a/consensus/hotstuff/cruisectl/aggregators.go b/consensus/hotstuff/cruisectl/aggregators.go new file mode 100644 index 00000000000..4ea7cd7437c --- /dev/null +++ b/consensus/hotstuff/cruisectl/aggregators.go @@ -0,0 +1,130 @@ +package cruisectl + +import ( + "fmt" +) + +// Ewma implements the exponentially weighted moving average with smoothing factor α. +// The Ewma is a filter commonly applied to time-discrete signals. Mathematically, +// it is represented by the recursive update formula +// +// value ← α·v + (1-α)·value +// +// where `v` the next observation. Intuitively, the loss factor `α` relates to the +// time window of N observations that we average over. For example, let +// α ≡ 1/N and consider an input that suddenly changes from x to y as a step +// function. Then N is _roughly_ the number of samples required to move the output +// average about 2/3 of the way from x to y. +// For numeric stability, we require α to satisfy 0 < a < 1. +// Not concurrency safe. +type Ewma struct { + alpha float64 + value float64 +} + +// NewEwma instantiates a new exponentially weighted moving average. +// The smoothing factor `alpha` relates to the averaging time window. Let `alpha` ≡ 1/N and +// consider an input that suddenly changes from x to y as a step function. Then N is roughly +// the number of samples required to move the output average about 2/3 of the way from x to y. +// For numeric stability, we require `alpha` to satisfy 0 < `alpha` < 1. +func NewEwma(alpha, initialValue float64) (Ewma, error) { + if (alpha <= 0) || (1 <= alpha) { + return Ewma{}, fmt.Errorf("for numeric stability, we require the smoothing factor to satisfy 0 < alpha < 1") + } + return Ewma{ + alpha: alpha, + value: initialValue, + }, nil +} + +// AddRepeatedObservation adds k consecutive observations with the same value v. Returns the updated value. +func (e *Ewma) AddRepeatedObservation(v float64, k int) float64 { + // closed from for k consecutive updates with the same observation v: + // value ← r·value + v·(1-r) with r := (1-α)^k + r := powWithIntegerExponent(1.0-e.alpha, k) + e.value = r*e.value + v*(1.0-r) + return e.value +} + +// AddObservation adds the value `v` to the EWMA. Returns the updated value. +func (e *Ewma) AddObservation(v float64) float64 { + // Update formula: value ← α·v + (1-α)·value = value + α·(v - value) + e.value = e.value + e.alpha*(v-e.value) + return e.value +} + +func (e *Ewma) Value() float64 { + return e.value +} + +// LeakyIntegrator is a filter commonly applied to time-discrete signals. +// Intuitively, it sums values over a limited time window. This implementation is +// parameterized by the loss factor `ß`: +// +// value ← v + (1-ß)·value +// +// where `v` the next observation. Intuitively, the loss factor `ß` relates to the +// time window of N observations that we integrate over. For example, let ß ≡ 1/N +// and consider a constant input x: +// - the integrator value will saturate at x·N +// - an integrator initialized at 0 reaches 2/3 of the saturation value after N samples +// +// For numeric stability, we require ß to satisfy 0 < ß < 1. +// Further details on Leaky Integrator: https://www.music.mcgill.ca/~gary/307/week2/node4.html +// Not concurrency safe. +type LeakyIntegrator struct { + feedbackCoef float64 // feedback coefficient := (1-ß) + value float64 +} + +// NewLeakyIntegrator instantiates a new leaky integrator with loss factor `beta`, where +// `beta relates to window of N observations that we integrate over. For example, let +// `beta` ≡ 1/N and consider a constant input x. The integrator value will saturate at x·N. +// An integrator initialized at 0 reaches 2/3 of the saturation value after N samples. +// For numeric stability, we require `beta` to satisfy 0 < `beta` < 1. +func NewLeakyIntegrator(beta, initialValue float64) (LeakyIntegrator, error) { + if (beta <= 0) || (1 <= beta) { + return LeakyIntegrator{}, fmt.Errorf("for numeric stability, we require the loss factor to satisfy 0 < beta < 1") + } + return LeakyIntegrator{ + feedbackCoef: 1.0 - beta, + value: initialValue, + }, nil +} + +// AddRepeatedObservation adds k consecutive observations with the same value v. Returns the updated value. +func (e *LeakyIntegrator) AddRepeatedObservation(v float64, k int) float64 { + // closed from for k consecutive updates with the same observation v: + // value ← r·value + v·(1-r) with r := α^k + r := powWithIntegerExponent(e.feedbackCoef, k) + e.value = r*e.value + v*(1.0-r)/(1.0-e.feedbackCoef) + return e.value +} + +// AddObservation adds the value `v` to the LeakyIntegrator. Returns the updated value. +func (e *LeakyIntegrator) AddObservation(v float64) float64 { + // Update formula: value ← v + feedbackCoef·value + // where feedbackCoef = (1-beta) + e.value = v + e.feedbackCoef*e.value + return e.value +} + +func (e *LeakyIntegrator) Value() float64 { + return e.value +} + +// powWithIntegerExponent implements exponentiation b^k optimized for integer k >=1 +func powWithIntegerExponent(b float64, k int) float64 { + r := 1.0 + for { + if k&1 == 1 { + r *= b + } + k >>= 1 + if k == 0 { + break + } + b *= b + } + return r +} diff --git a/consensus/hotstuff/cruisectl/aggregators_test.go b/consensus/hotstuff/cruisectl/aggregators_test.go new file mode 100644 index 00000000000..d508290c814 --- /dev/null +++ b/consensus/hotstuff/cruisectl/aggregators_test.go @@ -0,0 +1,167 @@ +package cruisectl + +import ( + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +// Test_Instantiation verifies successful instantiation of Ewma +func Test_EWMA_Instantiation(t *testing.T) { + w, err := NewEwma(0.5, 17.2) + require.NoError(t, err) + require.Equal(t, 17.2, w.Value()) +} + +// Test_EnforceNumericalBounds verifies that constructor only accepts +// alpha values that satisfy 0 < alpha < 1 +func Test_EWMA_EnforceNumericalBounds(t *testing.T) { + for _, alpha := range []float64{-1, 0, 1, 2} { + _, err := NewEwma(alpha, 17.2) + require.Error(t, err) + } +} + +// Test_AddingObservations verifies correct numerics when adding a single value. +// Reference values were generated via python +func Test_EWMA_AddingObservations(t *testing.T) { + alpha := math.Pi / 7.0 + initialValue := 17.0 + w, err := NewEwma(alpha, initialValue) + require.NoError(t, err) + + v := w.AddObservation(6.0) + require.InEpsilon(t, 12.063211544358897, v, 1e-12) + require.InEpsilon(t, 12.063211544358897, w.Value(), 1e-12) + v = w.AddObservation(-1.16) + require.InEpsilon(t, 6.128648080841518, v, 1e-12) + require.InEpsilon(t, 6.128648080841518, w.Value(), 1e-12) + v = w.AddObservation(1.23) + require.InEpsilon(t, 3.9301399632281675, v, 1e-12) + require.InEpsilon(t, 3.9301399632281675, w.Value(), 1e-12) +} + +// Test_AddingRepeatedObservations verifies correct numerics when repeated observations. +// Reference values were generated via python +func Test_EWMA_AddingRepeatedObservations(t *testing.T) { + alpha := math.Pi / 7.0 + initialValue := 17.0 + w, err := NewEwma(alpha, initialValue) + require.NoError(t, err) + + v := w.AddRepeatedObservation(6.0, 11) + require.InEpsilon(t, 6.015696509200239, v, 1e-12) + require.InEpsilon(t, 6.015696509200239, w.Value(), 1e-12) + v = w.AddRepeatedObservation(-1.16, 4) + require.InEpsilon(t, -0.49762458373978324, v, 1e-12) + require.InEpsilon(t, -0.49762458373978324, w.Value(), 1e-12) + v = w.AddRepeatedObservation(1.23, 1) + require.InEpsilon(t, 0.27773151632279214, v, 1e-12) + require.InEpsilon(t, 0.27773151632279214, w.Value(), 1e-12) +} + +// Test_AddingRepeatedObservations_selfConsistency applies a self-consistency check +// for repeated observations. +func Test_EWMA_AddingRepeatedObservations_selfConsistency(t *testing.T) { + alpha := math.Pi / 7.0 + initialValue := 17.0 + w1, err := NewEwma(alpha, initialValue) + require.NoError(t, err) + w2, err := NewEwma(alpha, initialValue) + require.NoError(t, err) + + for i := 7; i > 0; i-- { + w1.AddObservation(6.0) + } + v := w2.AddRepeatedObservation(6.0, 7) + require.InEpsilon(t, w1.Value(), v, 1e-12) + require.InEpsilon(t, w1.Value(), w2.Value(), 1e-12) + + for i := 4; i > 0; i-- { + w2.AddObservation(6.0) + } + v = w1.AddRepeatedObservation(6.0, 4) + require.InEpsilon(t, w2.Value(), v, 1e-12) + require.InEpsilon(t, w2.Value(), w1.Value(), 1e-12) +} + +// Test_LI_Instantiation verifies successful instantiation of LeakyIntegrator +func Test_LI_Instantiation(t *testing.T) { + li, err := NewLeakyIntegrator(0.5, 17.2) + require.NoError(t, err) + require.Equal(t, 17.2, li.Value()) +} + +// Test_EnforceNumericalBounds verifies that constructor only accepts +// alpha values that satisfy 0 < alpha < 1 +func Test_LI_EnforceNumericalBounds(t *testing.T) { + for _, beta := range []float64{-1, 0, 1, 2} { + _, err := NewLeakyIntegrator(beta, 17.2) + require.Error(t, err) + } +} + +// Test_AddingObservations verifies correct numerics when adding a single value. +// Reference values were generated via python +func Test_LI_AddingObservations(t *testing.T) { + beta := math.Pi / 7.0 + initialValue := 17.0 + li, err := NewLeakyIntegrator(beta, initialValue) + require.NoError(t, err) + + v := li.AddObservation(6.0) + require.InEpsilon(t, 15.370417841281931, v, 1e-12) + require.InEpsilon(t, 15.370417841281931, li.Value(), 1e-12) + v = li.AddObservation(-1.16) + require.InEpsilon(t, 7.312190445170959, v, 1e-12) + require.InEpsilon(t, 7.312190445170959, li.Value(), 1e-12) + v = li.AddObservation(1.23) + require.InEpsilon(t, 5.260487047428308, v, 1e-12) + require.InEpsilon(t, 5.260487047428308, li.Value(), 1e-12) +} + +// Test_AddingRepeatedObservations verifies correct numerics when repeated observations. +// Reference values were generated via python +func Test_LI_AddingRepeatedObservations(t *testing.T) { + beta := math.Pi / 7.0 + initialValue := 17.0 + li, err := NewLeakyIntegrator(beta, initialValue) + require.NoError(t, err) + + v := li.AddRepeatedObservation(6.0, 11) + require.InEpsilon(t, 13.374196472992809, v, 1e-12) + require.InEpsilon(t, 13.374196472992809, li.Value(), 1e-12) + v = li.AddRepeatedObservation(-1.16, 4) + require.InEpsilon(t, -1.1115419303895382, v, 1e-12) + require.InEpsilon(t, -1.1115419303895382, li.Value(), 1e-12) + v = li.AddRepeatedObservation(1.23, 1) + require.InEpsilon(t, 0.617316921420289, v, 1e-12) + require.InEpsilon(t, 0.617316921420289, li.Value(), 1e-12) + +} + +// Test_AddingRepeatedObservations_selfConsistency applies a self-consistency check +// for repeated observations. +func Test_LI_AddingRepeatedObservations_selfConsistency(t *testing.T) { + beta := math.Pi / 7.0 + initialValue := 17.0 + li1, err := NewLeakyIntegrator(beta, initialValue) + require.NoError(t, err) + li2, err := NewLeakyIntegrator(beta, initialValue) + require.NoError(t, err) + + for i := 7; i > 0; i-- { + li1.AddObservation(6.0) + } + v := li2.AddRepeatedObservation(6.0, 7) + require.InEpsilon(t, li1.Value(), v, 1e-12) + require.InEpsilon(t, li1.Value(), li2.Value(), 1e-12) + + for i := 4; i > 0; i-- { + li2.AddObservation(6.0) + } + v = li1.AddRepeatedObservation(6.0, 4) + require.InEpsilon(t, li2.Value(), v, 1e-12) + require.InEpsilon(t, li2.Value(), li1.Value(), 1e-12) +} diff --git a/consensus/hotstuff/cruisectl/block_time_controller.go b/consensus/hotstuff/cruisectl/block_time_controller.go new file mode 100644 index 00000000000..0748e8ec760 --- /dev/null +++ b/consensus/hotstuff/cruisectl/block_time_controller.go @@ -0,0 +1,462 @@ +// Package cruisectl implements a "cruise control" system for Flow by adjusting +// nodes' latest ProposalTiming in response to changes in the measured view rate and +// target epoch switchover time. +// +// It uses a PID controller with the projected epoch switchover time as the process +// variable and the set-point computed using epoch length config. The error is +// the difference between the projected epoch switchover time, assuming an +// ideal view time τ, and the target epoch switchover time (based on a schedule). +package cruisectl + +import ( + "fmt" + "time" + + "github.com/rs/zerolog" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/events" +) + +// TimedBlock represents a block, with a timestamp recording when the BlockTimeController received the block +type TimedBlock struct { + Block *model.Block + TimeObserved time.Time // timestamp when BlockTimeController received the block, per convention in UTC +} + +// epochInfo stores data about the current and next epoch. It is updated when we enter +// the first view of a new epoch, or the EpochSetup phase of the current epoch. +type epochInfo struct { + curEpochFirstView uint64 + curEpochFinalView uint64 // F[v] - the final view of the epoch + curEpochTargetEndTime time.Time // T[v] - the target end time of the current epoch + nextEpochFinalView *uint64 +} + +// targetViewTime returns τ[v], the ideal, steady-state view time for the current epoch. +// For numerical stability, we avoid repetitive conversions between seconds and time.Duration. +// Instead, internally within the controller, we work with float64 in units of seconds. +func (epoch *epochInfo) targetViewTime() float64 { + return epochLength.Seconds() / float64(epoch.curEpochFinalView-epoch.curEpochFirstView+1) +} + +// fractionComplete returns the percentage of views completed of the epoch for the given curView. +// curView must be within the range [curEpochFirstView, curEpochFinalView] +// Returns the completion percentage as a float between [0, 1] +func (epoch *epochInfo) fractionComplete(curView uint64) float64 { + return float64(curView-epoch.curEpochFirstView) / float64(epoch.curEpochFinalView-epoch.curEpochFirstView) +} + +// BlockTimeController dynamically adjusts the ProposalTiming of this node, +// based on the measured view rate of the consensus committee as a whole, in +// order to achieve a desired switchover time for each epoch. +// In a nutshell, the controller outputs the block time on the happy path, i.e. +// - Suppose the node is observing the parent block B0 at some time `x0`. +// - The controller determines the duration `d` of how much later the child block B1 +// should be observed by the committee. +// - The controller internally memorizes the latest B0 it has seen and outputs +// the tuple `(B0, x0, d)` +// +// This low-level controller output `(B0, x0, d)` is wrapped into a `ProposalTiming` +// interface, specifically `happyPathBlockTime` on the happy path. The purpose of the +// `ProposalTiming` wrapper is to translate the raw controller output into a form +// that is useful for the event handler. Edge cases, such as initialization or +// EECC are implemented by other implementations of `ProposalTiming`. +type BlockTimeController struct { + component.Component + protocol.Consumer // consumes protocol state events + + config *Config + + state protocol.State + log zerolog.Logger + metrics module.CruiseCtlMetrics + + epochInfo // scheduled transition view for current/next epoch + epochFallbackTriggered bool + + incorporatedBlocks chan TimedBlock // OnBlockIncorporated events, we desire these blocks to be processed in a timely manner and therefore use a small channel capacity + epochSetups chan *flow.Header // EpochSetupPhaseStarted events (block header within setup phase) + epochFallbacks chan struct{} // EpochFallbackTriggered events + + proportionalErr Ewma + integralErr LeakyIntegrator + + // latestProposalTiming holds the ProposalTiming that the controller generated in response to processing the latest observation + latestProposalTiming *atomic.Pointer[ProposalTiming] +} + +var _ hotstuff.ProposalDurationProvider = (*BlockTimeController)(nil) +var _ protocol.Consumer = (*BlockTimeController)(nil) +var _ component.Component = (*BlockTimeController)(nil) + +// NewBlockTimeController returns a new BlockTimeController. +func NewBlockTimeController(log zerolog.Logger, metrics module.CruiseCtlMetrics, config *Config, state protocol.State, curView uint64) (*BlockTimeController, error) { + // Initial error must be 0 unless we are making assumptions of the prior history of the proportional error `e[v]` + initProptlErr, initItgErr, initDrivErr := .0, .0, .0 + proportionalErr, err := NewEwma(config.alpha(), initProptlErr) + if err != nil { + return nil, fmt.Errorf("failed to initialize EWMA for computing the proportional error: %w", err) + } + integralErr, err := NewLeakyIntegrator(config.beta(), initItgErr) + if err != nil { + return nil, fmt.Errorf("failed to initialize LeakyIntegrator for computing the integral error: %w", err) + } + + ctl := &BlockTimeController{ + Consumer: events.NewNoop(), + config: config, + log: log.With().Str("hotstuff", "cruise_ctl").Logger(), + metrics: metrics, + state: state, + incorporatedBlocks: make(chan TimedBlock, 3), + epochSetups: make(chan *flow.Header, 5), + epochFallbacks: make(chan struct{}, 5), + proportionalErr: proportionalErr, + integralErr: integralErr, + latestProposalTiming: atomic.NewPointer[ProposalTiming](nil), // set in initProposalTiming + } + ctl.Component = component.NewComponentManagerBuilder(). + AddWorker(ctl.processEventsWorkerLogic). + Build() + + // initialize state + err = ctl.initEpochInfo(curView) + if err != nil { + return nil, fmt.Errorf("could not initialize epoch info: %w", err) + } + ctl.initProposalTiming(curView) + + ctl.log.Debug(). + Uint64("view", curView). + Msg("initialized BlockTimeController") + ctl.metrics.PIDError(initProptlErr, initItgErr, initDrivErr) + ctl.metrics.ControllerOutput(0) + ctl.metrics.TargetProposalDuration(0) + + return ctl, nil +} + +// initEpochInfo initializes the epochInfo state upon component startup. +// No errors are expected during normal operation. +func (ctl *BlockTimeController) initEpochInfo(curView uint64) error { + finalSnapshot := ctl.state.Final() + curEpoch := finalSnapshot.Epochs().Current() + + curEpochFirstView, err := curEpoch.FirstView() + if err != nil { + return fmt.Errorf("could not initialize current epoch first view: %w", err) + } + ctl.curEpochFirstView = curEpochFirstView + + curEpochFinalView, err := curEpoch.FinalView() + if err != nil { + return fmt.Errorf("could not initialize current epoch final view: %w", err) + } + ctl.curEpochFinalView = curEpochFinalView + + phase, err := finalSnapshot.Phase() + if err != nil { + return fmt.Errorf("could not check snapshot phase: %w", err) + } + if phase > flow.EpochPhaseStaking { + nextEpochFinalView, err := finalSnapshot.Epochs().Next().FinalView() + if err != nil { + return fmt.Errorf("could not initialize next epoch final view: %w", err) + } + ctl.epochInfo.nextEpochFinalView = &nextEpochFinalView + } + + ctl.curEpochTargetEndTime = ctl.config.TargetTransition.inferTargetEndTime(time.Now().UTC(), ctl.epochInfo.fractionComplete(curView)) + + epochFallbackTriggered, err := ctl.state.Params().EpochFallbackTriggered() + if err != nil { + return fmt.Errorf("could not check epoch fallback: %w", err) + } + ctl.epochFallbackTriggered = epochFallbackTriggered + + return nil +} + +// initProposalTiming initializes the ProposalTiming value upon startup. +// CAUTION: Must be called after initEpochInfo. +func (ctl *BlockTimeController) initProposalTiming(curView uint64) { + // When disabled, or in epoch fallback, use fallback timing (constant ProposalDuration) + if ctl.epochFallbackTriggered || !ctl.config.Enabled.Load() { + ctl.storeProposalTiming(newFallbackTiming(curView, time.Now().UTC(), ctl.config.FallbackProposalDelay.Load())) + return + } + // Otherwise, before we observe any view changes, publish blocks immediately + ctl.storeProposalTiming(newPublishImmediately(curView, time.Now().UTC())) +} + +// storeProposalTiming stores the latest ProposalTiming +// Concurrency safe. +func (ctl *BlockTimeController) storeProposalTiming(proposalTiming ProposalTiming) { + ctl.latestProposalTiming.Store(&proposalTiming) +} + +// GetProposalTiming returns the controller's latest ProposalTiming. Concurrency safe. +func (ctl *BlockTimeController) GetProposalTiming() ProposalTiming { + pt := ctl.latestProposalTiming.Load() + if pt == nil { // should never happen, as we always store non-nil instances of ProposalTiming. Though, this extra check makes `GetProposalTiming` universal. + return nil + } + return *pt +} + +func (ctl *BlockTimeController) TargetPublicationTime(proposalView uint64, timeViewEntered time.Time, parentBlockId flow.Identifier) time.Time { + return ctl.GetProposalTiming().TargetPublicationTime(proposalView, timeViewEntered, parentBlockId) +} + +// processEventsWorkerLogic is the logic for processing events received from other components. +// This method should be executed by a dedicated worker routine (not concurrency safe). +func (ctl *BlockTimeController) processEventsWorkerLogic(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + done := ctx.Done() + for { + + // Priority 1: EpochSetup + select { + case block := <-ctl.epochSetups: + snapshot := ctl.state.AtHeight(block.Height) + err := ctl.processEpochSetupPhaseStarted(snapshot) + if err != nil { + ctl.log.Err(err).Msgf("fatal error handling EpochSetupPhaseStarted event") + ctx.Throw(err) + return + } + default: + } + + // Priority 2: EpochFallbackTriggered + select { + case <-ctl.epochFallbacks: + err := ctl.processEpochFallbackTriggered() + if err != nil { + ctl.log.Err(err).Msgf("fatal error processing epoch EECC event") + ctx.Throw(err) + } + default: + } + + // Priority 3: OnBlockIncorporated + select { + case <-done: + return + case block := <-ctl.incorporatedBlocks: + err := ctl.processIncorporatedBlock(block) + if err != nil { + ctl.log.Err(err).Msgf("fatal error handling OnBlockIncorporated event") + ctx.Throw(err) + return + } + case block := <-ctl.epochSetups: + snapshot := ctl.state.AtHeight(block.Height) + err := ctl.processEpochSetupPhaseStarted(snapshot) + if err != nil { + ctl.log.Err(err).Msgf("fatal error handling EpochSetupPhaseStarted event") + ctx.Throw(err) + return + } + case <-ctl.epochFallbacks: + err := ctl.processEpochFallbackTriggered() + if err != nil { + ctl.log.Err(err).Msgf("fatal error processing epoch EECC event") + ctx.Throw(err) + return + } + } + } +} + +// processIncorporatedBlock processes `OnBlockIncorporated` events from HotStuff. +// Whenever the view changes, we: +// - updates epoch info, if this is the first observed view of a new epoch +// - compute error terms, compensation function output, and new ProposalTiming +// - compute a new projected epoch end time, assuming an ideal view rate +// +// No errors are expected during normal operation. +func (ctl *BlockTimeController) processIncorporatedBlock(tb TimedBlock) error { + // if epoch fallback is triggered, we always use fallbackProposalTiming + if ctl.epochFallbackTriggered { + return nil + } + + latest := ctl.GetProposalTiming() + if tb.Block.View <= latest.ObservationView() { // we don't care about older blocks that are incorporated into the protocol state + return nil + } + + err := ctl.checkForEpochTransition(tb) + if err != nil { + return fmt.Errorf("could not check for epoch transition: %w", err) + } + + err = ctl.measureViewDuration(tb) + if err != nil { + return fmt.Errorf("could not measure view rate: %w", err) + } + return nil +} + +// checkForEpochTransition updates the epochInfo to reflect an epoch transition if curView +// being entered causes a transition to the next epoch. Otherwise, this is a no-op. +// No errors are expected during normal operation. +func (ctl *BlockTimeController) checkForEpochTransition(tb TimedBlock) error { + view := tb.Block.View + if view <= ctl.curEpochFinalView { // prevalent case: we are still within the current epoch + return nil + } + + // sanity checks, since we are beyond the final view of the most recently processed epoch: + if ctl.nextEpochFinalView == nil { // final view of epoch we are entering should be known + return fmt.Errorf("cannot transition without nextEpochFinalView set") + } + if view > *ctl.nextEpochFinalView { // the block's view should be within the upcoming epoch + return fmt.Errorf("sanity check failed: curView %d is beyond both current epoch (final view %d) and next epoch (final view %d)", + view, ctl.curEpochFinalView, *ctl.nextEpochFinalView) + } + + ctl.curEpochFirstView = ctl.curEpochFinalView + 1 + ctl.curEpochFinalView = *ctl.nextEpochFinalView + ctl.nextEpochFinalView = nil + ctl.curEpochTargetEndTime = ctl.config.TargetTransition.inferTargetEndTime(tb.Block.Timestamp, ctl.epochInfo.fractionComplete(view)) + return nil +} + +// measureViewDuration computes a new measurement of projected epoch switchover time and error for the newly entered view. +// It updates the latest ProposalTiming based on the new error. +// No errors are expected during normal operation. +func (ctl *BlockTimeController) measureViewDuration(tb TimedBlock) error { + view := tb.Block.View + // if the controller is disabled, we don't update measurements and instead use a fallback timing + if !ctl.config.Enabled.Load() { + fallbackDelay := ctl.config.FallbackProposalDelay.Load() + ctl.storeProposalTiming(newFallbackTiming(view, tb.TimeObserved, fallbackDelay)) + ctl.log.Debug(). + Uint64("cur_view", view). + Dur("fallback_proposal_delay", fallbackDelay). + Msg("controller is disabled - using fallback timing") + return nil + } + + previousProposalTiming := ctl.GetProposalTiming() + previousPropErr := ctl.proportionalErr.Value() + + // Compute the projected time still needed for the remaining views, assuming that we progress through the remaining views with + // the idealized target view time. + // Note the '+1' term in the computation of `viewDurationsRemaining`. This is related to our convention that the epoch begins + // (happy path) when observing the first block of the epoch. Only by observing this block, the nodes transition to the first + // view of the epoch. Up to that point, the consensus replicas remain in the last view of the previous epoch, in the state of + // "having processed the last block of the old epoch and voted for it" (happy path). Replicas remain in this state until they + // see a confirmation of the view (either QC or TC for the last view of the previous epoch). + // In accordance with this convention, observing the proposal for the last view of an epoch, marks the start of the last view. + // By observing the proposal, nodes enter the last view, verify the block, vote for it, the primary aggregates the votes, + // constructs the child (for first view of new epoch). The last view of the epoch ends, when the child proposal is published. + tau := ctl.targetViewTime() // τ - idealized target view time in units of seconds + viewDurationsRemaining := ctl.curEpochFinalView + 1 - view // k[v] - views remaining in current epoch + durationRemaining := ctl.curEpochTargetEndTime.Sub(tb.TimeObserved) + + // Compute instantaneous error term: e[v] = k[v]·τ - T[v] i.e. the projected difference from target switchover + // and update PID controller's error terms. All UNITS in SECOND. + instErr := float64(viewDurationsRemaining)*tau - durationRemaining.Seconds() + propErr := ctl.proportionalErr.AddObservation(instErr) + itgErr := ctl.integralErr.AddObservation(instErr) + drivErr := propErr - previousPropErr + + // controller output u[v] in units of second + u := propErr*ctl.config.KP + itgErr*ctl.config.KI + drivErr*ctl.config.KD + + // compute the controller output for this observation + unconstrainedBlockTime := time.Duration((tau - u) * float64(time.Second)) // desired time between parent and child block, in units of seconds + proposalTiming := newHappyPathBlockTime(tb, unconstrainedBlockTime, ctl.config.TimingConfig) + constrainedBlockTime := proposalTiming.ConstrainedBlockTime() + + ctl.log.Debug(). + Uint64("last_observation", previousProposalTiming.ObservationView()). + Dur("duration_since_last_observation", tb.TimeObserved.Sub(previousProposalTiming.ObservationTime())). + Dur("projected_time_remaining", durationRemaining). + Uint64("view_durations_remaining", viewDurationsRemaining). + Float64("inst_err", instErr). + Float64("proportional_err", propErr). + Float64("integral_err", itgErr). + Float64("derivative_err", drivErr). + Dur("controller_output", time.Duration(u*float64(time.Second))). + Dur("unconstrained_block_time", unconstrainedBlockTime). + Dur("constrained_block_time", constrainedBlockTime). + Msg("measured error upon view change") + + ctl.metrics.PIDError(propErr, itgErr, drivErr) + ctl.metrics.ControllerOutput(time.Duration(u * float64(time.Second))) + ctl.metrics.TargetProposalDuration(proposalTiming.ConstrainedBlockTime()) + + ctl.storeProposalTiming(proposalTiming) + return nil +} + +// processEpochSetupPhaseStarted processes EpochSetupPhaseStarted events from the protocol state. +// Whenever we enter the EpochSetup phase, we: +// - store the next epoch's final view +// +// No errors are expected during normal operation. +func (ctl *BlockTimeController) processEpochSetupPhaseStarted(snapshot protocol.Snapshot) error { + if ctl.epochFallbackTriggered { + return nil + } + + nextEpoch := snapshot.Epochs().Next() + finalView, err := nextEpoch.FinalView() + if err != nil { + return fmt.Errorf("could not get next epochInfo final view: %w", err) + } + ctl.epochInfo.nextEpochFinalView = &finalView + return nil +} + +// processEpochFallbackTriggered processes EpochFallbackTriggered events from the protocol state. +// When epoch fallback mode is triggered, we: +// - set ProposalTiming to the default value +// - set epoch fallback triggered, to disable the controller +// +// No errors are expected during normal operation. +func (ctl *BlockTimeController) processEpochFallbackTriggered() error { + ctl.epochFallbackTriggered = true + latestFinalized, err := ctl.state.Final().Head() + if err != nil { + return fmt.Errorf("failed to retrieve latest finalized block from protocol state %w", err) + } + + ctl.storeProposalTiming(newFallbackTiming(latestFinalized.View, time.Now().UTC(), ctl.config.FallbackProposalDelay.Load())) + return nil +} + +// OnBlockIncorporated listens to notification from HotStuff about incorporating new blocks. +// The event is queued for async processing by the worker. If the channel is full, +// the event is discarded - since we are taking an average it doesn't matter if we +// occasionally miss a sample. +func (ctl *BlockTimeController) OnBlockIncorporated(block *model.Block) { + select { + case ctl.incorporatedBlocks <- TimedBlock{Block: block, TimeObserved: time.Now().UTC()}: + default: + } +} + +// EpochSetupPhaseStarted responds to the EpochSetup phase starting for the current epoch. +// The event is queued for async processing by the worker. +func (ctl *BlockTimeController) EpochSetupPhaseStarted(_ uint64, first *flow.Header) { + ctl.epochSetups <- first +} + +// EpochEmergencyFallbackTriggered responds to epoch fallback mode being triggered. +func (ctl *BlockTimeController) EpochEmergencyFallbackTriggered() { + ctl.epochFallbacks <- struct{}{} +} diff --git a/consensus/hotstuff/cruisectl/block_time_controller_test.go b/consensus/hotstuff/cruisectl/block_time_controller_test.go new file mode 100644 index 00000000000..d6cc074ab6b --- /dev/null +++ b/consensus/hotstuff/cruisectl/block_time_controller_test.go @@ -0,0 +1,676 @@ +package cruisectl + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + mockmodule "github.com/onflow/flow-go/module/mock" + mockprotocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/utils/unittest" + "github.com/onflow/flow-go/utils/unittest/mocks" +) + +// BlockTimeControllerSuite encapsulates tests for the BlockTimeController. +type BlockTimeControllerSuite struct { + suite.Suite + + initialView uint64 + epochCounter uint64 + curEpochFirstView uint64 + curEpochFinalView uint64 + epochFallbackTriggered bool + + metrics mockmodule.CruiseCtlMetrics + state mockprotocol.State + params mockprotocol.Params + snapshot mockprotocol.Snapshot + epochs mocks.EpochQuery + curEpoch mockprotocol.Epoch + + config *Config + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + ctl *BlockTimeController +} + +func TestBlockTimeController(t *testing.T) { + suite.Run(t, new(BlockTimeControllerSuite)) +} + +// SetupTest initializes mocks and default values. +func (bs *BlockTimeControllerSuite) SetupTest() { + bs.config = DefaultConfig() + bs.config.Enabled.Store(true) + bs.initialView = 0 + bs.epochCounter = uint64(0) + bs.curEpochFirstView = uint64(0) + bs.curEpochFinalView = uint64(604_800) // 1 view/sec + bs.epochFallbackTriggered = false + setupMocks(bs) +} + +func setupMocks(bs *BlockTimeControllerSuite) { + bs.metrics = *mockmodule.NewCruiseCtlMetrics(bs.T()) + bs.metrics.On("PIDError", mock.Anything, mock.Anything, mock.Anything).Maybe() + bs.metrics.On("TargetProposalDuration", mock.Anything).Maybe() + bs.metrics.On("ControllerOutput", mock.Anything).Maybe() + + bs.state = *mockprotocol.NewState(bs.T()) + bs.params = *mockprotocol.NewParams(bs.T()) + bs.snapshot = *mockprotocol.NewSnapshot(bs.T()) + bs.epochs = *mocks.NewEpochQuery(bs.T(), bs.epochCounter) + bs.curEpoch = *mockprotocol.NewEpoch(bs.T()) + + bs.state.On("Final").Return(&bs.snapshot) + bs.state.On("AtHeight", mock.Anything).Return(&bs.snapshot).Maybe() + bs.state.On("Params").Return(&bs.params) + bs.params.On("EpochFallbackTriggered").Return( + func() bool { return bs.epochFallbackTriggered }, + func() error { return nil }) + bs.snapshot.On("Phase").Return( + func() flow.EpochPhase { return bs.epochs.Phase() }, + func() error { return nil }) + bs.snapshot.On("Head").Return(unittest.BlockHeaderFixture(unittest.HeaderWithView(bs.initialView+11)), nil).Maybe() + bs.snapshot.On("Epochs").Return(&bs.epochs) + bs.curEpoch.On("Counter").Return(bs.epochCounter, nil) + bs.curEpoch.On("FirstView").Return(bs.curEpochFirstView, nil) + bs.curEpoch.On("FinalView").Return(bs.curEpochFinalView, nil) + bs.epochs.Add(&bs.curEpoch) + + bs.ctx, bs.cancel = irrecoverable.NewMockSignalerContextWithCancel(bs.T(), context.Background()) +} + +// CreateAndStartController creates and starts the BlockTimeController. +// Should be called only once per test case. +func (bs *BlockTimeControllerSuite) CreateAndStartController() { + ctl, err := NewBlockTimeController(unittest.Logger(), &bs.metrics, bs.config, &bs.state, bs.initialView) + require.NoError(bs.T(), err) + bs.ctl = ctl + bs.ctl.Start(bs.ctx) + unittest.RequireCloseBefore(bs.T(), bs.ctl.Ready(), time.Second, "component did not start") +} + +// StopController stops the BlockTimeController. +func (bs *BlockTimeControllerSuite) StopController() { + bs.cancel() + unittest.RequireCloseBefore(bs.T(), bs.ctl.Done(), time.Second, "component did not stop") +} + +// AssertCorrectInitialization checks that the controller is configured as expected after construction. +func (bs *BlockTimeControllerSuite) AssertCorrectInitialization() { + // at initialization, controller should be set up to release blocks without delay + controllerTiming := bs.ctl.GetProposalTiming() + now := time.Now().UTC() + + if bs.ctl.epochFallbackTriggered || !bs.ctl.config.Enabled.Load() { + // if epoch fallback is triggered or controller is disabled, use fallback timing + assert.Equal(bs.T(), now.Add(bs.ctl.config.FallbackProposalDelay.Load()), controllerTiming.TargetPublicationTime(7, now, unittest.IdentifierFixture())) + } else { + // otherwise should publish immediately + assert.Equal(bs.T(), now, controllerTiming.TargetPublicationTime(7, now, unittest.IdentifierFixture())) + } + if bs.ctl.epochFallbackTriggered { + return + } + + // should initialize epoch info + epoch := bs.ctl.epochInfo + expectedEndTime := bs.config.TargetTransition.inferTargetEndTime(time.Now(), epoch.fractionComplete(bs.initialView)) + assert.Equal(bs.T(), bs.curEpochFirstView, epoch.curEpochFirstView) + assert.Equal(bs.T(), bs.curEpochFinalView, epoch.curEpochFinalView) + assert.Equal(bs.T(), expectedEndTime, epoch.curEpochTargetEndTime) + + // if next epoch is set up, final view should be set + if phase := bs.epochs.Phase(); phase > flow.EpochPhaseStaking { + finalView, err := bs.epochs.Next().FinalView() + require.NoError(bs.T(), err) + assert.Equal(bs.T(), finalView, *epoch.nextEpochFinalView) + } else { + assert.Nil(bs.T(), epoch.nextEpochFinalView) + } + + // should create an initial measurement + assert.Equal(bs.T(), bs.initialView, controllerTiming.ObservationView()) + assert.WithinDuration(bs.T(), time.Now(), controllerTiming.ObservationTime(), time.Minute) + // errors should be initialized to zero + assert.Equal(bs.T(), float64(0), bs.ctl.proportionalErr.Value()) + assert.Equal(bs.T(), float64(0), bs.ctl.integralErr.Value()) +} + +// SanityCheckSubsequentMeasurements checks that two consecutive states of the BlockTimeController are different or equal and +// broadly reasonable. It does not assert exact values, because part of the measurements depend on timing in the worker. +func (bs *BlockTimeControllerSuite) SanityCheckSubsequentMeasurements(d1, d2 *controllerStateDigest, expectedEqual bool) { + if expectedEqual { + // later input should have left state invariant, including the Observation + assert.Equal(bs.T(), d1.latestProposalTiming.ObservationTime(), d2.latestProposalTiming.ObservationTime()) + assert.Equal(bs.T(), d1.latestProposalTiming.ObservationView(), d2.latestProposalTiming.ObservationView()) + // new measurement should have same error + assert.Equal(bs.T(), d1.proportionalErr.Value(), d2.proportionalErr.Value()) + assert.Equal(bs.T(), d1.integralErr.Value(), d2.integralErr.Value()) + } else { + // later input should have caused a new Observation to be recorded + assert.True(bs.T(), d1.latestProposalTiming.ObservationTime().Before(d2.latestProposalTiming.ObservationTime())) + // new measurement should have different error + assert.NotEqual(bs.T(), d1.proportionalErr.Value(), d2.proportionalErr.Value()) + assert.NotEqual(bs.T(), d1.integralErr.Value(), d2.integralErr.Value()) + } +} + +// PrintMeasurement prints the current state of the controller and the last measurement. +func (bs *BlockTimeControllerSuite) PrintMeasurement(parentBlockId flow.Identifier) { + ctl := bs.ctl + m := ctl.GetProposalTiming() + tpt := m.TargetPublicationTime(m.ObservationView()+1, m.ObservationTime(), parentBlockId) + fmt.Printf("v=%d\tt=%s\tPD=%s\te_N=%.3f\tI_M=%.3f\n", + m.ObservationView(), m.ObservationTime(), tpt.Sub(m.ObservationTime()), + ctl.proportionalErr.Value(), ctl.integralErr.Value()) +} + +// TestStartStop tests that the component can be started and stopped gracefully. +func (bs *BlockTimeControllerSuite) TestStartStop() { + bs.CreateAndStartController() + bs.StopController() +} + +// TestInit_EpochStakingPhase tests initializing the component in the EpochStaking phase. +// Measurement and epoch info should be initialized, next epoch final view should be nil. +func (bs *BlockTimeControllerSuite) TestInit_EpochStakingPhase() { + bs.CreateAndStartController() + defer bs.StopController() + bs.AssertCorrectInitialization() +} + +// TestInit_EpochStakingPhase tests initializing the component in the EpochSetup phase. +// Measurement and epoch info should be initialized, next epoch final view should be set. +func (bs *BlockTimeControllerSuite) TestInit_EpochSetupPhase() { + nextEpoch := mockprotocol.NewEpoch(bs.T()) + nextEpoch.On("Counter").Return(bs.epochCounter+1, nil) + nextEpoch.On("FinalView").Return(bs.curEpochFinalView+100_000, nil) + bs.epochs.Add(nextEpoch) + + bs.CreateAndStartController() + defer bs.StopController() + bs.AssertCorrectInitialization() +} + +// TestInit_EpochFallbackTriggered tests initializing the component when epoch fallback is triggered. +// Default GetProposalTiming should be set. +func (bs *BlockTimeControllerSuite) TestInit_EpochFallbackTriggered() { + bs.epochFallbackTriggered = true + bs.CreateAndStartController() + defer bs.StopController() + bs.AssertCorrectInitialization() +} + +// TestEpochFallbackTriggered tests epoch fallback: +// - the GetProposalTiming should revert to default +// - duplicate events should be no-ops +func (bs *BlockTimeControllerSuite) TestEpochFallbackTriggered() { + bs.CreateAndStartController() + defer bs.StopController() + + // update error so that GetProposalTiming is non-default + bs.ctl.proportionalErr.AddObservation(20.0) + bs.ctl.integralErr.AddObservation(20.0) + err := bs.ctl.measureViewDuration(makeTimedBlock(bs.initialView+1, unittest.IdentifierFixture(), time.Now())) + require.NoError(bs.T(), err) + assert.NotEqual(bs.T(), bs.config.FallbackProposalDelay, bs.ctl.GetProposalTiming()) + + // send the event + bs.ctl.EpochEmergencyFallbackTriggered() + // async: should revert to default GetProposalTiming + require.Eventually(bs.T(), func() bool { + now := time.Now().UTC() + return now.Add(bs.config.FallbackProposalDelay.Load()) == bs.ctl.GetProposalTiming().TargetPublicationTime(7, now, unittest.IdentifierFixture()) + }, time.Second, time.Millisecond) + + // additional EpochEmergencyFallbackTriggered events should be no-ops + // (send capacity+1 events to guarantee one is processed) + for i := 0; i <= cap(bs.ctl.epochFallbacks); i++ { + bs.ctl.EpochEmergencyFallbackTriggered() + } + // state should be unchanged + now := time.Now().UTC() + assert.Equal(bs.T(), now.Add(bs.config.FallbackProposalDelay.Load()), bs.ctl.GetProposalTiming().TargetPublicationTime(12, now, unittest.IdentifierFixture())) + + // additional OnBlockIncorporated events should be no-ops + for i := 0; i <= cap(bs.ctl.incorporatedBlocks); i++ { + header := unittest.BlockHeaderFixture(unittest.HeaderWithView(bs.initialView + 1 + uint64(i))) + header.ParentID = unittest.IdentifierFixture() + bs.ctl.OnBlockIncorporated(model.BlockFromFlow(header)) + } + // wait for the channel to drain, since OnBlockIncorporated doesn't block on sending + require.Eventually(bs.T(), func() bool { + return len(bs.ctl.incorporatedBlocks) == 0 + }, time.Second, time.Millisecond) + // state should be unchanged + now = time.Now().UTC() + assert.Equal(bs.T(), now.Add(bs.config.FallbackProposalDelay.Load()), bs.ctl.GetProposalTiming().TargetPublicationTime(17, now, unittest.IdentifierFixture())) +} + +// TestOnBlockIncorporated_UpdateProposalDelay tests that a new measurement is taken and +// GetProposalTiming updated upon receiving an OnBlockIncorporated event. +func (bs *BlockTimeControllerSuite) TestOnBlockIncorporated_UpdateProposalDelay() { + bs.CreateAndStartController() + defer bs.StopController() + + initialControllerState := captureControllerStateDigest(bs.ctl) // copy initial controller state + initialProposalDelay := bs.ctl.GetProposalTiming() + block := model.BlockFromFlow(unittest.BlockHeaderFixture(unittest.HeaderWithView(bs.initialView + 1))) + bs.ctl.OnBlockIncorporated(block) + require.Eventually(bs.T(), func() bool { + return bs.ctl.GetProposalTiming().ObservationView() > bs.initialView + }, time.Second, time.Millisecond) + nextControllerState := captureControllerStateDigest(bs.ctl) + nextProposalDelay := bs.ctl.GetProposalTiming() + + bs.SanityCheckSubsequentMeasurements(initialControllerState, nextControllerState, false) + // new measurement should update GetProposalTiming + now := time.Now().UTC() + assert.NotEqual(bs.T(), + initialProposalDelay.TargetPublicationTime(bs.initialView+2, now, unittest.IdentifierFixture()), + nextProposalDelay.TargetPublicationTime(bs.initialView+2, now, block.BlockID)) + + // duplicate events should be no-ops + for i := 0; i <= cap(bs.ctl.incorporatedBlocks); i++ { + bs.ctl.OnBlockIncorporated(block) + } + // wait for the channel to drain, since OnBlockIncorporated doesn't block on sending + require.Eventually(bs.T(), func() bool { + return len(bs.ctl.incorporatedBlocks) == 0 + }, time.Second, time.Millisecond) + + // state should be unchanged + finalControllerState := captureControllerStateDigest(bs.ctl) + bs.SanityCheckSubsequentMeasurements(nextControllerState, finalControllerState, true) + assert.Equal(bs.T(), nextProposalDelay, bs.ctl.GetProposalTiming()) +} + +// TestEnableDisable tests that the controller responds to enabling and disabling. +func (bs *BlockTimeControllerSuite) TestEnableDisable() { + // start in a disabled state + err := bs.config.SetEnabled(false) + require.NoError(bs.T(), err) + bs.CreateAndStartController() + defer bs.StopController() + + now := time.Now() + + initialControllerState := captureControllerStateDigest(bs.ctl) + initialProposalDelay := bs.ctl.GetProposalTiming() + // the initial proposal timing should use fallback timing + assert.Equal(bs.T(), now.Add(bs.ctl.config.FallbackProposalDelay.Load()), initialProposalDelay.TargetPublicationTime(bs.initialView+1, now, unittest.IdentifierFixture())) + + block := model.BlockFromFlow(unittest.BlockHeaderFixture(unittest.HeaderWithView(bs.initialView + 1))) + bs.ctl.OnBlockIncorporated(block) + require.Eventually(bs.T(), func() bool { + return bs.ctl.GetProposalTiming().ObservationView() > bs.initialView + }, time.Second, time.Millisecond) + secondProposalDelay := bs.ctl.GetProposalTiming() + + // new measurement should not change GetProposalTiming + assert.Equal(bs.T(), + initialProposalDelay.TargetPublicationTime(bs.initialView+2, now, unittest.IdentifierFixture()), + secondProposalDelay.TargetPublicationTime(bs.initialView+2, now, block.BlockID)) + + // now, enable the controller + err = bs.ctl.config.SetEnabled(true) + require.NoError(bs.T(), err) + + // send another block + block = model.BlockFromFlow(unittest.BlockHeaderFixture(unittest.HeaderWithView(bs.initialView + 2))) + bs.ctl.OnBlockIncorporated(block) + require.Eventually(bs.T(), func() bool { + return bs.ctl.GetProposalTiming().ObservationView() > bs.initialView + }, time.Second, time.Millisecond) + + thirdControllerState := captureControllerStateDigest(bs.ctl) + thirdProposalDelay := bs.ctl.GetProposalTiming() + + // new measurement should change GetProposalTiming + bs.SanityCheckSubsequentMeasurements(initialControllerState, thirdControllerState, false) + assert.NotEqual(bs.T(), + initialProposalDelay.TargetPublicationTime(bs.initialView+3, now, unittest.IdentifierFixture()), + thirdProposalDelay.TargetPublicationTime(bs.initialView+3, now, block.BlockID)) + +} + +// TestOnBlockIncorporated_EpochTransition_Enabled tests epoch transition with controller enabled. +func (bs *BlockTimeControllerSuite) TestOnBlockIncorporated_EpochTransition_Enabled() { + err := bs.ctl.config.SetEnabled(true) + require.NoError(bs.T(), err) + bs.testOnBlockIncorporated_EpochTransition() +} + +// TestOnBlockIncorporated_EpochTransition_Disabled tests epoch transition with controller disabled. +func (bs *BlockTimeControllerSuite) TestOnBlockIncorporated_EpochTransition_Disabled() { + err := bs.ctl.config.SetEnabled(false) + require.NoError(bs.T(), err) + bs.testOnBlockIncorporated_EpochTransition() +} + +// testOnBlockIncorporated_EpochTransition tests that a view change into the next epoch +// updates the local state to reflect the new epoch. +func (bs *BlockTimeControllerSuite) testOnBlockIncorporated_EpochTransition() { + nextEpoch := mockprotocol.NewEpoch(bs.T()) + nextEpoch.On("Counter").Return(bs.epochCounter+1, nil) + nextEpoch.On("FinalView").Return(bs.curEpochFinalView+100_000, nil) + bs.epochs.Add(nextEpoch) + bs.CreateAndStartController() + defer bs.StopController() + + initialControllerState := captureControllerStateDigest(bs.ctl) + bs.epochs.Transition() + timedBlock := makeTimedBlock(bs.curEpochFinalView+1, unittest.IdentifierFixture(), time.Now().UTC()) + err := bs.ctl.processIncorporatedBlock(timedBlock) + require.True(bs.T(), bs.ctl.GetProposalTiming().ObservationView() > bs.initialView) + require.NoError(bs.T(), err) + nextControllerState := captureControllerStateDigest(bs.ctl) + + bs.SanityCheckSubsequentMeasurements(initialControllerState, nextControllerState, false) + // epoch boundaries should be updated + assert.Equal(bs.T(), bs.curEpochFinalView+1, bs.ctl.epochInfo.curEpochFirstView) + assert.Equal(bs.T(), bs.ctl.epochInfo.curEpochFinalView, bs.curEpochFinalView+100_000) + assert.Nil(bs.T(), bs.ctl.nextEpochFinalView) +} + +// TestOnEpochSetupPhaseStarted ensures that the epoch info is updated when the next epoch is set up. +func (bs *BlockTimeControllerSuite) TestOnEpochSetupPhaseStarted() { + nextEpoch := mockprotocol.NewEpoch(bs.T()) + nextEpoch.On("Counter").Return(bs.epochCounter+1, nil) + nextEpoch.On("FinalView").Return(bs.curEpochFinalView+100_000, nil) + bs.epochs.Add(nextEpoch) + bs.CreateAndStartController() + defer bs.StopController() + + header := unittest.BlockHeaderFixture() + bs.ctl.EpochSetupPhaseStarted(bs.epochCounter, header) + require.Eventually(bs.T(), func() bool { + return bs.ctl.nextEpochFinalView != nil + }, time.Second, time.Millisecond) + + assert.Equal(bs.T(), bs.curEpochFinalView+100_000, *bs.ctl.nextEpochFinalView) + + // duplicate events should be no-ops + for i := 0; i <= cap(bs.ctl.epochSetups); i++ { + bs.ctl.EpochSetupPhaseStarted(bs.epochCounter, header) + } + assert.Equal(bs.T(), bs.curEpochFinalView+100_000, *bs.ctl.nextEpochFinalView) +} + +// TestProposalDelay_AfterTargetTransitionTime tests the behaviour of the controller +// when we have passed the target end time for the current epoch. +// We should approach the min GetProposalTiming (increase view rate as much as possible) +func (bs *BlockTimeControllerSuite) TestProposalDelay_AfterTargetTransitionTime() { + // we are near the end of the epoch in view terms + bs.initialView = uint64(float64(bs.curEpochFinalView) * .95) + bs.CreateAndStartController() + defer bs.StopController() + + lastProposalDelay := time.Hour // start with large dummy value + for view := bs.initialView + 1; view < bs.ctl.curEpochFinalView; view++ { + // we have passed the target end time of the epoch + receivedParentBlockAt := bs.ctl.curEpochTargetEndTime.Add(time.Duration(view) * time.Second) + timedBlock := makeTimedBlock(view, unittest.IdentifierFixture(), receivedParentBlockAt) + err := bs.ctl.measureViewDuration(timedBlock) + require.NoError(bs.T(), err) + + // compute proposal delay: + pubTime := bs.ctl.GetProposalTiming().TargetPublicationTime(view+1, time.Now().UTC(), timedBlock.Block.BlockID) // simulate building a child of `timedBlock` + delay := pubTime.Sub(receivedParentBlockAt) + + assert.LessOrEqual(bs.T(), delay, lastProposalDelay) + lastProposalDelay = delay + + // transition views until the end of the epoch, or for 100 views + if view-bs.initialView >= 100 { + break + } + } +} + +// TestProposalDelay_BehindSchedule tests the behaviour of the controller when the +// projected epoch switchover is LATER than the target switchover time, i.e. +// we are behind schedule. +// We should respond by lowering the GetProposalTiming (increasing view rate) +func (bs *BlockTimeControllerSuite) TestProposalDelay_BehindSchedule() { + // we are 50% of the way through the epoch in view terms + bs.initialView = uint64(float64(bs.curEpochFinalView) * .5) + bs.CreateAndStartController() + defer bs.StopController() + + lastProposalDelay := time.Hour // start with large dummy value + idealEnteredViewTime := bs.ctl.curEpochTargetEndTime.Add(-epochLength / 2) + + // 1s behind of schedule + receivedParentBlockAt := idealEnteredViewTime.Add(time.Second) + for view := bs.initialView + 1; view < bs.ctl.curEpochFinalView; view++ { + // hold the instantaneous error constant for each view + receivedParentBlockAt = receivedParentBlockAt.Add(seconds2Duration(bs.ctl.targetViewTime())) + timedBlock := makeTimedBlock(view, unittest.IdentifierFixture(), receivedParentBlockAt) + err := bs.ctl.measureViewDuration(timedBlock) + require.NoError(bs.T(), err) + + // compute proposal delay: + pubTime := bs.ctl.GetProposalTiming().TargetPublicationTime(view+1, time.Now().UTC(), timedBlock.Block.BlockID) // simulate building a child of `timedBlock` + delay := pubTime.Sub(receivedParentBlockAt) + // expecting decreasing GetProposalTiming + assert.LessOrEqual(bs.T(), delay, lastProposalDelay) + lastProposalDelay = delay + + // transition views until the end of the epoch, or for 100 views + if view-bs.initialView >= 100 { + break + } + } +} + +// TestProposalDelay_AheadOfSchedule tests the behaviour of the controller when the +// projected epoch switchover is EARLIER than the target switchover time, i.e. +// we are ahead of schedule. +// We should respond by increasing the GetProposalTiming (lowering view rate) +func (bs *BlockTimeControllerSuite) TestProposalDelay_AheadOfSchedule() { + // we are 50% of the way through the epoch in view terms + bs.initialView = uint64(float64(bs.curEpochFinalView) * .5) + bs.CreateAndStartController() + defer bs.StopController() + + lastProposalDelay := time.Duration(0) // start with large dummy value + idealEnteredViewTime := bs.ctl.curEpochTargetEndTime.Add(-epochLength / 2) + // 1s ahead of schedule + receivedParentBlockAt := idealEnteredViewTime.Add(-time.Second) + for view := bs.initialView + 1; view < bs.ctl.curEpochFinalView; view++ { + // hold the instantaneous error constant for each view + receivedParentBlockAt = receivedParentBlockAt.Add(seconds2Duration(bs.ctl.targetViewTime())) + timedBlock := makeTimedBlock(view, unittest.IdentifierFixture(), receivedParentBlockAt) + err := bs.ctl.measureViewDuration(timedBlock) + require.NoError(bs.T(), err) + + // compute proposal delay: + pubTime := bs.ctl.GetProposalTiming().TargetPublicationTime(view+1, time.Now().UTC(), timedBlock.Block.BlockID) // simulate building a child of `timedBlock` + delay := pubTime.Sub(receivedParentBlockAt) + + // expecting increasing GetProposalTiming + assert.GreaterOrEqual(bs.T(), delay, lastProposalDelay) + lastProposalDelay = delay + + // transition views until the end of the epoch, or for 100 views + if view-bs.initialView >= 100 { + break + } + } +} + +// TestMetrics tests that correct metrics are tracked when expected. +func (bs *BlockTimeControllerSuite) TestMetrics() { + bs.metrics = *mockmodule.NewCruiseCtlMetrics(bs.T()) + // should set metrics upon initialization + bs.metrics.On("PIDError", float64(0), float64(0), float64(0)).Once() + bs.metrics.On("TargetProposalDuration", time.Duration(0)).Once() + bs.metrics.On("ControllerOutput", time.Duration(0)).Once() + bs.CreateAndStartController() + defer bs.StopController() + bs.metrics.AssertExpectations(bs.T()) + + // we are at view 1 of the epoch, but the time is suddenly the target end time + enteredViewAt := bs.ctl.curEpochTargetEndTime + view := bs.initialView + 1 + // we should observe a large error + bs.metrics.On("PIDError", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + p := args[0].(float64) + i := args[1].(float64) + d := args[2].(float64) + assert.Greater(bs.T(), p, float64(0)) + assert.Greater(bs.T(), i, float64(0)) + assert.Greater(bs.T(), d, float64(0)) + }).Once() + // should immediately use min proposal duration + bs.metrics.On("TargetProposalDuration", bs.config.MinViewDuration.Load()).Once() + // should have a large negative controller output + bs.metrics.On("ControllerOutput", mock.Anything).Run(func(args mock.Arguments) { + output := args[0].(time.Duration) + assert.Greater(bs.T(), output, time.Duration(0)) + }).Once() + + timedBlock := makeTimedBlock(view, unittest.IdentifierFixture(), enteredViewAt) + err := bs.ctl.measureViewDuration(timedBlock) + require.NoError(bs.T(), err) +} + +// Test_vs_PythonSimulation performs a regression test. We implemented the controller in python +// together with a statistical model for the view duration. We used the python implementation to tune +// the PID controller parameters which we are using here. +// In this test, we feed values pre-generated with the python simulation into the Go implementation +// and compare the outputs to the pre-generated outputs from the python controller implementation. +func (bs *BlockTimeControllerSuite) Test_vs_PythonSimulation() { + // PART 1: setup system to mirror python simulation + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + totalEpochViews := 483000 + bs.initialView = 0 + bs.curEpochFirstView, bs.curEpochFinalView = uint64(0), uint64(totalEpochViews-1) // views [0, .., totalEpochViews-1] + bs.epochFallbackTriggered = false + + refT := time.Now().UTC() + refT = time.Date(refT.Year(), refT.Month(), refT.Day(), refT.Hour(), refT.Minute(), 0, 0, time.UTC) // truncate to past minute + bs.config = &Config{ + TimingConfig: TimingConfig{ + TargetTransition: EpochTransitionTime{day: refT.Weekday(), hour: uint8(refT.Hour()), minute: uint8(refT.Minute())}, + FallbackProposalDelay: atomic.NewDuration(500 * time.Millisecond), // irrelevant for this test, as controller should never enter fallback mode + MinViewDuration: atomic.NewDuration(470 * time.Millisecond), + MaxViewDuration: atomic.NewDuration(2010 * time.Millisecond), + Enabled: atomic.NewBool(true), + }, + ControllerParams: ControllerParams{KP: 2.0, KI: 0.06, KD: 3.0, N_ewma: 5, N_itg: 50}, + } + + setupMocks(bs) + bs.CreateAndStartController() + defer bs.StopController() + + // PART 2: timing generated from python simulation and corresponding controller response + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ref := struct { + // targetViewTime is the idealized view duration of a perfect system. + // In Python simulation, this is the array `EpochSimulation.ideal_view_time` + targetViewTime float64 // units: seconds + + // observedMinViewTimes[i] is the minimal time required time to execute the protocol for view i + // - Duration from the primary observing the parent block (indexed by i) to having its child proposal (block for view i+1) ready for publication. + // - This is the minimal time required to execute the protocol. Nodes can only delay their proposal but not progress any faster. + // - in Python simulation, this is the array `EpochSimulation.min_view_times + EpochSimulation.observation_noise` + // with is returned by function `EpochSimulation.current_view_observation()` + // Note that this is generally different than the time it takes the committee as a whole to transition + // through views. This is because the primary changes from view to view, and nodes observe blocks at slightly + // different times (small noise term). The real world (as well as the simulation) depend on collective swarm + // behaviour of the consensus committee, which is not observable by nodes individually. + // In contrast, our `observedMinViewTimes` here contains an additional noise term, to emulate + // the observations of a node in the real world. + observedMinViewTimes []float64 // units: seconds + + // controllerTargetedViewDuration[i] is the duration targeted by the Python controller : + // - measured from observing the parent until publishing the child block for view i+1 + controllerTargetedViewDuration []float64 // units: seconds + + // realWorldViewDuration[i] is the duration of the ith view for the entire committee. + // This value occurs in response to the controller output and is not observable by nodes individually. + // - in Python simulation, this is the array `EpochSimulation.real_world_view_duration` + // with is recorded by the environment upon the call of `EpochSimulation.delay()` + realWorldViewDuration []float64 // units: seconds + }{ + targetViewTime: 1.2521739130434784, + observedMinViewTimes: []float64{0.8139115907362099, 0.7093851608587579, 0.7370057913407495, 0.8378050305605419, 0.8221876685439506, 0.8129097289534515, 0.7835810854212116, 0.7419219104134447, 0.7122331139614623, 0.7263645183403751, 1.2481399484109290, 0.8741906105412369, 0.7082127929564489, 0.8175969272012624, 0.8040687048886446, 0.8163336940928989, 0.6354390018677689, 1.0568897015119771, 0.8283653995502240, 0.8649826738831023, 0.7249163864295024, 0.6572694879104934, 0.8796994117267707, 0.8251533370085626, 0.8383599333817994, 0.7561765091071196, 1.4239532706257330, 2.3848404271162811, 0.6997792104740760, 0.6783155065018911, 0.7397146999404549, 0.7568604144415827, 0.8224399309953295, 0.8635091458596464, 0.6292564656694590, 0.6399775559845721, 0.7551854294536755, 0.7493031513209824, 0.7916989850940226, 0.8584875376770561, 0.5733027665412744, 0.8190610271623866, 0.6664088123579012, 0.6856899641942998, 0.8235905136098289, 0.7673984464333541, 0.7514768668170753, 0.7145945518569533, 0.8076879859786521, 0.6890844388873341, 0.7782307638665685, 1.0031597171903470, 0.8056874789572074, 1.1894678554682030, 0.7751504335630999, 0.6598342159237116, 0.7198783916113262, 0.7231184452829420, 0.7291287772166142, 0.8941150065282033, 0.8216597987064465, 0.7074775436893693, 0.7886375844003763, 0.8028714839193359, 0.6473851384702657, 0.8247230728633490, 0.8268367270238434, 0.7776181863431995, 1.2870341252966155, 0.9022036087098005, 0.8608476621564736, 0.7448392402085238, 0.7030664985775897, 0.7343372879803260, 0.8501776646839836, 0.7949969493471933, 0.7030853022640485, 0.8506339844198412, 0.8520038195041865, 1.2159232403369129, 0.9501009619276108, 0.7063032843664507, 0.7676066345629766, 0.8050982844953996, 0.7460373897798731, 0.7531147127154058, 0.8276552672727131, 0.6777639708691676, 0.7759833549063068, 0.8861636486602165, 0.8272606701022402, 0.6742194284453155, 0.8270012408910985, 1.0799793512385585, 0.8343711941947437, 0.6424938240651709, 0.8314721058034046, 0.8687591599744876, 0.7681132139163648, 0.7993270549538212}, + realWorldViewDuration: []float64{1.2707067231074189, 1.3797713099533957, 1.1803368837187869, 1.0710943548975358, 1.3055277182347431, 1.3142312827952587, 1.2748087784689972, 1.2580713757160862, 1.2389594986278398, 1.2839951451881206, 0.8404551372521588, 1.7402295383244093, 1.2486807727203340, 1.1529076722170450, 1.2303564416007062, 1.1919067015405667, 1.4317417513319299, 0.8851802701506968, 1.4621618954558588, 1.2629599000198048, 1.3845528649513363, 1.3083813148510797, 1.0320875660949032, 1.2138806234836066, 1.2922205615230111, 1.3530469860253094, 1.5124780338765653, 2.4800000000000000, 0.8339877775027843, 0.7270580752471872, 0.8013511652567021, 0.7489973886099706, 0.9647668631144197, 1.4406086304771719, 1.6376005221775904, 1.3686144679115566, 1.2051140074616571, 1.2232170397428770, 1.1785015757024468, 1.2720488631325702, 1.4845607775546621, 1.0038608184511295, 1.4011693227324362, 1.2782420466946043, 1.0808595015305793, 1.2923716723984215, 1.2876404222029678, 1.3024029638718018, 1.1243308902566644, 1.3825311808461356, 1.1826028495527394, 1.0753560400260920, 1.4587594729770430, 1.3281281084314180, 1.1987898717701806, 1.3212567274973721, 1.2131355949220173, 1.2202213287069972, 1.2345177139086974, 1.1415707241388824, 1.2618615652263814, 1.3978228798726429, 1.1676202853133009, 1.2821402577607839, 1.4378331263208257, 1.0764974304705950, 1.1968636840861584, 1.3079197545950789, 1.3246769344178762, 1.0956265919521080, 1.3056225547363036, 1.3094504040915045, 1.2916519124885637, 1.2995343661957905, 1.0839793112463321, 1.2515453598485311, 1.3907042923175941, 1.1137329234266407, 1.2293962485228747, 1.4537855131563087, 1.1564260868809058, 1.2616419368628695, 1.1777963280146100, 1.2782540498222059, 1.2617698479511545, 1.2911000941099631, 1.1719344274281953, 1.3904853415093545, 1.1612440756337188, 1.1800751870755894, 1.2653752924717137, 1.3987404424771417, 1.1573292016433725, 1.2132227320045601, 1.2835627159341194, 1.3950341330597937, 1.0774862045842490, 1.2361956384863142, 1.3415505497959577, 1.1881870996394799}, + controllerTargetedViewDuration: []float64{1.2521739130434784, 1.2325291342837938, 1.0924796023620962, 1.1315714628442570, 1.3109201861848649, 1.2904005140483341, 1.2408200617376139, 1.2143186827596988, 1.2001258197216824, 1.2059386524427240, 1.1687014183641575, 1.5938588248347272, 1.1735049856838198, 1.1322996720968055, 1.2010702989934061, 1.2193620268012733, 1.2847380812524840, 1.1111384877632171, 1.4676632072726421, 1.3127404884038874, 1.3036822199799039, 1.1627828776831781, 1.0686746584877680, 1.2585854668086294, 1.3196479113378341, 1.3040688380370420, 1.2092520716891777, 0.9174437864843878, 0.4700000000000000, 0.4700000000000000, 0.4700000000000000, 0.4700000000000000, 0.9677983536241768, 1.4594930877396231, 1.4883132720086421, 1.2213393879261234, 1.1167787676139602, 1.1527862655996910, 1.1844688515164143, 1.2712560882996764, 1.2769188516898307, 1.0483030535756364, 1.2667785513482170, 1.1360673946540731, 1.0930571503977162, 1.2553993593963664, 1.2412509734564154, 1.2173708810202102, 1.1668170515618597, 1.2919854192770974, 1.1785774891590928, 1.2397180299682444, 1.4349751903776191, 1.2686663464941463, 1.1793337443757632, 1.2094760506747269, 1.1283680467942478, 1.1456014869605273, 1.1695603482439110, 1.1883473989997737, 1.3102878097954334, 1.3326636354319201, 1.2033095908546276, 1.2765637682955560, 1.2533105511679674, 1.0561925258579383, 1.1944030230453759, 1.2584181515051163, 1.2181701773236133, 1.1427643645565180, 1.2912929540520488, 1.2606456249879283, 1.2079980980125691, 1.1582846527456185, 1.0914599072895725, 1.2436632334468321, 1.2659732625682767, 1.1373906460646186, 1.2636670215783354, 1.3065542716228340, 1.1145058661373550, 1.1821457478344533, 1.1686494999739092, 1.2421504164081945, 1.2292642544361261, 1.2247229593559099, 1.1857675147732030, 1.2627704665069508, 1.1302481979483210, 1.2027256964130453, 1.2826968566299934, 1.2903197193121982, 1.1497164007008540, 1.2248494620352162, 1.2695192555858241, 1.2492112043621006, 1.1006141873118667, 1.2513218024356318, 1.2846249908259910, 1.2077144025965167}, + } + + // PART 3: run controller and ensure output matches pre-generated controller response from python ref implementation + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // sanity checks: + require.Equal(bs.T(), 604800.0, bs.ctl.curEpochTargetEndTime.UTC().Sub(refT).Seconds(), "Epoch should end 1 week from now, i.e. 604800s") + require.InEpsilon(bs.T(), ref.targetViewTime, bs.ctl.targetViewTime(), 1e-15) // ideal view time + require.Equal(bs.T(), len(ref.observedMinViewTimes), len(ref.realWorldViewDuration)) + + // Notes: + // - We specifically make the first observation at when the full time of the epoch is left. + // The python simulation we compare with proceed exactly the same way. + // - we first make an observation, before requesting the controller output. Thereby, we + // avoid artifacts of recalling a controller that was just initialized with fallback values. + // - we call `measureViewDuration(..)` (_not_ `processIncorporatedBlock(..)`) to + // interfering with the deduplication logic. Here we want to test correct numerics. + // Correctness of the deduplication logic is verified in the different test. + observationTime := refT + + for v := 0; v < len(ref.observedMinViewTimes); v++ { + observedBlock := makeTimedBlock(uint64(v), unittest.IdentifierFixture(), observationTime) + err := bs.ctl.measureViewDuration(observedBlock) + require.NoError(bs.T(), err) + proposalTiming := bs.ctl.GetProposalTiming() + tpt := proposalTiming.TargetPublicationTime(uint64(v+1), time.Now(), observedBlock.Block.BlockID) // value for `timeViewEntered` should be irrelevant here + + controllerTargetedViewDuration := tpt.Sub(observedBlock.TimeObserved).Seconds() + require.InEpsilon(bs.T(), ref.controllerTargetedViewDuration[v], controllerTargetedViewDuration, 1e-5, "implementations deviate for view %d", v) // ideal view time + + observationTime = observationTime.Add(time.Duration(int64(ref.realWorldViewDuration[v] * float64(time.Second)))) + } + +} + +func makeTimedBlock(view uint64, parentID flow.Identifier, time time.Time) TimedBlock { + header := unittest.BlockHeaderFixture(unittest.HeaderWithView(view)) + header.ParentID = parentID + return TimedBlock{ + Block: model.BlockFromFlow(header), + TimeObserved: time, + } +} + +type controllerStateDigest struct { + proportionalErr Ewma + integralErr LeakyIntegrator + + // latestProposalTiming holds the ProposalTiming that the controller generated in response to processing the latest observation + latestProposalTiming ProposalTiming +} + +func captureControllerStateDigest(ctl *BlockTimeController) *controllerStateDigest { + return &controllerStateDigest{ + proportionalErr: ctl.proportionalErr, + integralErr: ctl.integralErr, + latestProposalTiming: ctl.GetProposalTiming(), + } +} + +func seconds2Duration(durationinDeconds float64) time.Duration { + return time.Duration(int64(durationinDeconds * float64(time.Second))) +} diff --git a/consensus/hotstuff/cruisectl/config.go b/consensus/hotstuff/cruisectl/config.go new file mode 100644 index 00000000000..48a6f2b1139 --- /dev/null +++ b/consensus/hotstuff/cruisectl/config.go @@ -0,0 +1,126 @@ +package cruisectl + +import ( + "time" + + "go.uber.org/atomic" +) + +// DefaultConfig returns the default config for the BlockTimeController. +func DefaultConfig() *Config { + return &Config{ + TimingConfig{ + TargetTransition: DefaultEpochTransitionTime(), + FallbackProposalDelay: atomic.NewDuration(250 * time.Millisecond), + MinViewDuration: atomic.NewDuration(600 * time.Millisecond), + MaxViewDuration: atomic.NewDuration(1600 * time.Millisecond), + Enabled: atomic.NewBool(false), + }, + ControllerParams{ + N_ewma: 5, + N_itg: 50, + KP: 2.0, + KI: 0.6, + KD: 3.0, + }, + } +} + +// Config defines configuration for the BlockTimeController. +type Config struct { + TimingConfig + ControllerParams +} + +// TimingConfig specifies the BlockTimeController's limits of authority. +type TimingConfig struct { + // TargetTransition defines the target time to transition epochs each week. + TargetTransition EpochTransitionTime + + // FallbackProposalDelay is the minimal block construction delay. When used, it behaves like the + // old command line flag `block-rate-delay`. Specifically, the primary measures the duration from + // starting to construct its proposal to the proposal being ready to be published. If this + // duration is _less_ than FallbackProposalDelay, the primary delays broadcasting its proposal + // by the remainder needed to reach `FallbackProposalDelay` + // It is used: + // - when Enabled is false + // - when epoch fallback has been triggered + FallbackProposalDelay *atomic.Duration + + // MaxViewDuration is a hard maximum on the total view time targeted by ProposalTiming. + // If the BlockTimeController computes a larger desired ProposalTiming value + // based on the observed error and tuning, this value will be used instead. + MaxViewDuration *atomic.Duration + + // MinViewDuration is a hard maximum on the total view time targeted by ProposalTiming. + // If the BlockTimeController computes a smaller desired ProposalTiming value + // based on the observed error and tuning, this value will be used instead. + MinViewDuration *atomic.Duration + + // Enabled defines whether responsive control of the GetProposalTiming is enabled. + // When disabled, the FallbackProposalDelay is used. + Enabled *atomic.Bool +} + +// ControllerParams specifies the BlockTimeController's internal parameters. +type ControllerParams struct { + // N_ewma defines how historical measurements are incorporated into the EWMA for the proportional error term. + // Intuition: Suppose the input changes from x to y instantaneously: + // - N_ewma is the number of samples required to move the EWMA output about 2/3 of the way from x to y + // Per convention, this must be a _positive_ integer. + N_ewma uint + + // N_itg defines how historical measurements are incorporated into the integral error term. + // Intuition: For a constant error x: + // - the integrator value will saturate at `x•N_itg` + // - an integrator initialized at 0 reaches 2/3 of the saturation value after N_itg samples + // Per convention, this must be a _positive_ integer. + N_itg uint + + // KP, KI, KD, are the coefficients to the PID controller and define its response. + // KP adjusts the proportional term (responds to the magnitude of error). + // KI adjusts the integral term (responds to the error sum over a recent time interval). + // KD adjusts the derivative term (responds to the rate of change, i.e. time derivative, of the error). + KP, KI, KD float64 +} + +// alpha returns α, the inclusion parameter for the error EWMA. See N_ewma for details. +func (c *ControllerParams) alpha() float64 { + return 1.0 / float64(c.N_ewma) +} + +// beta returns ß, the memory parameter of the leaky error integrator. See N_itg for details. +func (c *ControllerParams) beta() float64 { + return 1.0 / float64(c.N_itg) +} + +func (ctl *TimingConfig) GetFallbackProposalDuration() time.Duration { + return ctl.FallbackProposalDelay.Load() +} +func (ctl *TimingConfig) GetMaxViewDuration() time.Duration { + return ctl.MaxViewDuration.Load() +} +func (ctl *TimingConfig) GetMinViewDuration() time.Duration { + return ctl.MinViewDuration.Load() +} +func (ctl *TimingConfig) GetEnabled() bool { + return ctl.Enabled.Load() +} + +func (ctl *TimingConfig) SetFallbackProposalDuration(dur time.Duration) error { + ctl.FallbackProposalDelay.Store(dur) + return nil +} +func (ctl *TimingConfig) SetMaxViewDuration(dur time.Duration) error { + ctl.MaxViewDuration.Store(dur) + return nil +} +func (ctl *TimingConfig) SetMinViewDuration(dur time.Duration) error { + ctl.MinViewDuration.Store(dur) + return nil + +} +func (ctl *TimingConfig) SetEnabled(enabled bool) error { + ctl.Enabled.Store(enabled) + return nil +} diff --git a/consensus/hotstuff/cruisectl/proposal_timing.go b/consensus/hotstuff/cruisectl/proposal_timing.go new file mode 100644 index 00000000000..acfa4deed28 --- /dev/null +++ b/consensus/hotstuff/cruisectl/proposal_timing.go @@ -0,0 +1,151 @@ +package cruisectl + +import ( + "time" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/model/flow" +) + +// ProposalTiming encapsulates the output of the BlockTimeController. On the happy path, +// the controller observes a block and generates a specific ProposalTiming in response. +// For the happy path, the ProposalTiming describes when the child proposal should be +// broadcast. +// However, observations other than blocks might also be used to instantiate ProposalTiming +// objects, e.g. controller instantiation, a disabled controller, etc. +// The purpose of ProposalTiming is to convert the controller output to timing information +// that the EventHandler understands. By convention, ProposalTiming should be treated as +// immutable. +type ProposalTiming interface { + hotstuff.ProposalDurationProvider + + // ObservationView returns the view of the observation that the controller + // processed and generated this ProposalTiming instance in response. + ObservationView() uint64 + + // ObservationTime returns the time, when the controller received the + // leading to the generation of this ProposalTiming instance. + ObservationTime() time.Time +} + +/* *************************************** publishImmediately *************************************** */ + +// publishImmediately implements ProposalTiming: it returns the time when the view +// was entered as the TargetPublicationTime. By convention, publishImmediately should +// be treated as immutable. +type publishImmediately struct { + observationView uint64 + observationTime time.Time +} + +var _ ProposalTiming = (*publishImmediately)(nil) + +func newPublishImmediately(observationView uint64, observationTime time.Time) *publishImmediately { + return &publishImmediately{ + observationView: observationView, + observationTime: observationTime, + } +} + +func (pt *publishImmediately) TargetPublicationTime(_ uint64, timeViewEntered time.Time, _ flow.Identifier) time.Time { + return timeViewEntered +} +func (pt *publishImmediately) ObservationView() uint64 { return pt.observationView } +func (pt *publishImmediately) ObservationTime() time.Time { return pt.observationTime } +func (pt *publishImmediately) ConstrainedBlockTime() time.Duration { return 0 } + +/* *************************************** happyPathBlockTime *************************************** */ + +// happyPathBlockTime implements ProposalTiming for the happy path. Here, `TimedBlock` _latest_ block that the +// controller observed, and the `unconstrainedBlockTime` for the _child_ of this block. +// This function internally holds the _unconstrained_ view duration as computed by the BlockTimeController. Caution, +// no limits of authority have been applied to this value yet. The final controller output satisfying the limits of +// authority is computed by function `ConstrainedBlockTime()` +// +// For a given view where we are the primary, suppose the parent block we are building on top of has been observed +// at time `t := TimedBlock.TimeObserved` and applying the limits of authority yields `d := ConstrainedBlockTime()` +// Then, `TargetPublicationTime(..)` returns `t + d` as the target publication time for the child block. +// +// By convention, happyPathBlockTime should be treated as immutable. +// TODO: any additional logic for assiting the EventHandler in determining the applied delay should be added to the ControllerViewDuration +type happyPathBlockTime struct { + TimedBlock // latest block observed by the controller, including the time stamp when the controller received the block [UTC] + TimingConfig // timing configuration for the controller, for retrieving the controller's limits of authority + + // unconstrainedBlockTime is the delay, relative to `TimedBlock.TimeObserved` when the controller would + // like the child block to be published. Caution, no limits of authority have been applied to this value yet. + // The final controller output after applying the limits of authority is returned by function `ConstrainedBlockTime` + unconstrainedBlockTime time.Duration // desired duration until releasing the child block, measured from `TimedBlock.TimeObserved` + + constrainedBlockTime time.Duration // block time _after_ applying limits of authority to unconstrainedBlockTime +} + +var _ ProposalTiming = (*happyPathBlockTime)(nil) + +// newHappyPathBlockTime instantiates a new happyPathBlockTime +func newHappyPathBlockTime(timedBlock TimedBlock, unconstrainedBlockTime time.Duration, timingConfig TimingConfig) *happyPathBlockTime { + return &happyPathBlockTime{ + TimingConfig: timingConfig, + TimedBlock: timedBlock, + unconstrainedBlockTime: unconstrainedBlockTime, + constrainedBlockTime: min(max(unconstrainedBlockTime, timingConfig.MinViewDuration.Load()), timingConfig.MaxViewDuration.Load()), + } +} + +func (pt *happyPathBlockTime) ObservationView() uint64 { return pt.Block.View } +func (pt *happyPathBlockTime) ObservationTime() time.Time { return pt.TimeObserved } +func (pt *happyPathBlockTime) ConstrainedBlockTime() time.Duration { return pt.constrainedBlockTime } + +// TargetPublicationTime operates in two possible modes: +// 1. If `parentBlockId` matches our `TimedBlock`, i.e. the EventHandler is just building the child block, then +// we return `TimedBlock.TimeObserved + ConstrainedBlockTime` as the target publication time for the child block. +// 2. If `parentBlockId` does _not_ match our `TimedBlock`, the EventHandler should release the block immediately. +// This heuristic is based on the intuition that Block time is expected to be very long when deviating from the happy path. +func (pt *happyPathBlockTime) TargetPublicationTime(proposalView uint64, timeViewEntered time.Time, parentBlockId flow.Identifier) time.Time { + if parentBlockId != pt.Block.BlockID { + return timeViewEntered // broadcast immediately + } + return pt.TimeObserved.Add(pt.ConstrainedBlockTime()) // happy path +} + +/* *************************************** fallbackTiming for EECC *************************************** */ + +// fallbackTiming implements ProposalTiming, for the basic fallback: +// function `TargetPublicationTime(..)` always returns `timeViewEntered + defaultProposalDuration` +type fallbackTiming struct { + observationView uint64 + observationTime time.Time + defaultProposalDuration time.Duration +} + +var _ ProposalTiming = (*fallbackTiming)(nil) + +func newFallbackTiming(observationView uint64, observationTime time.Time, defaultProposalDuration time.Duration) *fallbackTiming { + return &fallbackTiming{ + observationView: observationView, + observationTime: observationTime, + defaultProposalDuration: defaultProposalDuration, + } +} + +func (pt *fallbackTiming) TargetPublicationTime(_ uint64, timeViewEntered time.Time, _ flow.Identifier) time.Time { + return timeViewEntered.Add(pt.defaultProposalDuration) +} +func (pt *fallbackTiming) ObservationView() uint64 { return pt.observationView } +func (pt *fallbackTiming) ObservationTime() time.Time { return pt.observationTime } + +/* *************************************** auxiliary functions *************************************** */ + +func min(d1, d2 time.Duration) time.Duration { + if d1 < d2 { + return d1 + } + return d2 +} + +func max(d1, d2 time.Duration) time.Duration { + if d1 > d2 { + return d1 + } + return d2 +} diff --git a/consensus/hotstuff/cruisectl/transition_time.go b/consensus/hotstuff/cruisectl/transition_time.go new file mode 100644 index 00000000000..52bfad3486b --- /dev/null +++ b/consensus/hotstuff/cruisectl/transition_time.go @@ -0,0 +1,172 @@ +package cruisectl + +import ( + "fmt" + "strings" + "time" +) + +// weekdays is a lookup from canonical weekday strings to the time package constant. +var weekdays = map[string]time.Weekday{ + strings.ToLower(time.Sunday.String()): time.Sunday, + strings.ToLower(time.Monday.String()): time.Monday, + strings.ToLower(time.Tuesday.String()): time.Tuesday, + strings.ToLower(time.Wednesday.String()): time.Wednesday, + strings.ToLower(time.Thursday.String()): time.Thursday, + strings.ToLower(time.Friday.String()): time.Friday, + strings.ToLower(time.Saturday.String()): time.Saturday, +} + +// epochLength is the length of an epoch (7 days, or 1 week). +const epochLength = time.Hour * 24 * 7 + +var transitionFmt = "%s@%02d:%02d" // example: wednesday@08:00 + +// EpochTransitionTime represents the target epoch transition time. +// Epochs last one week, so the transition is defined in terms of a day-of-week and time-of-day. +// The target time is always in UTC to avoid confusion resulting from different +// representations of the same transition time and around daylight savings time. +type EpochTransitionTime struct { + day time.Weekday // day of every week to target epoch transition + hour uint8 // hour of the day to target epoch transition + minute uint8 // minute of the hour to target epoch transition +} + +// DefaultEpochTransitionTime is the default epoch transition target. +// The target switchover is Wednesday 12:00 PDT, which is 19:00 UTC. +// The string representation is `wednesday@19:00`. +func DefaultEpochTransitionTime() EpochTransitionTime { + return EpochTransitionTime{ + day: time.Wednesday, + hour: 19, + minute: 0, + } +} + +// String returns the canonical string representation of the transition time. +// This is the format expected as user input, when this value is configured manually. +// See ParseSwitchover for details of the format. +func (tt *EpochTransitionTime) String() string { + return fmt.Sprintf(transitionFmt, strings.ToLower(tt.day.String()), tt.hour, tt.minute) +} + +// newInvalidTransitionStrError returns an informational error about an invalid transition string. +func newInvalidTransitionStrError(s string, msg string, args ...any) error { + args = append([]any{s}, args...) + return fmt.Errorf("invalid transition string (%s): "+msg, args...) +} + +// ParseTransition parses a transition time string. +// A transition string must be specified according to the format: +// +// WD@HH:MM +// +// WD is the weekday string as defined by `strings.ToLower(time.Weekday.String)` +// HH is the 2-character hour of day, in the range [00-23] +// MM is the 2-character minute of hour, in the range [00-59] +// All times are in UTC. +// +// A generic error is returned if the input is an invalid transition string. +func ParseTransition(s string) (*EpochTransitionTime, error) { + strs := strings.Split(s, "@") + if len(strs) != 2 { + return nil, newInvalidTransitionStrError(s, "split on @ yielded %d substrings - expected %d", len(strs), 2) + } + dayStr := strs[0] + timeStr := strs[1] + if len(timeStr) != 5 || timeStr[2] != ':' { + return nil, newInvalidTransitionStrError(s, "time part must have form HH:MM") + } + + var hour uint8 + _, err := fmt.Sscanf(timeStr[0:2], "%02d", &hour) + if err != nil { + return nil, newInvalidTransitionStrError(s, "error scanning hour part: %w", err) + } + var minute uint8 + _, err = fmt.Sscanf(timeStr[3:5], "%02d", &minute) + if err != nil { + return nil, newInvalidTransitionStrError(s, "error scanning minute part: %w", err) + } + + day, ok := weekdays[strings.ToLower(dayStr)] + if !ok { + return nil, newInvalidTransitionStrError(s, "invalid weekday part %s", dayStr) + } + if hour > 23 { + return nil, newInvalidTransitionStrError(s, "invalid hour part: %d>23", hour) + } + if minute > 59 { + return nil, newInvalidTransitionStrError(s, "invalid minute part: %d>59", hour) + } + + return &EpochTransitionTime{ + day: day, + hour: hour, + minute: minute, + }, nil +} + +// inferTargetEndTime infers the target end time for the current epoch, based on +// the current progress through the epoch and the current time. +// We do this in 3 steps: +// 1. find the 3 candidate target end times nearest to the current time. +// 2. compute the estimated end time for the current epoch. +// 3. select the candidate target end time which is nearest to the estimated end time. +// +// NOTE 1: This method is effective only if the node's local notion of current view and +// time are accurate. If a node is, for example, catching up from a very old state, it +// will infer incorrect target end times. Since catching-up nodes don't produce usable +// proposals, this is OK. +// NOTE 2: In the long run, the target end time should be specified by the smart contract +// and stored along with the other protocol.Epoch information. This would remove the +// need for this imperfect inference logic. +func (tt *EpochTransitionTime) inferTargetEndTime(curTime time.Time, epochFractionComplete float64) time.Time { + now := curTime.UTC() + // find the nearest target end time, plus the targets one week before and after + nearestTargetDate := tt.findNearestTargetTime(now) + earlierTargetDate := nearestTargetDate.AddDate(0, 0, -7) + laterTargetDate := nearestTargetDate.AddDate(0, 0, 7) + + estimatedTimeRemainingInEpoch := time.Duration((1.0 - epochFractionComplete) * float64(epochLength)) + estimatedEpochEndTime := now.Add(estimatedTimeRemainingInEpoch) + + minDiff := estimatedEpochEndTime.Sub(nearestTargetDate).Abs() + inferredTargetEndTime := nearestTargetDate + for _, date := range []time.Time{earlierTargetDate, laterTargetDate} { + // compare estimate to actual based on the target + diff := estimatedEpochEndTime.Sub(date).Abs() + if diff < minDiff { + minDiff = diff + inferredTargetEndTime = date + } + } + + return inferredTargetEndTime +} + +// findNearestTargetTime interprets ref as a date (ignores time-of-day portion) +// and finds the nearest date, either before or after ref, which has the given weekday. +// We then return a time.Time with this date and the hour/minute specified by the EpochTransitionTime. +func (tt *EpochTransitionTime) findNearestTargetTime(ref time.Time) time.Time { + ref = ref.UTC() + hour := int(tt.hour) + minute := int(tt.minute) + date := time.Date(ref.Year(), ref.Month(), ref.Day(), hour, minute, 0, 0, time.UTC) + + // walk back and forth by date around the reference until we find the closest matching weekday + walk := 0 + for date.Weekday() != tt.day || date.Sub(ref).Abs().Hours() > float64(24*7/2) { + walk++ + if walk%2 == 0 { + date = date.AddDate(0, 0, walk) + } else { + date = date.AddDate(0, 0, -walk) + } + // sanity check to avoid an infinite loop: should be impossible + if walk > 14 { + panic(fmt.Sprintf("unexpected failure to find nearest target time with ref=%s, transition=%s", ref.String(), tt.String())) + } + } + return date +} diff --git a/consensus/hotstuff/cruisectl/transition_time_test.go b/consensus/hotstuff/cruisectl/transition_time_test.go new file mode 100644 index 00000000000..15bff07ce1e --- /dev/null +++ b/consensus/hotstuff/cruisectl/transition_time_test.go @@ -0,0 +1,145 @@ +package cruisectl + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "pgregory.net/rapid" +) + +// TestParseTransition_Valid tests that valid transition configurations have +// consistent parsing and formatting behaviour. +func TestParseTransition_Valid(t *testing.T) { + cases := []struct { + transition EpochTransitionTime + str string + }{{ + transition: EpochTransitionTime{time.Sunday, 0, 0}, + str: "sunday@00:00", + }, { + transition: EpochTransitionTime{time.Wednesday, 8, 1}, + str: "wednesday@08:01", + }, { + transition: EpochTransitionTime{time.Monday, 23, 59}, + str: "monday@23:59", + }, { + transition: EpochTransitionTime{time.Friday, 12, 21}, + str: "FrIdAy@12:21", + }} + + for _, c := range cases { + t.Run(c.str, func(t *testing.T) { + // 1 - the computed string representation should match the string fixture + assert.Equal(t, strings.ToLower(c.str), c.transition.String()) + // 2 - the parsed transition should match the transition fixture + parsed, err := ParseTransition(c.str) + assert.NoError(t, err) + assert.Equal(t, c.transition, *parsed) + }) + } +} + +// TestParseTransition_Invalid tests that a selection of invalid transition strings +// fail validation and return an error. +func TestParseTransition_Invalid(t *testing.T) { + cases := []string{ + // invalid WD part + "sundy@12:00", + "tue@12:00", + "@12:00", + // invalid HH part + "wednesday@24:00", + "wednesday@1:00", + "wednesday@:00", + "wednesday@012:00", + // invalid MM part + "wednesday@12:60", + "wednesday@12:1", + "wednesday@12:", + "wednesday@12:030", + // otherwise invalid + "", + "@:", + "monday@@12:00", + "monday@09:00am", + "monday@09:00PM", + "monday12:00", + "monday12::00", + "wednesday@1200", + } + + for _, transitionStr := range cases { + t.Run(transitionStr, func(t *testing.T) { + _, err := ParseTransition(transitionStr) + assert.Error(t, err) + }) + } +} + +// drawTransitionTime draws a random EpochTransitionTime. +func drawTransitionTime(t *rapid.T) EpochTransitionTime { + day := time.Weekday(rapid.IntRange(0, 6).Draw(t, "wd").(int)) + hour := rapid.Uint8Range(0, 23).Draw(t, "h").(uint8) + minute := rapid.Uint8Range(0, 59).Draw(t, "m").(uint8) + return EpochTransitionTime{day, hour, minute} +} + +// TestInferTargetEndTime_Fixture is a single human-readable fixture test, +// in addition to the property-based rapid tests. +func TestInferTargetEndTime_Fixture(t *testing.T) { + // The target time is around midday Wednesday + // |S|M|T|W|T|F|S| + // | * | + ett := EpochTransitionTime{day: time.Wednesday, hour: 13, minute: 24} + // The current time is mid-morning on Friday. We are about 28% through the epoch in time terms + // |S|M|T|W|T|F|S| + // | * | + // Friday, November 20, 2020 11:44 + curTime := time.Date(2020, 11, 20, 11, 44, 0, 0, time.UTC) + // We are 18% through the epoch in view terms - we are quite behind schedule + epochFractionComplete := .18 + // We should still be able to infer the target switchover time: + // Wednesday, November 25, 2020 13:24 + expectedTarget := time.Date(2020, 11, 25, 13, 24, 0, 0, time.UTC) + target := ett.inferTargetEndTime(curTime, epochFractionComplete) + assert.Equal(t, expectedTarget, target) +} + +// TestInferTargetEndTime tests that we can infer "the most reasonable" target time. +func TestInferTargetEndTime_Rapid(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + ett := drawTransitionTime(t) + curTime := time.Unix(rapid.Int64().Draw(t, "ref_unix").(int64), 0).UTC() + epochFractionComplete := rapid.Float64Range(0, 1).Draw(t, "pct_complete").(float64) + epochFractionRemaining := 1.0 - epochFractionComplete + + target := ett.inferTargetEndTime(curTime, epochFractionComplete) + computedEndTime := curTime.Add(time.Duration(float64(epochLength) * epochFractionRemaining)) + // selected target must be the nearest to the computed end time + delta := computedEndTime.Sub(target).Abs() + assert.LessOrEqual(t, delta.Hours(), float64(24*7)/2) + // nearest date must be a target time + assert.Equal(t, ett.day, target.Weekday()) + assert.Equal(t, int(ett.hour), target.Hour()) + assert.Equal(t, int(ett.minute), target.Minute()) + }) +} + +// TestFindNearestTargetTime tests finding the nearest target time to a reference time. +func TestFindNearestTargetTime(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + ett := drawTransitionTime(t) + ref := time.Unix(rapid.Int64().Draw(t, "ref_unix").(int64), 0).UTC() + + nearest := ett.findNearestTargetTime(ref) + distance := nearest.Sub(ref).Abs() + // nearest date must be at most 1/2 a week away + assert.LessOrEqual(t, distance.Hours(), float64(24*7)/2) + // nearest date must be a target time + assert.Equal(t, ett.day, nearest.Weekday()) + assert.Equal(t, int(ett.hour), nearest.Hour()) + assert.Equal(t, int(ett.minute), nearest.Minute()) + }) +} diff --git a/consensus/hotstuff/event_loop.go b/consensus/hotstuff/event_loop.go index f107449c457..cadc1cc61e3 100644 --- a/consensus/hotstuff/event_loop.go +++ b/consensus/hotstuff/event_loop.go @@ -8,5 +8,5 @@ import ( type EventLoop interface { module.HotStuff TimeoutCollectorConsumer - QCCreatedConsumer + VoteCollectorConsumer } diff --git a/consensus/hotstuff/eventhandler/event_handler.go b/consensus/hotstuff/eventhandler/event_handler.go index e1558d64144..93fa54ba56e 100644 --- a/consensus/hotstuff/eventhandler/event_handler.go +++ b/consensus/hotstuff/eventhandler/event_handler.go @@ -155,7 +155,7 @@ func (e *EventHandler) OnReceiveProposal(proposal *model.Proposal) error { } // store the block. - err := e.forks.AddProposal(proposal) + err := e.forks.AddValidatedBlock(block) if err != nil { return fmt.Errorf("cannot add proposal to forks (%x): %w", block.BlockID, err) } @@ -261,16 +261,10 @@ func (e *EventHandler) OnPartialTcCreated(partialTC *hotstuff.PartialTcCreated) // be executed by the same goroutine that also calls the other business logic // methods, or concurrency safety has to be implemented externally. func (e *EventHandler) Start(ctx context.Context) error { - // notify about commencing recovery procedure e.notifier.OnStart(e.paceMaker.CurView()) defer e.notifier.OnEventProcessed() e.paceMaker.Start(ctx) - - err := e.processPendingBlocks() - if err != nil { - return fmt.Errorf("could not process pending blocks: %w", err) - } - err = e.proposeForNewViewIfPrimary() + err := e.proposeForNewViewIfPrimary() if err != nil { return fmt.Errorf("could not start new view: %w", err) } @@ -314,47 +308,6 @@ func (e *EventHandler) broadcastTimeoutObjectIfAuthorized() error { return nil } -// processPendingBlocks performs processing of pending blocks that were applied to chain state but weren't processed -// by Hotstuff event loop. Due to asynchronous nature of our processing pipelines compliance engine can validate and apply -// blocks to the chain state but fail to deliver them to EventHandler because of shutdown or crash. To recover those QCs and TCs -// recovery logic puts them in Forks and EventHandler can traverse pending blocks by view to obtain them. -func (e *EventHandler) processPendingBlocks() error { - newestView := e.forks.NewestView() - currentView := e.paceMaker.CurView() - for { - paceMakerActiveView := e.paceMaker.CurView() - if currentView < paceMakerActiveView { - currentView = paceMakerActiveView - } - - if currentView > newestView { - return nil - } - - // check if there are pending proposals for active view - pendingProposals := e.forks.GetProposalsForView(currentView) - // process all proposals for view, we are dealing only with valid QCs and TCs so no harm in processing - // double proposals here. - for _, proposal := range pendingProposals { - block := proposal.Block - _, err := e.paceMaker.ProcessQC(block.QC) - if err != nil { - return fmt.Errorf("could not process QC for block %x: %w", block.BlockID, err) - } - - _, err = e.paceMaker.ProcessTC(proposal.LastViewTC) - if err != nil { - return fmt.Errorf("could not process TC for block %x: %w", block.BlockID, err) - } - - // TODO(active-pacemaker): generally speaking we are only interested in QC and TC, but in some cases - // we might want to vote for blocks as well. Discuss if it's needed. - } - - currentView++ - } -} - // proposeForNewViewIfPrimary will only be called when we may able to propose a block, after processing a new event. // - after entering a new view as a result of processing a QC or TC, then we may propose for the newly entered view // - after receiving a proposal (but not changing view), if that proposal is referenced by our highest known QC, @@ -381,10 +334,14 @@ func (e *EventHandler) proposeForNewViewIfPrimary() error { if e.committee.Self() != currentLeader { return nil } - for _, p := range e.forks.GetProposalsForView(curView) { - if p.Block.ProposerID == e.committee.Self() { + for _, b := range e.forks.GetBlocksForView(curView) { // on the happy path, this slice is empty + if b.ProposerID == e.committee.Self() { log.Debug().Msg("already proposed for current view") return nil + } else { + // sanity check: the following code should never be reached, as this node is the current leader, i.e. + // we should _not_ consider a proposal for this view from any other as valid and store it in forks. + return fmt.Errorf("this node (%v) is leader for the current view %d, but have a proposal from node %v for this view", currentLeader, curView, b.ProposerID) } } @@ -392,7 +349,7 @@ func (e *EventHandler) proposeForNewViewIfPrimary() error { newestQC := e.paceMaker.NewestQC() lastViewTC := e.paceMaker.LastViewTC() - _, found := e.forks.GetProposal(newestQC.BlockID) + _, found := e.forks.GetBlock(newestQC.BlockID) if !found { // we don't know anything about block referenced by our newest QC, in this case we can't // create a valid proposal since we can't guarantee validity of block payload. @@ -406,7 +363,7 @@ func (e *EventHandler) proposeForNewViewIfPrimary() error { // Sanity checks to make sure that resulting proposal is valid: // In its proposal, the leader for view N needs to present evidence that it has legitimately entered view N. // As evidence, we include a QC or TC for view N-1, which should always be available as the PaceMaker advances - // to view N only after observing a QC or TC from view N-1. Moreover QC and TC are always processed together. As + // to view N only after observing a QC or TC from view N-1. Moreover, QC and TC are always processed together. As // EventHandler is strictly single-threaded without reentrancy, we must have a QC or TC for the prior view (curView-1). // Failing one of these sanity checks is a symptom of state corruption or a severe implementation bug. if newestQC.View+1 != curView { @@ -428,27 +385,28 @@ func (e *EventHandler) proposeForNewViewIfPrimary() error { if err != nil { return fmt.Errorf("can not make block proposal for curView %v: %w", curView, err) } - proposal := model.ProposalFromFlow(flowProposal) // turn the signed flow header into a proposal + proposedBlock := model.BlockFromFlow(flowProposal) // turn the signed flow header into a proposal + // determine target publication time (CAUTION: must happen before AddValidatedBlock) + targetPublicationTime := e.paceMaker.TargetPublicationTime(flowProposal.View, start, flowProposal.ParentID) // we want to store created proposal in forks to make sure that we don't create more proposals for // current view. Due to asynchronous nature of our design it's possible that after creating proposal // we will be asked to propose again for same view. - err = e.forks.AddProposal(proposal) + err = e.forks.AddValidatedBlock(proposedBlock) if err != nil { - return fmt.Errorf("could not add newly created proposal (%v): %w", proposal.Block.BlockID, err) + return fmt.Errorf("could not add newly created proposal (%v): %w", proposedBlock.BlockID, err) } - block := proposal.Block log.Debug(). - Uint64("block_view", block.View). - Hex("block_id", block.BlockID[:]). + Uint64("block_view", proposedBlock.View). + Time("target_publication", targetPublicationTime). + Hex("block_id", proposedBlock.BlockID[:]). Uint64("parent_view", newestQC.View). Hex("parent_id", newestQC.BlockID[:]). - Hex("signer", block.ProposerID[:]). + Hex("signer", proposedBlock.ProposerID[:]). Msg("forwarding proposal to communicator for broadcasting") // raise a notification with proposal (also triggers broadcast) - targetPublicationTime := start.Add(e.paceMaker.BlockRateDelay()) e.notifier.OnOwnProposal(flowProposal, targetPublicationTime) return nil } @@ -502,7 +460,7 @@ func (e *EventHandler) ownVote(proposal *model.Proposal, curView uint64, nextLea Hex("signer", block.ProposerID[:]). Logger() - _, found := e.forks.GetProposal(proposal.Block.QC.BlockID) + _, found := e.forks.GetBlock(proposal.Block.QC.BlockID) if !found { // we don't have parent for this proposal, we can't vote since we can't guarantee validity of proposals // payload. Strictly speaking this shouldn't ever happen because compliance engine makes sure that we diff --git a/consensus/hotstuff/eventhandler/event_handler_test.go b/consensus/hotstuff/eventhandler/event_handler_test.go index 485b0cc91f2..1e4dbf08317 100644 --- a/consensus/hotstuff/eventhandler/event_handler_test.go +++ b/consensus/hotstuff/eventhandler/event_handler_test.go @@ -38,11 +38,13 @@ type TestPaceMaker struct { var _ hotstuff.PaceMaker = (*TestPaceMaker)(nil) -func NewTestPaceMaker(timeoutController *timeout.Controller, +func NewTestPaceMaker( + timeoutController *timeout.Controller, + proposalDelayProvider hotstuff.ProposalDurationProvider, notifier hotstuff.Consumer, persist hotstuff.Persister, ) *TestPaceMaker { - p, err := pacemaker.New(timeoutController, notifier, persist) + p, err := pacemaker.New(timeoutController, proposalDelayProvider, notifier, persist) if err != nil { panic(err) } @@ -74,18 +76,12 @@ func (p *TestPaceMaker) LastViewTC() *flow.TimeoutCertificate { // using a real pacemaker for testing event handler func initPaceMaker(t require.TestingT, ctx context.Context, livenessData *hotstuff.LivenessData) hotstuff.PaceMaker { notifier := &mocks.Consumer{} - tc, err := timeout.NewConfig( - time.Duration(minRepTimeout*1e6), - time.Duration(maxRepTimeout*1e6), - multiplicativeIncrease, - happyPathMaxRoundFailures, - 0, - time.Duration(maxRepTimeout*1e6)) + tc, err := timeout.NewConfig(time.Duration(minRepTimeout*1e6), time.Duration(maxRepTimeout*1e6), multiplicativeIncrease, happyPathMaxRoundFailures, time.Duration(maxRepTimeout*1e6)) require.NoError(t, err) persist := &mocks.Persister{} persist.On("PutLivenessData", mock.Anything).Return(nil).Maybe() persist.On("GetLivenessData").Return(livenessData, nil).Once() - pm := NewTestPaceMaker(timeout.NewController(tc), notifier, persist) + pm := NewTestPaceMaker(timeout.NewController(tc), pacemaker.NoProposalDelay(), notifier, persist) notifier.On("OnStartingTimeout", mock.Anything).Return() notifier.On("OnQcTriggeredViewChange", mock.Anything, mock.Anything, mock.Anything).Return() notifier.On("OnTcTriggeredViewChange", mock.Anything, mock.Anything, mock.Anything).Return() @@ -168,22 +164,22 @@ func NewSafetyRules(t *testing.T) *SafetyRules { type Forks struct { *mocks.Forks // proposals stores all the proposals that have been added to the forks - proposals map[flow.Identifier]*model.Proposal + proposals map[flow.Identifier]*model.Block finalized uint64 t require.TestingT // addProposal is to customize the logic to change finalized view - addProposal func(block *model.Proposal) error + addProposal func(block *model.Block) error } func NewForks(t *testing.T, finalized uint64) *Forks { f := &Forks{ Forks: mocks.NewForks(t), - proposals: make(map[flow.Identifier]*model.Proposal), + proposals: make(map[flow.Identifier]*model.Block), finalized: finalized, } - f.On("AddProposal", mock.Anything).Return(func(proposal *model.Proposal) error { - log.Info().Msgf("forks.AddProposal received Proposal for view: %v, QC: %v\n", proposal.Block.View, proposal.Block.QC.View) + f.On("AddValidatedBlock", mock.Anything).Return(func(proposal *model.Block) error { + log.Info().Msgf("forks.AddValidatedBlock received Proposal for view: %v, QC: %v\n", proposal.View, proposal.QC.View) return f.addProposal(proposal) }).Maybe() @@ -191,33 +187,32 @@ func NewForks(t *testing.T, finalized uint64) *Forks { return f.finalized }).Maybe() - f.On("GetProposal", mock.Anything).Return(func(blockID flow.Identifier) *model.Proposal { + f.On("GetBlock", mock.Anything).Return(func(blockID flow.Identifier) *model.Block { b := f.proposals[blockID] return b }, func(blockID flow.Identifier) bool { b, ok := f.proposals[blockID] var view uint64 if ok { - view = b.Block.View + view = b.View } - log.Info().Msgf("forks.GetProposal found %v: view: %v\n", ok, view) + log.Info().Msgf("forks.GetBlock found %v: view: %v\n", ok, view) return ok }).Maybe() - f.On("GetProposalsForView", mock.Anything).Return(func(view uint64) []*model.Proposal { - proposals := make([]*model.Proposal, 0) + f.On("GetBlocksForView", mock.Anything).Return(func(view uint64) []*model.Block { + proposals := make([]*model.Block, 0) for _, b := range f.proposals { - if b.Block.View == view { + if b.View == view { proposals = append(proposals, b) } } - log.Info().Msgf("forks.GetProposalsForView found %v block(s) for view %v\n", len(proposals), view) + log.Info().Msgf("forks.GetBlocksForView found %v block(s) for view %v\n", len(proposals), view) return proposals }).Maybe() - f.addProposal = func(proposal *model.Proposal) error { - block := proposal.Block - f.proposals[block.BlockID] = proposal + f.addProposal = func(block *model.Block) error { + f.proposals[block.BlockID] = block if block.QC == nil { panic(fmt.Sprintf("block has no QC: %v", block.View)) } @@ -330,7 +325,7 @@ func (es *EventHandlerSuite) SetupTest() { } // add es.parentProposal into forks, otherwise we won't vote or propose based on it's QC sicne the parent is unknown - es.forks.proposals[es.parentProposal.Block.BlockID] = es.parentProposal + es.forks.proposals[es.parentProposal.Block.BlockID] = es.parentProposal.Block } // TestStartNewView_ParentProposalNotFound tests next scenario: constructed TC, it contains NewestQC that references block that we @@ -349,7 +344,7 @@ func (es *EventHandlerSuite) TestStartNewView_ParentProposalNotFound() { require.NoError(es.T(), err) require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.forks.AssertCalled(es.T(), "GetProposal", newestQC.BlockID) + es.forks.AssertCalled(es.T(), "GetBlock", newestQC.BlockID) es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) } @@ -371,7 +366,7 @@ func (es *EventHandlerSuite) TestOnReceiveProposal_QCOlderThanCurView() { err := es.eventhandler.OnReceiveProposal(proposal) require.NoError(es.T(), err) require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.forks.AssertCalled(es.T(), "AddProposal", proposal) + es.forks.AssertCalled(es.T(), "AddValidatedBlock", proposal.Block) } // TestOnReceiveProposal_TCOlderThanCurView tests scenario: received a valid proposal with QC and TC that has older view, @@ -384,7 +379,7 @@ func (es *EventHandlerSuite) TestOnReceiveProposal_TCOlderThanCurView() { err := es.eventhandler.OnReceiveProposal(proposal) require.NoError(es.T(), err) require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.forks.AssertCalled(es.T(), "AddProposal", proposal) + es.forks.AssertCalled(es.T(), "AddValidatedBlock", proposal.Block) } // TestOnReceiveProposal_NoVote tests scenario: received a valid proposal for cur view, but not a safe node to vote, and I'm the next leader @@ -398,7 +393,7 @@ func (es *EventHandlerSuite) TestOnReceiveProposal_NoVote() { err := es.eventhandler.OnReceiveProposal(proposal) require.NoError(es.T(), err) require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.forks.AssertCalled(es.T(), "AddProposal", proposal) + es.forks.AssertCalled(es.T(), "AddValidatedBlock", proposal.Block) } // TestOnReceiveProposal_NoVote_ParentProposalNotFound tests scenario: received a valid proposal for cur view, no parent for this proposal found @@ -413,7 +408,7 @@ func (es *EventHandlerSuite) TestOnReceiveProposal_NoVote_ParentProposalNotFound err := es.eventhandler.OnReceiveProposal(proposal) require.Error(es.T(), err) require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.forks.AssertCalled(es.T(), "AddProposal", proposal) + es.forks.AssertCalled(es.T(), "AddValidatedBlock", proposal.Block) } // TestOnReceiveProposal_Vote_NextLeader tests scenario: received a valid proposal for cur view, safe to vote, I'm the next leader @@ -521,7 +516,7 @@ func (es *EventHandlerSuite) TestOnReceiveProposal_ProposeAfterReceivingTC() { // round, so no proposal is expected. func (es *EventHandlerSuite) TestOnReceiveQc_HappyPath() { // voting block exists - es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal + es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal.Block // a qc is built qc := createQC(es.votingProposal.Block) @@ -563,9 +558,9 @@ func (es *EventHandlerSuite) TestOnReceiveQc_FutureView() { qc3 := createQC(b3.Block) // all three proposals are known - es.forks.proposals[b1.Block.BlockID] = b1 - es.forks.proposals[b2.Block.BlockID] = b2 - es.forks.proposals[b3.Block.BlockID] = b3 + es.forks.proposals[b1.Block.BlockID] = b1.Block + es.forks.proposals[b2.Block.BlockID] = b2.Block + es.forks.proposals[b3.Block.BlockID] = b3.Block // test that qc for future view should trigger view change err := es.eventhandler.OnReceiveQc(qc3) @@ -617,7 +612,7 @@ func (es *EventHandlerSuite) TestOnReceiveQc_NextLeaderProposes() { require.NoError(es.T(), err) require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.forks.AssertCalled(es.T(), "AddProposal", proposal) + es.forks.AssertCalled(es.T(), "AddValidatedBlock", proposal.Block) } // TestOnReceiveQc_ProposeOnce tests that after constructing proposal we don't attempt to create another @@ -648,7 +643,7 @@ func (es *EventHandlerSuite) TestOnReceiveQc_ProposeOnce() { // TestOnTCConstructed_HappyPath tests that building a TC for current view triggers view change func (es *EventHandlerSuite) TestOnReceiveTc_HappyPath() { // voting block exists - es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal + es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal.Block // a tc is built tc := helper.MakeTC(helper.WithTCView(es.initView), helper.WithTCNewestQC(es.votingProposal.Block.QC)) @@ -707,7 +702,7 @@ func (es *EventHandlerSuite) TestOnTimeout() { // need to make sure that EventHandler filters out TC for last view if we know about QC for same view. func (es *EventHandlerSuite) TestOnTimeout_SanityChecks() { // voting block exists - es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal + es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal.Block // a tc is built tc := helper.MakeTC(helper.WithTCView(es.initView), helper.WithTCNewestQC(es.votingProposal.Block.QC)) @@ -785,13 +780,11 @@ func (es *EventHandlerSuite) TestLeaderBuild100Blocks() { // for first proposal we need to store the parent otherwise it won't be voted for if i == 0 { - parentBlock := helper.MakeProposal( - helper.WithBlock( - helper.MakeBlock(func(block *model.Block) { - block.BlockID = proposal.Block.QC.BlockID - block.View = proposal.Block.QC.View - }))) - es.forks.proposals[parentBlock.Block.BlockID] = parentBlock + parentBlock := helper.MakeBlock(func(block *model.Block) { + block.BlockID = proposal.Block.QC.BlockID + block.View = proposal.Block.QC.View + }) + es.forks.proposals[parentBlock.BlockID] = parentBlock } es.safetyRules.votable[proposal.Block.BlockID] = struct{}{} @@ -819,7 +812,7 @@ func (es *EventHandlerSuite) TestLeaderBuild100Blocks() { func (es *EventHandlerSuite) TestFollowerFollows100Blocks() { // add parent proposal otherwise we can't propose parentProposal := createProposal(es.initView, es.initView-1) - es.forks.proposals[parentProposal.Block.BlockID] = parentProposal + es.forks.proposals[parentProposal.Block.BlockID] = parentProposal.Block for i := 0; i < 100; i++ { // create each proposal as if they are created by some leader proposal := createProposal(es.initView+uint64(i)+1, es.initView+uint64(i)) @@ -849,68 +842,31 @@ func (es *EventHandlerSuite) TestFollowerReceives100Forks() { require.Equal(es.T(), 100, len(es.forks.proposals)-1) } -// TestStart_PendingBlocksRecovery tests a scenario where node has unprocessed pending proposals that were not processed -// by event handler yet. After startup, we need to process all pending proposals. -func (es *EventHandlerSuite) TestStart_PendingBlocksRecovery() { - - var pendingProposals []*model.Proposal - proposal := createProposal(es.initView+1, es.initView) - pendingProposals = append(pendingProposals, proposal) - proposalWithTC := helper.MakeProposal(helper.WithBlock( - helper.MakeBlock( - helper.WithBlockView(es.initView+10), - helper.WithBlockQC(proposal.Block.QC))), - func(proposal *model.Proposal) { - proposal.LastViewTC = helper.MakeTC( - helper.WithTCView(proposal.Block.View-1), - helper.WithTCNewestQC(proposal.Block.QC)) - }, - ) - pendingProposals = append(pendingProposals, proposalWithTC) - proposal = createProposal(proposalWithTC.Block.View+1, proposalWithTC.Block.View) - pendingProposals = append(pendingProposals, proposal) - - for _, proposal := range pendingProposals { - es.forks.proposals[proposal.Block.BlockID] = proposal - } - - lastProposal := pendingProposals[len(pendingProposals)-1] - es.endView = lastProposal.Block.View - - es.forks.On("NewestView").Return(es.endView).Once() - - err := es.eventhandler.Start(es.ctx) - require.NoError(es.T(), err) - require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") -} - // TestStart_ProposeOnce tests that after starting event handler we don't create proposal in case we have already proposed // for this view. func (es *EventHandlerSuite) TestStart_ProposeOnce() { // I'm the next leader es.committee.leaders[es.initView+1] = struct{}{} - es.endView++ + // STEP 1: simulating events _before_ a crash: EventHandler receives proposal and then a QC for the proposal (from VoteAggregator) es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Once() - err := es.eventhandler.OnReceiveProposal(es.votingProposal) require.NoError(es.T(), err) // constructing QC triggers making block proposal err = es.eventhandler.OnReceiveQc(es.qc) require.NoError(es.T(), err) - es.notifier.AssertNumberOfCalls(es.T(), "OnOwnProposal", 1) - es.forks.On("NewestView").Return(es.endView).Once() - - // Start triggers proposing logic, make sure that we don't propose again. + // Here, a hypothetical crash would happen. + // During crash recovery, Forks and PaceMaker are recovered to have exactly the same in-memory state as before + // Start triggers proposing logic. But as our own proposal for the view is already in Forks, we should not propose again. err = es.eventhandler.Start(es.ctx) require.NoError(es.T(), err) - require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - // assert that broadcast wasn't trigger again + + // assert that broadcast wasn't trigger again, i.e. there should have been only one event `OnOwnProposal` in total es.notifier.AssertNumberOfCalls(es.T(), "OnOwnProposal", 1) } @@ -921,7 +877,7 @@ func (es *EventHandlerSuite) TestCreateProposal_SanityChecks() { tc := helper.MakeTC(helper.WithTCView(es.initView), helper.WithTCNewestQC(helper.MakeQC(helper.WithQCBlock(es.votingProposal.Block)))) - es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal + es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal.Block // I'm the next leader es.committee.leaders[tc.View+1] = struct{}{} diff --git a/consensus/hotstuff/eventloop/event_loop.go b/consensus/hotstuff/eventloop/event_loop.go index 95a06db8fda..d7bc478490b 100644 --- a/consensus/hotstuff/eventloop/event_loop.go +++ b/consensus/hotstuff/eventloop/event_loop.go @@ -32,6 +32,7 @@ type EventLoop struct { log zerolog.Logger eventHandler hotstuff.EventHandler metrics module.HotstuffMetrics + mempoolMetrics module.MempoolMetrics proposals chan queuedProposal newestSubmittedTc *tracker.NewestTCTracker newestSubmittedQc *tracker.NewestQCTracker @@ -46,19 +47,25 @@ var _ hotstuff.EventLoop = (*EventLoop)(nil) var _ component.Component = (*EventLoop)(nil) // NewEventLoop creates an instance of EventLoop. -func NewEventLoop(log zerolog.Logger, metrics module.HotstuffMetrics, eventHandler hotstuff.EventHandler, startTime time.Time) (*EventLoop, error) { +func NewEventLoop( + log zerolog.Logger, + metrics module.HotstuffMetrics, + mempoolMetrics module.MempoolMetrics, + eventHandler hotstuff.EventHandler, + startTime time.Time, +) (*EventLoop, error) { // we will use a buffered channel to avoid blocking of caller // we can't afford to drop messages since it undermines liveness, but we also want to avoid blocking of compliance // engine. We assume that we should be able to process proposals faster than compliance engine feeds them, worst case // we will fill the buffer and block compliance engine worker but that should happen only if compliance engine receives // large number of blocks in short period of time(when catching up for instance). - // TODO(active-pacemaker) add metrics for length of inbound channels proposals := make(chan queuedProposal, 1000) el := &EventLoop{ log: log, eventHandler: eventHandler, metrics: metrics, + mempoolMetrics: mempoolMetrics, proposals: proposals, tcSubmittedNotifier: engine.NewNotifier(), qcSubmittedNotifier: engine.NewNotifier(), @@ -92,18 +99,18 @@ func NewEventLoop(log zerolog.Logger, metrics module.HotstuffMetrics, eventHandl return el, nil } +// loop executes the core HotStuff logic in a single thread. It picks inputs from the various +// inbound channels and executes the EventHandler's respective method for processing this input. +// During normal operations, the EventHandler is not expected to return any errors, as all inputs +// are assumed to be fully validated (or produced by trusted components within the node). Therefore, +// any error is a symptom of state corruption, bugs or violation of API contracts. In all cases, +// continuing operations is not an option, i.e. we exit the event loop and return an exception. func (el *EventLoop) loop(ctx context.Context) error { err := el.eventHandler.Start(ctx) // must be called by the same go-routine that also executes the business logic! if err != nil { return fmt.Errorf("could not start event handler: %w", err) } - // hotstuff will run in an event loop to process all events synchronously. And this is what will happen when hitting errors: - // if hotstuff hits a known critical error, it will exit the loop (for instance, there is a conflicting block with a QC against finalized blocks - // if hotstuff hits a known error indicating some assumption between components is broken, it will exit the loop (for instance, hotstuff receives a block whose parent is missing) - // if hotstuff hits a known error that is safe to be ignored, it will not exit the loop (for instance, invalid proposal) - // if hotstuff hits any unknown error, it will exit the loop - shutdownSignaled := ctx.Done() timeoutCertificates := el.tcSubmittedNotifier.Channel() quorumCertificates := el.qcSubmittedNotifier.Channel() @@ -129,39 +136,34 @@ func (el *EventLoop) loop(ctx context.Context) error { case <-timeoutChannel: processStart := time.Now() - err := el.eventHandler.OnLocalTimeout() - - // measure how long it takes for a timeout event to be processed - el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeLocalTimeout) - + err = el.eventHandler.OnLocalTimeout() if err != nil { return fmt.Errorf("could not process timeout: %w", err) } + // measure how long it takes for a timeout event to be processed + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeLocalTimeout) // At this point, we have received and processed an event from the timeout channel. - // A timeout also means, we have made progress. A new timeout will have - // been started and el.eventHandler.TimeoutChannel() will be a NEW channel (for the just-started timeout) + // A timeout also means that we have made progress. A new timeout will have + // been started and el.eventHandler.TimeoutChannel() will be a NEW channel (for the just-started timeout). // Very important to start the for loop from the beginning, to continue the with the new timeout channel! continue case <-partialTCs: processStart := time.Now() - err := el.eventHandler.OnPartialTcCreated(el.newestSubmittedPartialTc.NewestPartialTc()) - - // measure how long it takes for a partial TC to be processed - el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnPartialTc) - + err = el.eventHandler.OnPartialTcCreated(el.newestSubmittedPartialTc.NewestPartialTc()) if err != nil { return fmt.Errorf("could no process partial created TC event: %w", err) } + // measure how long it takes for a partial TC to be processed + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnPartialTc) // At this point, we have received and processed partial TC event, it could have resulted in several scenarios: // 1. a view change with potential voting or proposal creation // 2. a created and broadcast timeout object // 3. QC and TC didn't result in view change and no timeout was created since we have already timed out or // the partial TC was created for view different from current one. - continue default: @@ -184,15 +186,12 @@ func (el *EventLoop) loop(ctx context.Context) error { el.metrics.HotStuffIdleDuration(time.Since(idleStart)) processStart := time.Now() - - err := el.eventHandler.OnLocalTimeout() - - // measure how long it takes for a timeout event to be processed - el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeLocalTimeout) - + err = el.eventHandler.OnLocalTimeout() if err != nil { return fmt.Errorf("could not process timeout: %w", err) } + // measure how long it takes for a timeout event to be processed + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeLocalTimeout) // if we have a new proposal, process it case queuedItem := <-el.proposals: @@ -205,17 +204,13 @@ func (el *EventLoop) loop(ctx context.Context) error { el.metrics.HotStuffIdleDuration(time.Since(idleStart)) processStart := time.Now() - proposal := queuedItem.proposal - - err := el.eventHandler.OnReceiveProposal(proposal) - - // measure how long it takes for a proposal to be processed - el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnProposal) - + err = el.eventHandler.OnReceiveProposal(proposal) if err != nil { return fmt.Errorf("could not process proposal %v: %w", proposal.Block.BlockID, err) } + // measure how long it takes for a proposal to be processed + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnProposal) el.log.Info(). Dur("dur_ms", time.Since(processStart)). @@ -230,14 +225,12 @@ func (el *EventLoop) loop(ctx context.Context) error { el.metrics.HotStuffIdleDuration(time.Since(idleStart)) processStart := time.Now() - err := el.eventHandler.OnReceiveQc(el.newestSubmittedQc.NewestQC()) - - // measure how long it takes for a QC to be processed - el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnQC) - + err = el.eventHandler.OnReceiveQc(el.newestSubmittedQc.NewestQC()) if err != nil { return fmt.Errorf("could not process QC: %w", err) } + // measure how long it takes for a QC to be processed + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnQC) // if we have a new TC, process it case <-timeoutCertificates: @@ -246,14 +239,12 @@ func (el *EventLoop) loop(ctx context.Context) error { el.metrics.HotStuffIdleDuration(time.Since(idleStart)) processStart := time.Now() - err := el.eventHandler.OnReceiveTc(el.newestSubmittedTc.NewestTC()) - - // measure how long it takes for a TC to be processed - el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnTC) - + err = el.eventHandler.OnReceiveTc(el.newestSubmittedTc.NewestTC()) if err != nil { return fmt.Errorf("could not process TC: %w", err) } + // measure how long it takes for a TC to be processed + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnTC) case <-partialTCs: // measure how long the event loop was idle waiting for an @@ -261,14 +252,12 @@ func (el *EventLoop) loop(ctx context.Context) error { el.metrics.HotStuffIdleDuration(time.Since(idleStart)) processStart := time.Now() - err := el.eventHandler.OnPartialTcCreated(el.newestSubmittedPartialTc.NewestPartialTc()) - - // measure how long it takes for a partial TC to be processed - el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnPartialTc) - + err = el.eventHandler.OnPartialTcCreated(el.newestSubmittedPartialTc.NewestPartialTc()) if err != nil { return fmt.Errorf("could no process partial created TC event: %w", err) } + // measure how long it takes for a partial TC to be processed + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnPartialTc) } } } @@ -284,7 +273,7 @@ func (el *EventLoop) SubmitProposal(proposal *model.Proposal) { case <-el.ComponentManager.ShutdownSignal(): return } - + el.mempoolMetrics.MempoolEntries(metrics.HotstuffEventTypeOnProposal, uint(len(el.proposals))) } // onTrustedQC pushes the received QC(which MUST be validated) to the quorumCertificates channel @@ -331,7 +320,13 @@ func (el *EventLoop) OnNewTcDiscovered(tc *flow.TimeoutCertificate) { el.onTrustedTC(tc) } -// OnQcConstructedFromVotes implements hotstuff.QCCreatedConsumer and pushes received qc into processing pipeline. +// OnQcConstructedFromVotes implements hotstuff.VoteCollectorConsumer and pushes received qc into processing pipeline. func (el *EventLoop) OnQcConstructedFromVotes(qc *flow.QuorumCertificate) { el.onTrustedQC(qc) } + +// OnTimeoutProcessed implements hotstuff.TimeoutCollectorConsumer and is no-op +func (el *EventLoop) OnTimeoutProcessed(timeout *model.TimeoutObject) {} + +// OnVoteProcessed implements hotstuff.VoteCollectorConsumer and is no-op +func (el *EventLoop) OnVoteProcessed(vote *model.Vote) {} diff --git a/consensus/hotstuff/eventloop/event_loop_test.go b/consensus/hotstuff/eventloop/event_loop_test.go index 3f63b76f8d9..8b6eeed5b25 100644 --- a/consensus/hotstuff/eventloop/event_loop_test.go +++ b/consensus/hotstuff/eventloop/event_loop_test.go @@ -46,7 +46,7 @@ func (s *EventLoopTestSuite) SetupTest() { log := zerolog.New(io.Discard) - eventLoop, err := NewEventLoop(log, metrics.NewNoopCollector(), s.eh, time.Time{}) + eventLoop, err := NewEventLoop(log, metrics.NewNoopCollector(), metrics.NewNoopCollector(), s.eh, time.Time{}) require.NoError(s.T(), err) s.eventLoop = eventLoop @@ -200,7 +200,8 @@ func TestEventLoop_Timeout(t *testing.T) { log := zerolog.New(io.Discard) - eventLoop, err := NewEventLoop(log, metrics.NewNoopCollector(), eh, time.Time{}) + metricsCollector := metrics.NewNoopCollector() + eventLoop, err := NewEventLoop(log, metricsCollector, metricsCollector, eh, time.Time{}) require.NoError(t, err) eh.On("TimeoutChannel").Return(time.After(100 * time.Millisecond)) @@ -253,7 +254,7 @@ func TestReadyDoneWithStartTime(t *testing.T) { startTimeDuration := 2 * time.Second startTime := time.Now().Add(startTimeDuration) - eventLoop, err := NewEventLoop(log, metrics, eh, startTime) + eventLoop, err := NewEventLoop(log, metrics, metrics, eh, startTime) require.NoError(t, err) done := make(chan struct{}) diff --git a/consensus/hotstuff/follower/follower.go b/consensus/hotstuff/follower/follower.go deleted file mode 100644 index cef8b3d0c1b..00000000000 --- a/consensus/hotstuff/follower/follower.go +++ /dev/null @@ -1,82 +0,0 @@ -package follower - -import ( - "errors" - "fmt" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/utils/logging" -) - -// FollowerLogic runs in non-consensus nodes. It informs other components within the node -// about finalization of blocks. The consensus Follower consumes all block proposals -// broadcasts by the consensus node, verifies the block header and locally evaluates -// the finalization rules. -// -// CAUTION: Follower is NOT CONCURRENCY safe -type FollowerLogic struct { - log zerolog.Logger - validator hotstuff.Validator - finalizationLogic hotstuff.Forks -} - -// New creates a new FollowerLogic instance -func New( - log zerolog.Logger, - validator hotstuff.Validator, - finalizationLogic hotstuff.Forks, -) (*FollowerLogic, error) { - return &FollowerLogic{ - log: log.With().Str("hotstuff", "follower").Logger(), - validator: validator, - finalizationLogic: finalizationLogic, - }, nil -} - -// FinalizedBlock returns the latest finalized block -func (f *FollowerLogic) FinalizedBlock() *model.Block { - return f.finalizationLogic.FinalizedBlock() -} - -// AddBlock processes the given block proposal -func (f *FollowerLogic) AddBlock(blockProposal *model.Proposal) error { - // validate the block. skip if the proposal is invalid - err := f.validator.ValidateProposal(blockProposal) - if err != nil { - if model.IsInvalidBlockError(err) { - f.log.Warn().Err(err). - Hex("block_id", logging.ID(blockProposal.Block.BlockID)). - Msg("invalid proposal") - return nil - } else if errors.Is(err, model.ErrViewForUnknownEpoch) { - f.log.Warn().Err(err). - Hex("block_id", logging.ID(blockProposal.Block.BlockID)). - Hex("qc_block_id", logging.ID(blockProposal.Block.QC.BlockID)). - Uint64("block_view", blockProposal.Block.View). - Msg("proposal for unknown epoch") - return nil - } else if errors.Is(err, model.ErrUnverifiableBlock) { - f.log.Warn().Err(err). - Hex("block_id", logging.ID(blockProposal.Block.BlockID)). - Hex("qc_block_id", logging.ID(blockProposal.Block.QC.BlockID)). - Uint64("block_view", blockProposal.Block.View). - Msg("unverifiable proposal") - // even if the block is unverifiable because the QC has been - // pruned, it still needs to be added to the forks, otherwise, - // a new block with a QC to this block will fail to be added - // to forks and crash the event loop. - } else if err != nil { - return fmt.Errorf("cannot validate block proposal %x: %w", blockProposal.Block.BlockID, err) - } - } - - err = f.finalizationLogic.AddProposal(blockProposal) - if err != nil { - return fmt.Errorf("finalization logic cannot process block proposal %x: %w", blockProposal.Block.BlockID, err) - } - - return nil -} diff --git a/consensus/hotstuff/follower_logic.go b/consensus/hotstuff/follower_logic.go deleted file mode 100644 index cebddc33604..00000000000 --- a/consensus/hotstuff/follower_logic.go +++ /dev/null @@ -1,14 +0,0 @@ -package hotstuff - -import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" -) - -// FollowerLogic runs a state machine to process proposals -type FollowerLogic interface { - // FinalizedBlock returns the latest finalized block - FinalizedBlock() *model.Block - - // AddBlock processes a block proposal - AddBlock(proposal *model.Proposal) error -} diff --git a/consensus/hotstuff/follower_loop.go b/consensus/hotstuff/follower_loop.go index ae9289c1860..1cde8c8a629 100644 --- a/consensus/hotstuff/follower_loop.go +++ b/consensus/hotstuff/follower_loop.go @@ -1,6 +1,7 @@ package hotstuff import ( + "fmt" "time" "github.com/rs/zerolog" @@ -9,6 +10,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/utils/logging" ) @@ -18,24 +20,29 @@ import ( // Concurrency safe. type FollowerLoop struct { *component.ComponentManager - log zerolog.Logger - followerLogic FollowerLogic - proposals chan *model.Proposal + log zerolog.Logger + mempoolMetrics module.MempoolMetrics + certifiedBlocks chan *model.CertifiedBlock + forks Forks } var _ component.Component = (*FollowerLoop)(nil) var _ module.HotStuffFollower = (*FollowerLoop)(nil) -// NewFollowerLoop creates an instance of EventLoop -func NewFollowerLoop(log zerolog.Logger, followerLogic FollowerLogic) (*FollowerLoop, error) { - // TODO(active-pacemaker) add metrics for length of inbound channels - // we will use a buffered channel to avoid blocking of caller - proposals := make(chan *model.Proposal, 1000) +// NewFollowerLoop creates an instance of HotStuffFollower +func NewFollowerLoop(log zerolog.Logger, mempoolMetrics module.MempoolMetrics, forks Forks) (*FollowerLoop, error) { + // We can't afford to drop messages since it undermines liveness, but we also want to avoid blocking + // the compliance layer. Generally, the follower loop should be able to process inbound blocks faster + // than they pass through the compliance layer. Nevertheless, in the worst case we will fill the + // channel and block the compliance layer's workers. Though, that should happen only if compliance + // engine receives large number of blocks in short periods of time (e.g. when catching up). + certifiedBlocks := make(chan *model.CertifiedBlock, 1000) fl := &FollowerLoop{ - log: log, - followerLogic: followerLogic, - proposals: proposals, + log: log.With().Str("hotstuff", "FollowerLoop").Logger(), + mempoolMetrics: mempoolMetrics, + certifiedBlocks: certifiedBlocks, + forks: forks, } fl.ComponentManager = component.NewComponentManagerBuilder(). @@ -45,16 +52,25 @@ func NewFollowerLoop(log zerolog.Logger, followerLogic FollowerLogic) (*Follower return fl, nil } -// SubmitProposal feeds a new block proposal (header) into the FollowerLoop. -// This method blocks until the proposal is accepted to the event queue. +// AddCertifiedBlock appends the given certified block to the tree of pending +// blocks and updates the latest finalized block (if finalization progressed). +// Unless the parent is below the pruning threshold (latest finalized view), we +// require that the parent has previously been added. // -// Block proposals must be submitted in order, i.e. a proposal's parent must -// have been previously processed by the FollowerLoop. -func (fl *FollowerLoop) SubmitProposal(proposal *model.Proposal) { +// Notes: +// - Under normal operations, this method is non-blocking. The follower internally +// queues incoming blocks and processes them in its own worker routine. However, +// when the inbound queue is, we block until there is space in the queue. This +// behavior is intentional, because we cannot drop blocks (otherwise, we would +// cause disconnected blocks). Instead, we simply block the compliance layer to +// avoid any pathological edge cases. +// - Blocks whose views are below the latest finalized view are dropped. +// - Inputs are idempotent (repetitions are no-ops). +func (fl *FollowerLoop) AddCertifiedBlock(certifiedBlock *model.CertifiedBlock) { received := time.Now() select { - case fl.proposals <- proposal: + case fl.certifiedBlocks <- certifiedBlock: case <-fl.ComponentManager.ShutdownSignal(): return } @@ -62,10 +78,14 @@ func (fl *FollowerLoop) SubmitProposal(proposal *model.Proposal) { // the busy duration is measured as how long it takes from a block being // received to a block being handled by the event handler. busyDuration := time.Since(received) - fl.log.Debug().Hex("block_id", logging.ID(proposal.Block.BlockID)). - Uint64("view", proposal.Block.View). - Dur("busy_duration", busyDuration). - Msg("busy duration to handle a proposal") + + blocksQueued := uint(len(fl.certifiedBlocks)) + fl.mempoolMetrics.MempoolEntries(metrics.ResourceFollowerLoopCertifiedBlocksChannel, blocksQueued) + fl.log.Debug().Hex("block_id", logging.ID(certifiedBlock.ID())). + Uint64("view", certifiedBlock.View()). + Uint("blocks_queued", blocksQueued). + Dur("wait_time", busyDuration). + Msg("wait time to queue inbound certified block") } // loop will synchronously process all events. @@ -83,12 +103,13 @@ func (fl *FollowerLoop) loop(ctx irrecoverable.SignalerContext, ready component. } select { - case p := <-fl.proposals: - err := fl.followerLogic.AddBlock(p) + case b := <-fl.certifiedBlocks: + err := fl.forks.AddCertifiedBlock(b) if err != nil { // all errors are fatal + err = fmt.Errorf("finalization logic failes to process certified block %v: %w", b.ID(), err) fl.log.Error(). - Hex("block_id", logging.ID(p.Block.BlockID)). - Uint64("view", p.Block.View). + Hex("block_id", logging.ID(b.ID())). + Uint64("view", b.View()). Err(err). Msg("irrecoverable follower loop error") ctx.Throw(err) diff --git a/consensus/hotstuff/forks.go b/consensus/hotstuff/forks.go index 8cdbdc241d2..5940eb35789 100644 --- a/consensus/hotstuff/forks.go +++ b/consensus/hotstuff/forks.go @@ -5,7 +5,16 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// Forks maintains an in-memory data-structure of all proposals whose view-number is larger or equal to +// FinalityProof represents a finality proof for a Block. By convention, a FinalityProof +// is immutable. Finality in Jolteon/HotStuff is determined by the 2-chain rule: +// +// There exists a _certified_ block C, such that Block.View + 1 = C.View +type FinalityProof struct { + Block *model.Block + CertifiedChild model.CertifiedBlock +} + +// Forks maintains an in-memory data-structure of all blocks whose view-number is larger or equal to // the latest finalized block. The latest finalized block is defined as the finalized block with the largest view number. // When adding blocks, Forks automatically updates its internal state (including finalized blocks). // Furthermore, blocks whose view number is smaller than the latest finalized block are pruned automatically. @@ -16,12 +25,12 @@ import ( // and ignore the block. type Forks interface { - // GetProposalsForView returns all BlockProposals at the given view number. - GetProposalsForView(view uint64) []*model.Proposal + // GetBlocksForView returns all known blocks for the given view + GetBlocksForView(view uint64) []*model.Block - // GetProposal returns (BlockProposal, true) if the block with the specified - // id was found (nil, false) otherwise. - GetProposal(id flow.Identifier) (*model.Proposal, bool) + // GetBlock returns (BlockProposal, true) if the block with the specified + // id was found and (nil, false) otherwise. + GetBlock(blockID flow.Identifier) (*model.Block, bool) // FinalizedView returns the largest view number where a finalized block is known FinalizedView() uint64 @@ -29,16 +38,58 @@ type Forks interface { // FinalizedBlock returns the finalized block with the largest view number FinalizedBlock() *model.Block - // NewestView returns the largest view number of all proposals that were added to Forks. - NewestView() uint64 - - // AddProposal adds the block proposal to Forks. This might cause an update of the finalized block - // and pruning of older blocks. - // Handles duplicated addition of blocks (at the potential cost of additional computation time). - // PREREQUISITE: - // Forks must be able to connect `proposal` to its latest finalized block - // (without missing interim ancestors). Otherwise, an exception is raised. - // Expected errors during normal operations: - // * model.ByzantineThresholdExceededError - new block results in conflicting finalized blocks - AddProposal(proposal *model.Proposal) error + // FinalityProof returns the latest finalized block and a certified child from + // the subsequent view, which proves finality. + // CAUTION: method returns (nil, false), when Forks has not yet finalized any + // blocks beyond the finalized root block it was initialized with. + FinalityProof() (*FinalityProof, bool) + + // AddValidatedBlock appends the validated block to the tree of pending + // blocks and updates the latest finalized block (if applicable). Unless the parent is + // below the pruning threshold (latest finalized view), we require that the parent is + // already stored in Forks. Calling this method with previously processed blocks + // leaves the consensus state invariant (though, it will potentially cause some + // duplicate processing). + // Notes: + // - Method `AddCertifiedBlock(..)` should be used preferably, if a QC certifying + // `block` is already known. This is generally the case for the consensus follower. + // Method `AddValidatedBlock` is intended for active consensus participants, which fully + // validate blocks (incl. payload), i.e. QCs are processed as part of validated proposals. + // + // Possible error returns: + // - model.MissingBlockError if the parent does not exist in the forest (but is above + // the pruned view). From the perspective of Forks, this error is benign (no-op). + // - model.InvalidBlockError if the block is invalid (see `Forks.EnsureBlockIsValidExtension` + // for details). From the perspective of Forks, this error is benign (no-op). However, we + // assume all blocks are fully verified, i.e. they should satisfy all consistency + // requirements. Hence, this error is likely an indicator of a bug in the compliance layer. + // - model.ByzantineThresholdExceededError if conflicting QCs or conflicting finalized + // blocks have been detected (violating a foundational consensus guarantees). This + // indicates that there are 1/3+ Byzantine nodes (weighted by stake) in the network, + // breaking the safety guarantees of HotStuff (or there is a critical bug / data + // corruption). Forks cannot recover from this exception. + // - All other errors are potential symptoms of bugs or state corruption. + AddValidatedBlock(proposal *model.Block) error + + // AddCertifiedBlock appends the given certified block to the tree of pending + // blocks and updates the latest finalized block (if finalization progressed). + // Unless the parent is below the pruning threshold (latest finalized view), we + // require that the parent is already stored in Forks. Calling this method with + // previously processed blocks leaves the consensus state invariant (though, + // it will potentially cause some duplicate processing). + // + // Possible error returns: + // - model.MissingBlockError if the parent does not exist in the forest (but is above + // the pruned view). From the perspective of Forks, this error is benign (no-op). + // - model.InvalidBlockError if the block is invalid (see `Forks.EnsureBlockIsValidExtension` + // for details). From the perspective of Forks, this error is benign (no-op). However, we + // assume all blocks are fully verified, i.e. they should satisfy all consistency + // requirements. Hence, this error is likely an indicator of a bug in the compliance layer. + // - model.ByzantineThresholdExceededError if conflicting QCs or conflicting finalized + // blocks have been detected (violating a foundational consensus guarantees). This + // indicates that there are 1/3+ Byzantine nodes (weighted by stake) in the network, + // breaking the safety guarantees of HotStuff (or there is a critical bug / data + // corruption). Forks cannot recover from this exception. + // - All other errors are potential symptoms of bugs or state corruption. + AddCertifiedBlock(certifiedBlock *model.CertifiedBlock) error } diff --git a/consensus/hotstuff/forks/block_builder_test.go b/consensus/hotstuff/forks/block_builder_test.go index 64844feb412..03daec535c1 100644 --- a/consensus/hotstuff/forks/block_builder_test.go +++ b/consensus/hotstuff/forks/block_builder_test.go @@ -75,9 +75,9 @@ func (bb *BlockBuilder) AddVersioned(qcView uint64, blockView uint64, qcVersion return bb } -// Blocks returns a list of all blocks added to the BlockBuilder. +// Proposals returns a list of all proposals added to the BlockBuilder. // Returns an error if the blocks do not form a connected tree rooted at genesis. -func (bb *BlockBuilder) Blocks() ([]*model.Proposal, error) { +func (bb *BlockBuilder) Proposals() ([]*model.Proposal, error) { blocks := make([]*model.Proposal, 0, len(bb.blockViews)) genesisBlock := makeGenesis() @@ -124,6 +124,16 @@ func (bb *BlockBuilder) Blocks() ([]*model.Proposal, error) { return blocks, nil } +// Blocks returns a list of all blocks added to the BlockBuilder. +// Returns an error if the blocks do not form a connected tree rooted at genesis. +func (bb *BlockBuilder) Blocks() ([]*model.Block, error) { + proposals, err := bb.Proposals() + if err != nil { + return nil, fmt.Errorf("BlockBuilder failed to generate proposals: %w", err) + } + return toBlocks(proposals), nil +} + func makePayloadHash(view uint64, qc *flow.QuorumCertificate, blockVersion int) flow.Identifier { return flow.MakeID(struct { View uint64 @@ -165,3 +175,12 @@ func makeGenesis() *model.CertifiedBlock { } return &certifiedGenesisBlock } + +// toBlocks converts the given proposals to slice of blocks +func toBlocks(proposals []*model.Proposal) []*model.Block { + blocks := make([]*model.Block, 0, len(proposals)) + for _, b := range proposals { + blocks = append(blocks, b.Block) + } + return blocks +} diff --git a/consensus/hotstuff/forks/blockcontainer.go b/consensus/hotstuff/forks/blockcontainer.go index b474a0827a0..799fa80bb17 100644 --- a/consensus/hotstuff/forks/blockcontainer.go +++ b/consensus/hotstuff/forks/blockcontainer.go @@ -8,30 +8,23 @@ import ( // BlockContainer wraps a block proposal to implement forest.Vertex // so the proposal can be stored in forest.LevelledForest -type BlockContainer struct { - Proposal *model.Proposal -} +type BlockContainer model.Block var _ forest.Vertex = (*BlockContainer)(nil) +func ToBlockContainer2(block *model.Block) *BlockContainer { return (*BlockContainer)(block) } +func (b *BlockContainer) Block() *model.Block { return (*model.Block)(b) } + // Functions implementing forest.Vertex +func (b *BlockContainer) VertexID() flow.Identifier { return b.BlockID } +func (b *BlockContainer) Level() uint64 { return b.View } -func (b *BlockContainer) VertexID() flow.Identifier { return b.Proposal.Block.BlockID } -func (b *BlockContainer) Level() uint64 { return b.Proposal.Block.View } func (b *BlockContainer) Parent() (flow.Identifier, uint64) { - return b.Proposal.Block.QC.BlockID, b.Proposal.Block.QC.View + // Caution: not all blocks have a QC for the parent, such as the spork root blocks. + // Per API contract, we are obliged to return a value to prevent panics during logging. + // (see vertex `forest.VertexToString` method). + if b.QC == nil { + return flow.ZeroID, 0 + } + return b.QC.BlockID, b.QC.View } - -// BlockContainer wraps a block proposal to implement forest.Vertex -// so the proposal can be stored in forest.LevelledForest -type BlockContainer2 model.Block - -var _ forest.Vertex = (*BlockContainer2)(nil) - -func ToBlockContainer2(block *model.Block) *BlockContainer2 { return (*BlockContainer2)(block) } -func (b *BlockContainer2) Block() *model.Block { return (*model.Block)(b) } - -// Functions implementing forest.Vertex -func (b *BlockContainer2) VertexID() flow.Identifier { return b.BlockID } -func (b *BlockContainer2) Level() uint64 { return b.View } -func (b *BlockContainer2) Parent() (flow.Identifier, uint64) { return b.QC.BlockID, b.QC.View } diff --git a/consensus/hotstuff/forks/forks.go b/consensus/hotstuff/forks/forks.go index dd53916dc8c..aa4db7f9853 100644 --- a/consensus/hotstuff/forks/forks.go +++ b/consensus/hotstuff/forks/forks.go @@ -1,7 +1,6 @@ package forks import ( - "errors" "fmt" "github.com/onflow/flow-go/consensus/hotstuff" @@ -9,222 +8,318 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/forest" - "github.com/onflow/flow-go/module/mempool" ) -// ErrPrunedAncestry is a sentinel error: cannot resolve ancestry of block due to pruning -var ErrPrunedAncestry = errors.New("cannot resolve pruned ancestor") - -// ancestryChain encapsulates a block, its parent (oneChain) and its grand-parent (twoChain). -// Given a chain structure like: -// -// b <~ b' <~ b* -// -// where the QC certifying b is qc_b, this data structure looks like: -// -// twoChain oneChain block -// [b<-qc_b] [b'<-qc_b'] [b*] -type ancestryChain struct { - block *BlockContainer - oneChain *model.CertifiedBlock - twoChain *model.CertifiedBlock -} - // Forks enforces structural validity of the consensus state and implements // finalization rules as defined in Jolteon consensus https://arxiv.org/abs/2106.10362 // The same approach has later been adopted by the Diem team resulting in DiemBFT v4: // https://developers.diem.com/papers/diem-consensus-state-machine-replication-in-the-diem-blockchain/2021-08-17.pdf // Forks is NOT safe for concurrent use by multiple goroutines. type Forks struct { - notifier hotstuff.FinalizationConsumer - forest forest.LevelledForest - finalizationCallback module.Finalizer - newestView uint64 // newestView is the highest view of block proposal stored in Forks - lastFinalized *model.CertifiedBlock // the most recently finalized block and the QC that certifies it + notifier hotstuff.FollowerConsumer + forest forest.LevelledForest + trustedRoot *model.CertifiedBlock + + // finalityProof holds the latest finalized block including the certified child as proof of finality. + // CAUTION: is nil, when Forks has not yet finalized any blocks beyond the finalized root block it was initialized with + finalityProof *hotstuff.FinalityProof } var _ hotstuff.Forks = (*Forks)(nil) -func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.FinalizationConsumer) (*Forks, error) { +func New(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.FollowerConsumer) (*Forks, error) { if (trustedRoot.Block.BlockID != trustedRoot.CertifyingQC.BlockID) || (trustedRoot.Block.View != trustedRoot.CertifyingQC.View) { return nil, model.NewConfigurationErrorf("invalid root: root QC is not pointing to root block") } forks := Forks{ - notifier: notifier, finalizationCallback: finalizationCallback, + notifier: notifier, forest: *forest.NewLevelledForest(trustedRoot.Block.View), - lastFinalized: trustedRoot, - newestView: trustedRoot.Block.View, - } - - // CAUTION: instead of a proposal, we use a normal block (without `SigData` and `LastViewTC`, - // which would be possibly included in a full proposal). Per convention, we consider the - // root block as already committed and enter a higher view. - // Therefore, the root block's proposer signature and TC are irrelevant for consensus. - trustedRootProposal := &model.Proposal{ - Block: trustedRoot.Block, + trustedRoot: trustedRoot, + finalityProof: nil, } // verify and add root block to levelled forest - err := forks.VerifyProposal(trustedRootProposal) + err := forks.EnsureBlockIsValidExtension(trustedRoot.Block) if err != nil { - return nil, fmt.Errorf("invalid root block: %w", err) + return nil, fmt.Errorf("invalid root block %v: %w", trustedRoot.ID(), err) } - forks.forest.AddVertex(&BlockContainer{Proposal: trustedRootProposal}) + forks.forest.AddVertex(ToBlockContainer2(trustedRoot.Block)) return &forks, nil } -func (f *Forks) FinalizedBlock() *model.Block { return f.lastFinalized.Block } -func (f *Forks) FinalizedView() uint64 { return f.lastFinalized.Block.View } -func (f *Forks) NewestView() uint64 { return f.newestView } +// FinalizedView returns the largest view number where a finalized block is known +func (f *Forks) FinalizedView() uint64 { + if f.finalityProof == nil { + return f.trustedRoot.Block.View + } + return f.finalityProof.Block.View +} + +// FinalizedBlock returns the finalized block with the largest view number +func (f *Forks) FinalizedBlock() *model.Block { + if f.finalityProof == nil { + return f.trustedRoot.Block + } + return f.finalityProof.Block +} + +// FinalityProof returns the latest finalized block and a certified child from +// the subsequent view, which proves finality. +// CAUTION: method returns (nil, false), when Forks has not yet finalized any +// blocks beyond the finalized root block it was initialized with. +func (f *Forks) FinalityProof() (*hotstuff.FinalityProof, bool) { + return f.finalityProof, f.finalityProof != nil +} -// GetProposal returns block for given ID -func (f *Forks) GetProposal(blockID flow.Identifier) (*model.Proposal, bool) { +// GetBlock returns (BlockProposal, true) if the block with the specified +// id was found and (nil, false) otherwise. +func (f *Forks) GetBlock(blockID flow.Identifier) (*model.Block, bool) { blockContainer, hasBlock := f.forest.GetVertex(blockID) if !hasBlock { return nil, false } - return blockContainer.(*BlockContainer).Proposal, true + return blockContainer.(*BlockContainer).Block(), true } -// GetProposalsForView returns all known proposals for the given view -func (f *Forks) GetProposalsForView(view uint64) []*model.Proposal { +// GetBlocksForView returns all known blocks for the given view +func (f *Forks) GetBlocksForView(view uint64) []*model.Block { vertexIterator := f.forest.GetVerticesAtLevel(view) - l := make([]*model.Proposal, 0, 1) // in the vast majority of cases, there will only be one proposal for a particular view + blocks := make([]*model.Block, 0, 1) // in the vast majority of cases, there will only be one proposal for a particular view for vertexIterator.HasNext() { - v := vertexIterator.NextVertex().(*BlockContainer) - l = append(l, v.Proposal) + v := vertexIterator.NextVertex() + blocks = append(blocks, v.(*BlockContainer).Block()) } - return l -} - -// AddProposal adds proposal to the consensus state. Performs verification to make sure that we don't -// add invalid proposals into consensus state. -// We assume that all blocks are fully verified. A valid block must satisfy all consistency -// requirements; otherwise we have a bug in the compliance layer. -// Expected errors during normal operations: -// - model.ByzantineThresholdExceededError - new block results in conflicting finalized blocks -func (f *Forks) AddProposal(proposal *model.Proposal) error { - err := f.VerifyProposal(proposal) - if err != nil { - if model.IsMissingBlockError(err) { - return fmt.Errorf("cannot add proposal with missing parent: %s", err.Error()) - } - // technically, this not strictly required. However, we leave this as a sanity check for now - return fmt.Errorf("cannot add invalid proposal to Forks: %w", err) - } - err = f.UnverifiedAddProposal(proposal) - if err != nil { - return fmt.Errorf("error storing proposal in Forks: %w", err) - } - - return nil + return blocks } // IsKnownBlock checks whether block is known. -// UNVALIDATED: expects block to pass Forks.VerifyProposal(block) -func (f *Forks) IsKnownBlock(block *model.Block) bool { - _, hasBlock := f.forest.GetVertex(block.BlockID) +func (f *Forks) IsKnownBlock(blockID flow.Identifier) bool { + _, hasBlock := f.forest.GetVertex(blockID) return hasBlock } -// IsProcessingNeeded performs basic checks to determine whether block needs processing, -// only considering the block's height and hash. +// IsProcessingNeeded determines whether the given block needs processing, +// based on the block's view and hash. // Returns false if any of the following conditions applies // - block view is _below_ the most recently finalized block // - the block already exists in the consensus state // -// UNVALIDATED: expects block to pass Forks.VerifyProposal(block) +// UNVALIDATED: expects block to pass Forks.EnsureBlockIsValidExtension(block) func (f *Forks) IsProcessingNeeded(block *model.Block) bool { - if block.View < f.lastFinalized.Block.View || f.IsKnownBlock(block) { + if block.View < f.FinalizedView() || f.IsKnownBlock(block.BlockID) { return false } return true } -// UnverifiedAddProposal adds `proposal` to the consensus state and updates the -// latest finalized block, if possible. -// Calling this method with previously-processed blocks leaves the consensus state invariant -// (though, it will potentially cause some duplicate processing). -// UNVALIDATED: expects block to pass Forks.VerifyProposal(block) +// EnsureBlockIsValidExtension checks that the given block is a valid extension to the tree +// of blocks already stored (no state modifications). Specifically, the following conditions +// are enforced, which are critical to the correctness of Forks: +// +// 1. If a block with the same ID is already stored, their views must be identical. +// 2. The block's view must be strictly larger than the view of its parent. +// 3. The parent must already be stored (or below the pruning height). +// +// Exclusions to these rules (by design): +// Let W denote the view of block's parent (i.e. W := block.QC.View) and F the latest +// finalized view. +// +// (i) If block.View < F, adding the block would be a no-op. Such blocks are considered +// compatible (principle of vacuous truth), i.e. we skip checking 1, 2, 3. +// (ii) If block.View == F, we do not inspect the QC / parent at all (skip 2 and 3). +// This exception is important for compatability with genesis or spork-root blocks, +// which do not contain a QC. +// (iii) If block.View > F, but block.QC.View < F the parent has already been pruned. In +// this case, we omit rule 3. (principle of vacuous truth applied to the parent) +// +// We assume that all blocks are fully verified. A valid block must satisfy all consistency +// requirements; otherwise we have a bug in the compliance layer. +// // Error returns: -// * model.ByzantineThresholdExceededError if proposal's QC conflicts with an existing QC. -// * generic error in case of unexpected bug or internal state corruption -func (f *Forks) UnverifiedAddProposal(proposal *model.Proposal) error { - if !f.IsProcessingNeeded(proposal.Block) { +// - model.MissingBlockError if the parent of the input proposal does not exist in the +// forest (but is above the pruned view). Represents violation of condition 3. +// - model.InvalidBlockError if the block violates condition 1. or 2. +// - generic error in case of unexpected bug or internal state corruption +func (f *Forks) EnsureBlockIsValidExtension(block *model.Block) error { + if block.View < f.forest.LowestLevel { // exclusion (i) + return nil + } + + // LevelledForest enforces conditions 1. and 2. including the respective exclusions (ii) and (iii). + blockContainer := ToBlockContainer2(block) + err := f.forest.VerifyVertex(blockContainer) + if err != nil { + if forest.IsInvalidVertexError(err) { + return model.NewInvalidBlockErrorf(block, "not a valid vertex for block tree: %w", err) + } + return fmt.Errorf("block tree generated unexpected error validating vertex: %w", err) + } + + // Condition 3: + // LevelledForest implements a more generalized algorithm that also works for disjoint graphs. + // Therefore, LevelledForest _not_ enforce condition 3. Here, we additionally require that the + // pending blocks form a tree (connected graph), i.e. we need to enforce condition 3 + if (block.View == f.forest.LowestLevel) || (block.QC.View < f.forest.LowestLevel) { // exclusion (ii) and (iii) + return nil + } + // For a block whose parent is _not_ below the pruning height, we expect the parent to be known. + if _, isParentKnown := f.forest.GetVertex(block.QC.BlockID); !isParentKnown { // missing parent + return model.MissingBlockError{ + View: block.QC.View, + BlockID: block.QC.BlockID, + } + } + return nil +} + +// AddCertifiedBlock appends the given certified block to the tree of pending +// blocks and updates the latest finalized block (if finalization progressed). +// Unless the parent is below the pruning threshold (latest finalized view), we +// require that the parent is already stored in Forks. Calling this method with +// previously processed blocks leaves the consensus state invariant (though, +// it will potentially cause some duplicate processing). +// +// Possible error returns: +// - model.MissingBlockError if the parent does not exist in the forest (but is above +// the pruned view). From the perspective of Forks, this error is benign (no-op). +// - model.InvalidBlockError if the block is invalid (see `Forks.EnsureBlockIsValidExtension` +// for details). From the perspective of Forks, this error is benign (no-op). However, we +// assume all blocks are fully verified, i.e. they should satisfy all consistency +// requirements. Hence, this error is likely an indicator of a bug in the compliance layer. +// - model.ByzantineThresholdExceededError if conflicting QCs or conflicting finalized +// blocks have been detected (violating a foundational consensus guarantees). This +// indicates that there are 1/3+ Byzantine nodes (weighted by stake) in the network, +// breaking the safety guarantees of HotStuff (or there is a critical bug / data +// corruption). Forks cannot recover from this exception. +// - All other errors are potential symptoms of bugs or state corruption. +func (f *Forks) AddCertifiedBlock(certifiedBlock *model.CertifiedBlock) error { + if !f.IsProcessingNeeded(certifiedBlock.Block) { return nil } - blockContainer := &BlockContainer{Proposal: proposal} - block := blockContainer.Proposal.Block - err := f.checkForConflictingQCs(block.QC) + // Check proposal for byzantine evidence, store it and emit `OnBlockIncorporated` notification. + // Note: `checkForByzantineEvidence` only inspects the block, but _not_ its certifying QC. Hence, + // we have to additionally check here, whether the certifying QC conflicts with any known QCs. + err := f.checkForByzantineEvidence(certifiedBlock.Block) if err != nil { - return err + return fmt.Errorf("cannot check for Byzantine evidence in certified block %v: %w", certifiedBlock.Block.BlockID, err) } - f.checkForDoubleProposal(blockContainer) - f.forest.AddVertex(blockContainer) - if f.newestView < block.View { - f.newestView = block.View + err = f.checkForConflictingQCs(certifiedBlock.CertifyingQC) + if err != nil { + return fmt.Errorf("certifying QC for block %v failed check for conflicts: %w", certifiedBlock.Block.BlockID, err) } + f.forest.AddVertex(ToBlockContainer2(certifiedBlock.Block)) + f.notifier.OnBlockIncorporated(certifiedBlock.Block) - err = f.updateFinalizedBlockQC(blockContainer) + // Update finality status: + err = f.checkForAdvancingFinalization(certifiedBlock) if err != nil { - return fmt.Errorf("updating consensus state failed: %w", err) + return fmt.Errorf("updating finalization failed: %w", err) } - f.notifier.OnBlockIncorporated(block) return nil } -// VerifyProposal checks a block for internal consistency and consistency with -// the current forest state. See forest.VerifyVertex for more detail. -// We assume that all blocks are fully verified. A valid block must satisfy all consistency -// requirements; otherwise we have a bug in the compliance layer. -// Error returns: -// - model.MissingBlockError if the parent of the input proposal does not exist in the forest -// (but is above the pruned view) -// - generic error in case of unexpected bug or internal state corruption -func (f *Forks) VerifyProposal(proposal *model.Proposal) error { - block := proposal.Block - if block.View < f.forest.LowestLevel { +// AddValidatedBlock appends the validated block to the tree of pending +// blocks and updates the latest finalized block (if applicable). Unless the parent is +// below the pruning threshold (latest finalized view), we require that the parent is +// already stored in Forks. Calling this method with previously processed blocks +// leaves the consensus state invariant (though, it will potentially cause some +// duplicate processing). +// Notes: +// - Method `AddCertifiedBlock(..)` should be used preferably, if a QC certifying +// `block` is already known. This is generally the case for the consensus follower. +// Method `AddValidatedBlock` is intended for active consensus participants, which fully +// validate blocks (incl. payload), i.e. QCs are processed as part of validated proposals. +// +// Possible error returns: +// - model.MissingBlockError if the parent does not exist in the forest (but is above +// the pruned view). From the perspective of Forks, this error is benign (no-op). +// - model.InvalidBlockError if the block is invalid (see `Forks.EnsureBlockIsValidExtension` +// for details). From the perspective of Forks, this error is benign (no-op). However, we +// assume all blocks are fully verified, i.e. they should satisfy all consistency +// requirements. Hence, this error is likely an indicator of a bug in the compliance layer. +// - model.ByzantineThresholdExceededError if conflicting QCs or conflicting finalized +// blocks have been detected (violating a foundational consensus guarantees). This +// indicates that there are 1/3+ Byzantine nodes (weighted by stake) in the network, +// breaking the safety guarantees of HotStuff (or there is a critical bug / data +// corruption). Forks cannot recover from this exception. +// - All other errors are potential symptoms of bugs or state corruption. +func (f *Forks) AddValidatedBlock(proposal *model.Block) error { + if !f.IsProcessingNeeded(proposal) { return nil } - blockContainer := &BlockContainer{Proposal: proposal} - err := f.forest.VerifyVertex(blockContainer) + + // Check proposal for byzantine evidence, store it and emit `OnBlockIncorporated` notification: + err := f.checkForByzantineEvidence(proposal) if err != nil { - if forest.IsInvalidVertexError(err) { - return fmt.Errorf("cannot add proposal %x to forest: %s", block.BlockID, err.Error()) - } - return fmt.Errorf("unexpected error verifying proposal vertex: %w", err) + return fmt.Errorf("cannot check Byzantine evidence for block %v: %w", proposal.BlockID, err) } + f.forest.AddVertex(ToBlockContainer2(proposal)) + f.notifier.OnBlockIncorporated(proposal) - // omit checking existence of parent if block at lowest non-pruned view number - if (block.View == f.forest.LowestLevel) || (block.QC.View < f.forest.LowestLevel) { + // Update finality status: In the implementation, our notion of finality is based on certified blocks. + // The certified parent essentially combines the parent, with the QC contained in block, to drive finalization. + parent, found := f.GetBlock(proposal.QC.BlockID) + if !found { + // Not finding the parent means it is already pruned; hence this block does not change the finalization state. return nil } - // for block whose parents are _not_ below the pruning height, we expect the parent to be known. - if _, isParentKnown := f.forest.GetVertex(block.QC.BlockID); !isParentKnown { // we are missing the parent - return model.MissingBlockError{ - View: block.QC.View, - BlockID: block.QC.BlockID, - } + certifiedParent, err := model.NewCertifiedBlock(parent, proposal.QC) + if err != nil { + return fmt.Errorf("mismatching QC with parent (corrupted Forks state):%w", err) + } + err = f.checkForAdvancingFinalization(&certifiedParent) + if err != nil { + return fmt.Errorf("updating finalization failed: %w", err) } return nil } +// checkForByzantineEvidence inspects whether the given `block` together with the already +// known information yields evidence of byzantine behaviour. Furthermore, the method enforces +// that `block` is a valid extension of the tree of pending blocks. If the block is a double +// proposal, we emit an `OnBlockIncorporated` notification. Though, provided the block is a +// valid extension of the block tree by itself, it passes this method without an error. +// +// Possible error returns: +// - model.MissingBlockError if the parent does not exist in the forest (but is above +// the pruned view). From the perspective of Forks, this error is benign (no-op). +// - model.InvalidBlockError if the block is invalid (see `Forks.EnsureBlockIsValidExtension` +// for details). From the perspective of Forks, this error is benign (no-op). However, we +// assume all blocks are fully verified, i.e. they should satisfy all consistency +// requirements. Hence, this error is likely an indicator of a bug in the compliance layer. +// - model.ByzantineThresholdExceededError if conflicting QCs have been detected. +// Forks cannot recover from this exception. +// - All other errors are potential symptoms of bugs or state corruption. +func (f *Forks) checkForByzantineEvidence(block *model.Block) error { + err := f.EnsureBlockIsValidExtension(block) + if err != nil { + return fmt.Errorf("consistency check on block failed: %w", err) + } + err = f.checkForConflictingQCs(block.QC) + if err != nil { + return fmt.Errorf("checking QC for conflicts failed: %w", err) + } + f.checkForDoubleProposal(block) + return nil +} + // checkForConflictingQCs checks if QC conflicts with a stored Quorum Certificate. // In case a conflicting QC is found, an ByzantineThresholdExceededError is returned. -// // Two Quorum Certificates q1 and q2 are defined as conflicting iff: -// - q1.View == q2.View -// - q1.BlockID != q2.BlockID +// +// q1.View == q2.View AND q1.BlockID ≠ q2.BlockID // // This means there are two Quorums for conflicting blocks at the same view. -// Per 'Observation 1' from the Jolteon paper https://arxiv.org/pdf/2106.10362v1.pdf, two -// conflicting QCs can exist if and only if the Byzantine threshold is exceeded. +// Per 'Observation 1' from the Jolteon paper https://arxiv.org/pdf/2106.10362v1.pdf, +// two conflicting QCs can exist if and only if the Byzantine threshold is exceeded. // Error returns: -// * model.ByzantineThresholdExceededError if input QC conflicts with an existing QC. +// - model.ByzantineThresholdExceededError if conflicting QCs have been detected. +// Forks cannot recover from this exception. +// - All other errors are potential symptoms of bugs or state corruption. func (f *Forks) checkForConflictingQCs(qc *flow.QuorumCertificate) error { it := f.forest.GetVerticesAtLevel(qc.View) for it.HasNext() { @@ -237,8 +332,8 @@ func (f *Forks) checkForConflictingQCs(qc *flow.QuorumCertificate) error { // => conflicting qc otherChildren := f.forest.GetChildren(otherBlock.VertexID()) if otherChildren.HasNext() { - otherChild := otherChildren.NextVertex() - conflictingQC := otherChild.(*BlockContainer).Proposal.Block.QC + otherChild := otherChildren.NextVertex().(*BlockContainer).Block() + conflictingQC := otherChild.QC return model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( "conflicting QCs at view %d: %v and %v", qc.View, qc.BlockID, conflictingQC.BlockID, @@ -252,192 +347,162 @@ func (f *Forks) checkForConflictingQCs(qc *flow.QuorumCertificate) error { // checkForDoubleProposal checks if the input proposal is a double proposal. // A double proposal occurs when two proposals with the same view exist in Forks. // If there is a double proposal, notifier.OnDoubleProposeDetected is triggered. -func (f *Forks) checkForDoubleProposal(container *BlockContainer) { - block := container.Proposal.Block +func (f *Forks) checkForDoubleProposal(block *model.Block) { it := f.forest.GetVerticesAtLevel(block.View) for it.HasNext() { - otherVertex := it.NextVertex() // by construction, must have same view as parentView - if container.VertexID() != otherVertex.VertexID() { - f.notifier.OnDoubleProposeDetected(block, otherVertex.(*BlockContainer).Proposal.Block) + otherVertex := it.NextVertex() // by construction, must have same view as block + otherBlock := otherVertex.(*BlockContainer).Block() + if block.BlockID != otherBlock.BlockID { + f.notifier.OnDoubleProposeDetected(block, otherBlock) } } } -// updateFinalizedBlockQC updates the latest finalized block, if possible. -// This function should be called every time a new block is added to Forks. -// If the new block is the head of a 2-chain satisfying the finalization rule, -// then we update Forks.lastFinalizedBlockQC to the new latest finalized block. -// Calling this method with previously-processed blocks leaves the consensus state invariant. +// checkForAdvancingFinalization checks whether observing certifiedBlock leads to progress of +// finalization. This function should be called every time a new block is added to Forks. If the new +// block is the head of a 2-chain satisfying the finalization rule, we update `Forks.finalityProof` to +// the new latest finalized block. Calling this method with previously-processed blocks leaves the +// consensus state invariant. // UNVALIDATED: assumes that relevant block properties are consistent with previous blocks // Error returns: -// - model.ByzantineThresholdExceededError if we are finalizing a block which is invalid to finalize. -// This either indicates a critical internal bug / data corruption, or that the network Byzantine -// threshold was exceeded, breaking the safety guarantees of HotStuff. +// - model.MissingBlockError if the parent does not exist in the forest (but is above +// the pruned view). From the perspective of Forks, this error is benign (no-op). +// - model.ByzantineThresholdExceededError in case we detect a finalization fork (violating +// a foundational consensus guarantee). This indicates that there are 1/3+ Byzantine nodes +// (weighted by stake) in the network, breaking the safety guarantees of HotStuff (or there +// is a critical bug / data corruption). Forks cannot recover from this exception. // - generic error in case of unexpected bug or internal state corruption -func (f *Forks) updateFinalizedBlockQC(blockContainer *BlockContainer) error { - ancestryChain, err := f.getTwoChain(blockContainer) - if err != nil { - // We expect that getTwoChain might error with a ErrPrunedAncestry. This error indicates that the - // 2-chain of this block reaches _beyond_ the last finalized block. It is straight forward to show: +func (f *Forks) checkForAdvancingFinalization(certifiedBlock *model.CertifiedBlock) error { + // We prune all blocks in forest which are below the most recently finalized block. + // Hence, we have a pruned ancestry if and only if either of the following conditions applies: + // (a) If a block's parent view (i.e. block.QC.View) is below the most recently finalized block. + // (b) If a block's view is equal to the most recently finalized block. + // Caution: + // * Under normal operation, case (b) is covered by the logic for case (a) + // * However, the existence of a genesis block requires handling case (b) explicitly: + // The root block is specified and trusted by the node operator. If the root block is the + // genesis block, it might not contain a QC pointing to a parent (as there is no parent). + // In this case, condition (a) cannot be evaluated. + lastFinalizedView := f.FinalizedView() + if (certifiedBlock.View() <= lastFinalizedView) || (certifiedBlock.Block.QC.View < lastFinalizedView) { + // Repeated blocks are expected during normal operations. We enter this code block if and only + // if the parent's view is _below_ the last finalized block. It is straight forward to show: // Lemma: Let B be a block whose 2-chain reaches beyond the last finalized block // => B will not update the locked or finalized block - if errors.Is(err, ErrPrunedAncestry) { - // blockContainer's 2-chain reaches beyond the last finalized block - // based on Lemma from above, we can skip attempting to update locked or finalized block - return nil - } - if model.IsMissingBlockError(err) { - // we are missing some un-pruned ancestry of blockContainer -> indicates corrupted internal state - return fmt.Errorf("unexpected missing block while updating consensus state: %s", err.Error()) - } - return fmt.Errorf("retrieving 2-chain ancestry failed: %w", err) + return nil + } + + // retrieve parent; always expected to succeed, because we passed the checks above + qcForParent := certifiedBlock.Block.QC + parentVertex, parentBlockKnown := f.forest.GetVertex(qcForParent.BlockID) + if !parentBlockKnown { + return model.MissingBlockError{View: qcForParent.View, BlockID: qcForParent.BlockID} } + parentBlock := parentVertex.(*BlockContainer).Block() - // Note: we assume that all stored blocks pass Forks.VerifyProposal(block); - // specifically, that Proposal's ViewNumber is strictly monotonously + // Note: we assume that all stored blocks pass Forks.EnsureBlockIsValidExtension(block); + // specifically, that Proposal's ViewNumber is strictly monotonically // increasing which is enforced by LevelledForest.VerifyVertex(...) // We denote: // * a DIRECT 1-chain as '<-' // * a general 1-chain as '<~' (direct or indirect) - // Jolteon's rule for finalizing block b is - // b <- b' <~ b* (aka a DIRECT 1-chain PLUS any 1-chain) - // where b* is the head block of the ancestryChain - // Hence, we can finalize b as head of 2-chain, if and only the viewNumber of b' is exactly 1 higher than the view of b - b := ancestryChain.twoChain - if ancestryChain.oneChain.Block.View != b.Block.View+1 { + // Jolteon's rule for finalizing `parentBlock` is + // parentBlock <- Block <~ certifyingQC (i.e. a DIRECT 1-chain PLUS any 1-chain) + // ╰─────────────────────╯ + // certifiedBlock + // Hence, we can finalize `parentBlock` as head of a 2-chain, + // if and only if `Block.View` is exactly 1 higher than the view of `parentBlock` + if parentBlock.View+1 != certifiedBlock.View() { return nil } - return f.finalizeUpToBlock(b.CertifyingQC) -} -// getTwoChain returns the 2-chain for the input block container b. -// See ancestryChain for documentation on the structure of the 2-chain. -// Returns ErrPrunedAncestry if any part of the 2-chain is below the last pruned view. -// Error returns: -// - ErrPrunedAncestry if any part of the 2-chain is below the last pruned view. -// - model.MissingBlockError if any block in the 2-chain does not exist in the forest -// (but is above the pruned view) -// - generic error in case of unexpected bug or internal state corruption -func (f *Forks) getTwoChain(blockContainer *BlockContainer) (*ancestryChain, error) { - ancestryChain := ancestryChain{block: blockContainer} + // `parentBlock` is now finalized: + // * While Forks is single-threaded, there is still the possibility of reentrancy. Specifically, the + // consumers of our finalization events are served by the goroutine executing Forks. It is conceivable + // that a consumer might access Forks and query the latest finalization proof. This would be legal, if + // the component supplying the goroutine to Forks also consumes the notifications. + // * Therefore, for API safety, we want to first update Fork's `finalityProof` before we emit any notifications. - var err error - ancestryChain.oneChain, err = f.getNextAncestryLevel(blockContainer.Proposal.Block) + // Advancing finalization step (i): we collect all blocks for finalization (no notifications are emitted) + blocksToBeFinalized, err := f.collectBlocksForFinalization(qcForParent) if err != nil { - return nil, err + return fmt.Errorf("advancing finalization to block %v from view %d failed: %w", qcForParent.BlockID, qcForParent.View, err) } - ancestryChain.twoChain, err = f.getNextAncestryLevel(ancestryChain.oneChain.Block) - if err != nil { - return nil, err - } - return &ancestryChain, nil -} -// getNextAncestryLevel retrieves parent from forest. Returns QCBlock for the parent, -// i.e. the parent block itself and the qc pointing to the parent, i.e. block.QC(). -// UNVALIDATED: expects block to pass Forks.VerifyProposal(block) -// Error returns: -// - ErrPrunedAncestry if the input block's parent is below the pruned view. -// - model.MissingBlockError if the parent block does not exist in the forest -// (but is above the pruned view) -// - generic error in case of unexpected bug or internal state corruption -func (f *Forks) getNextAncestryLevel(block *model.Block) (*model.CertifiedBlock, error) { - // The finalizer prunes all blocks in forest which are below the most recently finalized block. - // Hence, we have a pruned ancestry if and only if either of the following conditions applies: - // (a) if a block's parent view (i.e. block.QC.View) is below the most recently finalized block. - // (b) if a block's view is equal to the most recently finalized block. - // Caution: - // * Under normal operation, case (b) is covered by the logic for case (a) - // * However, the existence of a genesis block requires handling case (b) explicitly: - // The root block is specified and trusted by the node operator. If the root block is the - // genesis block, it might not contain a qc pointing to a parent (as there is no parent). - // In this case, condition (a) cannot be evaluated. - if (block.View <= f.lastFinalized.Block.View) || (block.QC.View < f.lastFinalized.Block.View) { - return nil, ErrPrunedAncestry + // Advancing finalization step (ii): update `finalityProof` and prune `LevelledForest` + f.finalityProof = &hotstuff.FinalityProof{Block: parentBlock, CertifiedChild: *certifiedBlock} + err = f.forest.PruneUpToLevel(f.FinalizedView()) + if err != nil { + return fmt.Errorf("pruning levelled forest failed unexpectedly: %w", err) } - parentVertex, parentBlockKnown := f.forest.GetVertex(block.QC.BlockID) - if !parentBlockKnown { - return nil, model.MissingBlockError{View: block.QC.View, BlockID: block.QC.BlockID} - } - parentBlock := parentVertex.(*BlockContainer).Proposal.Block - // sanity check consistency between input block and parent - if parentBlock.BlockID != block.QC.BlockID || parentBlock.View != block.QC.View { - return nil, fmt.Errorf("parent/child mismatch while getting ancestry level: child: (id=%x, view=%d, qc.view=%d, qc.block_id=%x) parent: (id=%x, view=%d)", - block.BlockID, block.View, block.QC.View, block.QC.BlockID, parentBlock.BlockID, parentBlock.View) - } + // Advancing finalization step (iii): iterate over the blocks from (i) and emit finalization events + for _, b := range blocksToBeFinalized { + // first notify other critical components about finalized block - all errors returned here are fatal exceptions + err = f.finalizationCallback.MakeFinal(b.BlockID) + if err != nil { + return fmt.Errorf("finalization error in other component: %w", err) + } - certifiedBlock, err := model.NewCertifiedBlock(parentBlock, block.QC) - if err != nil { - return nil, fmt.Errorf("constructing certified block failed: %w", err) + // notify less important components about finalized block + f.notifier.OnFinalizedBlock(b) } - return &certifiedBlock, nil + return nil } -// finalizeUpToBlock finalizes all blocks up to (and including) the block pointed to by `qc`. -// Finalization starts with the child of `lastFinalizedBlockQC` (explicitly checked); -// and calls OnFinalizedBlock on the newly finalized blocks in increasing height order. +// collectBlocksForFinalization collects and returns all newly finalized blocks up to (and including) +// the block pointed to by `qc`. The blocks are listed in order of increasing height. // Error returns: -// - model.ByzantineThresholdExceededError if we are finalizing a block which is invalid to finalize. -// This either indicates a critical internal bug / data corruption, or that the network Byzantine -// threshold was exceeded, breaking the safety guarantees of HotStuff. +// - model.ByzantineThresholdExceededError in case we detect a finalization fork (violating +// a foundational consensus guarantee). This indicates that there are 1/3+ Byzantine nodes +// (weighted by stake) in the network, breaking the safety guarantees of HotStuff (or there +// is a critical bug / data corruption). Forks cannot recover from this exception. // - generic error in case of bug or internal state corruption -func (f *Forks) finalizeUpToBlock(qc *flow.QuorumCertificate) error { - if qc.View < f.lastFinalized.Block.View { - return model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( - "finalizing blocks with view %d which is lower than previously finalized block at view %d", - qc.View, f.lastFinalized.Block.View, +func (f *Forks) collectBlocksForFinalization(qc *flow.QuorumCertificate) ([]*model.Block, error) { + lastFinalized := f.FinalizedBlock() + if qc.View < lastFinalized.View { + return nil, model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( + "finalizing block with view %d which is lower than previously finalized block at view %d", + qc.View, lastFinalized.View, )} } - if qc.View == f.lastFinalized.Block.View { - // Sanity check: the previously last Finalized Proposal must be an ancestor of `block` - if f.lastFinalized.Block.BlockID != qc.BlockID { - return model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( - "finalizing blocks with view %d at conflicting forks: %x and %x", - qc.View, qc.BlockID, f.lastFinalized.Block.BlockID, - )} - } - return nil - } - // Have: qc.View > f.lastFinalizedBlockQC.View => finalizing new block - - // get Proposal and finalize everything up to the block's parent - blockVertex, ok := f.forest.GetVertex(qc.BlockID) // require block to resolve parent - if !ok { - return fmt.Errorf("failed to get parent while finalizing blocks (qc.view=%d, qc.block_id=%x)", qc.View, qc.BlockID) - } - blockContainer := blockVertex.(*BlockContainer) - block := blockContainer.Proposal.Block - err := f.finalizeUpToBlock(block.QC) // finalize Parent, i.e. the block pointed to by the block's QC - if err != nil { - return err - } - - if block.BlockID != qc.BlockID || block.View != qc.View { - return fmt.Errorf("mismatch between finalized block and QC") + if qc.View == lastFinalized.View { // no new blocks to be finalized + return nil, nil } - // finalize block itself: - *f.lastFinalized, err = model.NewCertifiedBlock(block, qc) - if err != nil { - return fmt.Errorf("constructing certified block failed: %w", err) - } - err = f.forest.PruneUpToLevel(block.View) - if err != nil { - if mempool.IsBelowPrunedThresholdError(err) { - // we should never see this error because we finalize blocks in strictly increasing view order - return fmt.Errorf("unexpected error pruning forest, indicates corrupted state: %s", err.Error()) + // Collect all blocks that are pending finalization in slice. While we crawl the blocks starting + // from the newest finalized block backwards (decreasing views), we would like to return them in + // order of _increasing_ view. Therefore, we fill the slice starting with the highest index. + l := qc.View - lastFinalized.View // l is an upper limit to the number of blocks that can be maximally finalized + blocksToBeFinalized := make([]*model.Block, l) + for qc.View > lastFinalized.View { + b, ok := f.GetBlock(qc.BlockID) + if !ok { + return nil, fmt.Errorf("failed to get block (view=%d, blockID=%x) for finalization", qc.View, qc.BlockID) } - return fmt.Errorf("unexpected error while pruning forest: %w", err) + l-- + blocksToBeFinalized[l] = b + qc = b.QC // move to parent + } + // Now, `l` is the index where we stored the oldest block that should be finalized. Note that `l` + // might be larger than zero, if some views have no finalized blocks. Hence, `blocksToBeFinalized` + // might start with nil entries, which we remove: + blocksToBeFinalized = blocksToBeFinalized[l:] + + // qc should now point to the latest finalized block. Otherwise, the + // consensus committee is compromised (or we have a critical internal bug). + if qc.View < lastFinalized.View { + return nil, model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( + "finalizing block with view %d which is lower than previously finalized block at view %d", + qc.View, lastFinalized.View, + )} } - - // notify other critical components about finalized block - all errors returned are considered critical - err = f.finalizationCallback.MakeFinal(blockContainer.VertexID()) - if err != nil { - return fmt.Errorf("finalization error in other component: %w", err) + if qc.View == lastFinalized.View && lastFinalized.BlockID != qc.BlockID { + return nil, model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( + "finalizing blocks with view %d at conflicting forks: %x and %x", + qc.View, qc.BlockID, lastFinalized.BlockID, + )} } - // notify less important components about finalized block - f.notifier.OnFinalizedBlock(block) - return nil + return blocksToBeFinalized, nil } diff --git a/consensus/hotstuff/forks/forks2.go b/consensus/hotstuff/forks/forks2.go deleted file mode 100644 index 7cf71ae297a..00000000000 --- a/consensus/hotstuff/forks/forks2.go +++ /dev/null @@ -1,518 +0,0 @@ -package forks - -import ( - "fmt" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/forest" -) - -// FinalityProof represents a finality proof for a Block. By convention, a FinalityProof -// is immutable. Finality in Jolteon/HotStuff is determined by the 2-chain rule: -// -// There exists a _certified_ block C, such that Block.View + 1 = C.View -type FinalityProof struct { - Block *model.Block - CertifiedChild model.CertifiedBlock -} - -// Forks enforces structural validity of the consensus state and implements -// finalization rules as defined in Jolteon consensus https://arxiv.org/abs/2106.10362 -// The same approach has later been adopted by the Diem team resulting in DiemBFT v4: -// https://developers.diem.com/papers/diem-consensus-state-machine-replication-in-the-diem-blockchain/2021-08-17.pdf -// Forks is NOT safe for concurrent use by multiple goroutines. -type Forks2 struct { - finalizationCallback module.Finalizer - notifier hotstuff.FinalizationConsumer - forest forest.LevelledForest - trustedRoot *model.CertifiedBlock - - // finalityProof holds the latest finalized block including the certified child as proof of finality. - // CAUTION: is nil, when Forks has not yet finalized any blocks beyond the finalized root block it was initialized with - finalityProof *FinalityProof -} - -// TODO: -// • update `hotstuff.Forks` interface to represent Forks2 -// • update business logic to of consensus participant and follower to use Forks2 -// As the result, the following should apply again -// var _ hotstuff.Forks = (*Forks2)(nil) - -func NewForks2(trustedRoot *model.CertifiedBlock, finalizationCallback module.Finalizer, notifier hotstuff.FinalizationConsumer) (*Forks2, error) { - if (trustedRoot.Block.BlockID != trustedRoot.CertifyingQC.BlockID) || (trustedRoot.Block.View != trustedRoot.CertifyingQC.View) { - return nil, model.NewConfigurationErrorf("invalid root: root QC is not pointing to root block") - } - - forks := Forks2{ - finalizationCallback: finalizationCallback, - notifier: notifier, - forest: *forest.NewLevelledForest(trustedRoot.Block.View), - trustedRoot: trustedRoot, - finalityProof: nil, - } - - // verify and add root block to levelled forest - err := forks.EnsureBlockIsValidExtension(trustedRoot.Block) - if err != nil { - return nil, fmt.Errorf("invalid root block %v: %w", trustedRoot.ID(), err) - } - forks.forest.AddVertex(ToBlockContainer2(trustedRoot.Block)) - return &forks, nil -} - -// FinalizedView returns the largest view number that has been finalized so far -func (f *Forks2) FinalizedView() uint64 { - if f.finalityProof == nil { - return f.trustedRoot.Block.View - } - return f.finalityProof.Block.View -} - -// FinalizedBlock returns the finalized block with the largest view number -func (f *Forks2) FinalizedBlock() *model.Block { - if f.finalityProof == nil { - return f.trustedRoot.Block - } - return f.finalityProof.Block -} - -// FinalityProof returns the latest finalized block and a certified child from -// the subsequent view, which proves finality. -// CAUTION: method returns (nil, false), when Forks has not yet finalized any -// blocks beyond the finalized root block it was initialized with. -func (f *Forks2) FinalityProof() (*FinalityProof, bool) { - return f.finalityProof, f.finalityProof != nil -} - -// GetBlock returns block for given ID -func (f *Forks2) GetBlock(blockID flow.Identifier) (*model.Block, bool) { - blockContainer, hasBlock := f.forest.GetVertex(blockID) - if !hasBlock { - return nil, false - } - return blockContainer.(*BlockContainer2).Block(), true -} - -// GetBlocksForView returns all known blocks for the given view -func (f *Forks2) GetBlocksForView(view uint64) []*model.Block { - vertexIterator := f.forest.GetVerticesAtLevel(view) - blocks := make([]*model.Block, 0, 1) // in the vast majority of cases, there will only be one proposal for a particular view - for vertexIterator.HasNext() { - v := vertexIterator.NextVertex() - blocks = append(blocks, v.(*BlockContainer2).Block()) - } - return blocks -} - -// IsKnownBlock checks whether block is known. -func (f *Forks2) IsKnownBlock(blockID flow.Identifier) bool { - _, hasBlock := f.forest.GetVertex(blockID) - return hasBlock -} - -// IsProcessingNeeded determines whether the given block needs processing, -// based on the block's view and hash. -// Returns false if any of the following conditions applies -// - block view is _below_ the most recently finalized block -// - the block already exists in the consensus state -// -// UNVALIDATED: expects block to pass Forks.EnsureBlockIsValidExtension(block) -func (f *Forks2) IsProcessingNeeded(block *model.Block) bool { - if block.View < f.FinalizedView() || f.IsKnownBlock(block.BlockID) { - return false - } - return true -} - -// EnsureBlockIsValidExtension checks that the given block is a valid extension to the tree -// of blocks already stored (no state modifications). Specifically, the following conditions -// are enforced, which are critical to the correctness of Forks: -// -// 1. If a block with the same ID is already stored, their views must be identical. -// 2. The block's view must be strictly larger than the view of its parent. -// 3. The parent must already be stored (or below the pruning height). -// -// Exclusions to these rules (by design): -// Let W denote the view of block's parent (i.e. W := block.QC.View) and F the latest -// finalized view. -// -// (i) If block.View < F, adding the block would be a no-op. Such blocks are considered -// compatible (principle of vacuous truth), i.e. we skip checking 1, 2, 3. -// (ii) If block.View == F, we do not inspect the QC / parent at all (skip 2 and 3). -// This exception is important for compatability with genesis or spork-root blocks, -// which do not contain a QC. -// (iii) If block.View > F, but block.QC.View < F the parent has already been pruned. In -// this case, we omit rule 3. (principle of vacuous truth applied to the parent) -// -// We assume that all blocks are fully verified. A valid block must satisfy all consistency -// requirements; otherwise we have a bug in the compliance layer. -// -// Error returns: -// - model.MissingBlockError if the parent of the input proposal does not exist in the -// forest (but is above the pruned view). Represents violation of condition 3. -// - model.InvalidBlockError if the block violates condition 1. or 2. -// - generic error in case of unexpected bug or internal state corruption -func (f *Forks2) EnsureBlockIsValidExtension(block *model.Block) error { - if block.View < f.forest.LowestLevel { // exclusion (i) - return nil - } - - // LevelledForest enforces conditions 1. and 2. including the respective exclusions (ii) and (iii). - blockContainer := ToBlockContainer2(block) - err := f.forest.VerifyVertex(blockContainer) - if err != nil { - if forest.IsInvalidVertexError(err) { - return model.NewInvalidBlockError(block.BlockID, block.View, fmt.Errorf("not a valid vertex for block tree: %w", err)) - } - return fmt.Errorf("block tree generated unexpected error validating vertex: %w", err) - } - - // Condition 3: - // LevelledForest implements a more generalized algorithm that also works for disjoint graphs. - // Therefore, LevelledForest _not_ enforce condition 3. Here, we additionally require that the - // pending blocks form a tree (connected graph), i.e. we need to enforce condition 3 - if (block.View == f.forest.LowestLevel) || (block.QC.View < f.forest.LowestLevel) { // exclusion (ii) and (iii) - return nil - } - // For a block whose parent is _not_ below the pruning height, we expect the parent to be known. - if _, isParentKnown := f.forest.GetVertex(block.QC.BlockID); !isParentKnown { // missing parent - return model.MissingBlockError{ - View: block.QC.View, - BlockID: block.QC.BlockID, - } - } - return nil -} - -// AddCertifiedBlock appends the given certified block to the tree of pending -// blocks and updates the latest finalized block (if finalization progressed). -// Unless the parent is below the pruning threshold (latest finalized view), we -// require that the parent is already stored in Forks. -// -// Possible error returns: -// - model.MissingBlockError if the parent does not exist in the forest (but is above -// the pruned view). From the perspective of Forks, this error is benign (no-op). -// - model.InvalidBlockError if the block is invalid (see `Forks.EnsureBlockIsValidExtension` -// for details). From the perspective of Forks, this error is benign (no-op). However, we -// assume all blocks are fully verified, i.e. they should satisfy all consistency -// requirements. Hence, this error is likely an indicator of a bug in the compliance layer. -// - model.ByzantineThresholdExceededError if conflicting QCs or conflicting finalized -// blocks have been detected (violating a foundational consensus guarantees). This -// indicates that there are 1/3+ Byzantine nodes (weighted by stake) in the network, -// breaking the safety guarantees of HotStuff (or there is a critical bug / data -// corruption). Forks cannot recover from this exception. -// - All other errors are potential symptoms of bugs or state corruption. -func (f *Forks2) AddCertifiedBlock(certifiedBlock *model.CertifiedBlock) error { - if !f.IsProcessingNeeded(certifiedBlock.Block) { - return nil - } - - // Check proposal for byzantine evidence, store it and emit `OnBlockIncorporated` notification. - // Note: `checkForByzantineEvidence` only inspects the block, but _not_ its certifying QC. Hence, - // we have to additionally check here, whether the certifying QC conflicts with any known QCs. - err := f.checkForByzantineEvidence(certifiedBlock.Block) - if err != nil { - return fmt.Errorf("cannot check for Byzantine evidence in certified block %v: %w", certifiedBlock.Block.BlockID, err) - } - err = f.checkForConflictingQCs(certifiedBlock.CertifyingQC) - if err != nil { - return fmt.Errorf("certifying QC for block %v failed check for conflicts: %w", certifiedBlock.Block.BlockID, err) - } - f.forest.AddVertex(ToBlockContainer2(certifiedBlock.Block)) - f.notifier.OnBlockIncorporated(certifiedBlock.Block) - - // Update finality status: - err = f.checkForAdvancingFinalization(certifiedBlock) - if err != nil { - return fmt.Errorf("updating finalization failed: %w", err) - } - return nil -} - -// AddProposal appends the given block to the tree of pending -// blocks and updates the latest finalized block (if applicable). Unless the parent is -// below the pruning threshold (latest finalized view), we require that the parent is -// already stored in Forks. Calling this method with previously processed blocks -// leaves the consensus state invariant (though, it will potentially cause some -// duplicate processing). -// Notes: -// - Method `AddCertifiedBlock(..)` should be used preferably, if a QC certifying -// `block` is already known. This is generally the case for the consensus follower. -// Method `AddProposal` is intended for active consensus participants, which fully -// validate blocks (incl. payload), i.e. QCs are processed as part of validated proposals. -// -// Possible error returns: -// - model.MissingBlockError if the parent does not exist in the forest (but is above -// the pruned view). From the perspective of Forks, this error is benign (no-op). -// - model.InvalidBlockError if the block is invalid (see `Forks.EnsureBlockIsValidExtension` -// for details). From the perspective of Forks, this error is benign (no-op). However, we -// assume all blocks are fully verified, i.e. they should satisfy all consistency -// requirements. Hence, this error is likely an indicator of a bug in the compliance layer. -// - model.ByzantineThresholdExceededError if conflicting QCs or conflicting finalized -// blocks have been detected (violating a foundational consensus guarantees). This -// indicates that there are 1/3+ Byzantine nodes (weighted by stake) in the network, -// breaking the safety guarantees of HotStuff (or there is a critical bug / data -// corruption). Forks cannot recover from this exception. -// - All other errors are potential symptoms of bugs or state corruption. -func (f *Forks2) AddProposal(proposal *model.Block) error { - if !f.IsProcessingNeeded(proposal) { - return nil - } - - // Check proposal for byzantine evidence, store it and emit `OnBlockIncorporated` notification: - err := f.checkForByzantineEvidence(proposal) - if err != nil { - return fmt.Errorf("cannot check Byzantine evidence for block %v: %w", proposal.BlockID, err) - } - f.forest.AddVertex(ToBlockContainer2(proposal)) - f.notifier.OnBlockIncorporated(proposal) - - // Update finality status: In the implementation, our notion of finality is based on certified blocks. - // The certified parent essentially combines the parent, with the QC contained in block, to drive finalization. - parent, found := f.GetBlock(proposal.QC.BlockID) - if !found { - // Not finding the parent means it is already pruned; hence this block does not change the finalization state. - return nil - } - certifiedParent, err := model.NewCertifiedBlock(parent, proposal.QC) - if err != nil { - return fmt.Errorf("mismatching QC with parent (corrupted Forks state):%w", err) - } - err = f.checkForAdvancingFinalization(&certifiedParent) - if err != nil { - return fmt.Errorf("updating finalization failed: %w", err) - } - return nil -} - -// checkForByzantineEvidence inspects whether the given `block` together with the already -// known information yields evidence of byzantine behaviour. Furthermore, the method enforces -// that `block` is a valid extension of the tree of pending blocks. If the block is a double -// proposal, we emit an `OnBlockIncorporated` notification. Though, provided the block is a -// valid extension of the block tree by itself, it passes this method without an error. -// -// Possible error returns: -// - model.MissingBlockError if the parent does not exist in the forest (but is above -// the pruned view). From the perspective of Forks, this error is benign (no-op). -// - model.InvalidBlockError if the block is invalid (see `Forks.EnsureBlockIsValidExtension` -// for details). From the perspective of Forks, this error is benign (no-op). However, we -// assume all blocks are fully verified, i.e. they should satisfy all consistency -// requirements. Hence, this error is likely an indicator of a bug in the compliance layer. -// - model.ByzantineThresholdExceededError if conflicting QCs have been detected. -// Forks cannot recover from this exception. -// - All other errors are potential symptoms of bugs or state corruption. -func (f *Forks2) checkForByzantineEvidence(block *model.Block) error { - err := f.EnsureBlockIsValidExtension(block) - if err != nil { - return fmt.Errorf("consistency check on block failed: %w", err) - } - err = f.checkForConflictingQCs(block.QC) - if err != nil { - return fmt.Errorf("checking QC for conflicts failed: %w", err) - } - f.checkForDoubleProposal(block) - return nil -} - -// checkForConflictingQCs checks if QC conflicts with a stored Quorum Certificate. -// In case a conflicting QC is found, an ByzantineThresholdExceededError is returned. -// Two Quorum Certificates q1 and q2 are defined as conflicting iff: -// -// q1.View == q2.View AND q1.BlockID ≠ q2.BlockID -// -// This means there are two Quorums for conflicting blocks at the same view. -// Per 'Observation 1' from the Jolteon paper https://arxiv.org/pdf/2106.10362v1.pdf, -// two conflicting QCs can exist if and only if the Byzantine threshold is exceeded. -// Error returns: -// - model.ByzantineThresholdExceededError if conflicting QCs have been detected. -// Forks cannot recover from this exception. -// - All other errors are potential symptoms of bugs or state corruption. -func (f *Forks2) checkForConflictingQCs(qc *flow.QuorumCertificate) error { - it := f.forest.GetVerticesAtLevel(qc.View) - for it.HasNext() { - otherBlock := it.NextVertex() // by construction, must have same view as qc.View - if qc.BlockID != otherBlock.VertexID() { - // * we have just found another block at the same view number as qc.View but with different hash - // * if this block has a child c, this child will have - // c.qc.view = parentView - // c.qc.ID != parentBlockID - // => conflicting qc - otherChildren := f.forest.GetChildren(otherBlock.VertexID()) - if otherChildren.HasNext() { - otherChild := otherChildren.NextVertex().(*BlockContainer2).Block() - conflictingQC := otherChild.QC - return model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( - "conflicting QCs at view %d: %v and %v", - qc.View, qc.BlockID, conflictingQC.BlockID, - )} - } - } - } - return nil -} - -// checkForDoubleProposal checks if the input proposal is a double proposal. -// A double proposal occurs when two proposals with the same view exist in Forks. -// If there is a double proposal, notifier.OnDoubleProposeDetected is triggered. -func (f *Forks2) checkForDoubleProposal(block *model.Block) { - it := f.forest.GetVerticesAtLevel(block.View) - for it.HasNext() { - otherVertex := it.NextVertex() // by construction, must have same view as block - otherBlock := otherVertex.(*BlockContainer2).Block() - if block.BlockID != otherBlock.BlockID { - f.notifier.OnDoubleProposeDetected(block, otherBlock) - } - } -} - -// checkForAdvancingFinalization checks whether observing certifiedBlock leads to progress of -// finalization. This function should be called every time a new block is added to Forks. If the new -// block is the head of a 2-chain satisfying the finalization rule, we update `Forks.finalityProof` to -// the new latest finalized block. Calling this method with previously-processed blocks leaves the -// consensus state invariant. -// UNVALIDATED: assumes that relevant block properties are consistent with previous blocks -// Error returns: -// - model.MissingBlockError if the parent does not exist in the forest (but is above -// the pruned view). From the perspective of Forks, this error is benign (no-op). -// - model.ByzantineThresholdExceededError in case we detect a finalization fork (violating -// a foundational consensus guarantee). This indicates that there are 1/3+ Byzantine nodes -// (weighted by stake) in the network, breaking the safety guarantees of HotStuff (or there -// is a critical bug / data corruption). Forks cannot recover from this exception. -// - generic error in case of unexpected bug or internal state corruption -func (f *Forks2) checkForAdvancingFinalization(certifiedBlock *model.CertifiedBlock) error { - // We prune all blocks in forest which are below the most recently finalized block. - // Hence, we have a pruned ancestry if and only if either of the following conditions applies: - // (a) If a block's parent view (i.e. block.QC.View) is below the most recently finalized block. - // (b) If a block's view is equal to the most recently finalized block. - // Caution: - // * Under normal operation, case (b) is covered by the logic for case (a) - // * However, the existence of a genesis block requires handling case (b) explicitly: - // The root block is specified and trusted by the node operator. If the root block is the - // genesis block, it might not contain a QC pointing to a parent (as there is no parent). - // In this case, condition (a) cannot be evaluated. - lastFinalizedView := f.FinalizedView() - if (certifiedBlock.View() <= lastFinalizedView) || (certifiedBlock.Block.QC.View < lastFinalizedView) { - // Repeated blocks are expected during normal operations. We enter this code block if and only - // if the parent's view is _below_ the last finalized block. It is straight forward to show: - // Lemma: Let B be a block whose 2-chain reaches beyond the last finalized block - // => B will not update the locked or finalized block - return nil - } - - // retrieve parent; always expected to succeed, because we passed the checks above - qcForParent := certifiedBlock.Block.QC - parentVertex, parentBlockKnown := f.forest.GetVertex(qcForParent.BlockID) - if !parentBlockKnown { - return model.MissingBlockError{View: qcForParent.View, BlockID: qcForParent.BlockID} - } - parentBlock := parentVertex.(*BlockContainer2).Block() - - // Note: we assume that all stored blocks pass Forks.EnsureBlockIsValidExtension(block); - // specifically, that Proposal's ViewNumber is strictly monotonically - // increasing which is enforced by LevelledForest.VerifyVertex(...) - // We denote: - // * a DIRECT 1-chain as '<-' - // * a general 1-chain as '<~' (direct or indirect) - // Jolteon's rule for finalizing `parentBlock` is - // parentBlock <- Block <~ certifyingQC (i.e. a DIRECT 1-chain PLUS any 1-chain) - // ╰─────────────────────╯ - // certifiedBlock - // Hence, we can finalize `parentBlock` as head of a 2-chain, - // if and only if `Block.View` is exactly 1 higher than the view of `parentBlock` - if parentBlock.View+1 != certifiedBlock.View() { - return nil - } - - // `parentBlock` is now finalized: - // * While Forks is single-threaded, there is still the possibility of reentrancy. Specifically, the - // consumers of our finalization events are served by the goroutine executing Forks. It is conceivable - // that a consumer might access Forks and query the latest finalization proof. This would be legal, if - // the component supplying the goroutine to Forks also consumes the notifications. - // * Therefore, for API safety, we want to first update Fork's `finalityProof` before we emit any notifications. - - // Advancing finalization step (i): we collect all blocks for finalization (no notifications are emitted) - blocksToBeFinalized, err := f.collectBlocksForFinalization(qcForParent) - if err != nil { - return fmt.Errorf("advancing finalization to block %v from view %d failed: %w", qcForParent.BlockID, qcForParent.View, err) - } - - // Advancing finalization step (ii): update `finalityProof` and prune `LevelledForest` - f.finalityProof = &FinalityProof{Block: parentBlock, CertifiedChild: *certifiedBlock} - err = f.forest.PruneUpToLevel(f.FinalizedView()) - if err != nil { - return fmt.Errorf("pruning levelled forest failed unexpectedly: %w", err) - } - - // Advancing finalization step (iii): iterate over the blocks from (i) and emit finalization events - for _, b := range blocksToBeFinalized { - // first notify other critical components about finalized block - all errors returned here are fatal exceptions - err = f.finalizationCallback.MakeFinal(b.BlockID) - if err != nil { - return fmt.Errorf("finalization error in other component: %w", err) - } - - // notify less important components about finalized block - f.notifier.OnFinalizedBlock(b) - } - return nil -} - -// collectBlocksForFinalization collects and returns all newly finalized blocks up to (and including) -// the block pointed to by `qc`. The blocks are listed in order of increasing height. -// Error returns: -// - model.ByzantineThresholdExceededError in case we detect a finalization fork (violating -// a foundational consensus guarantee). This indicates that there are 1/3+ Byzantine nodes -// (weighted by stake) in the network, breaking the safety guarantees of HotStuff (or there -// is a critical bug / data corruption). Forks cannot recover from this exception. -// - generic error in case of bug or internal state corruption -func (f *Forks2) collectBlocksForFinalization(qc *flow.QuorumCertificate) ([]*model.Block, error) { - lastFinalized := f.FinalizedBlock() - if qc.View < lastFinalized.View { - return nil, model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( - "finalizing block with view %d which is lower than previously finalized block at view %d", - qc.View, lastFinalized.View, - )} - } - if qc.View == lastFinalized.View { // no new blocks to be finalized - return nil, nil - } - - // Collect all blocks that are pending finalization in slice. While we crawl the blocks starting - // from the newest finalized block backwards (decreasing views), we would like to return them in - // order of _increasing_ view. Therefore, we fill the slice starting with the highest index. - l := qc.View - lastFinalized.View // l is an upper limit to the number of blocks that can be maximally finalized - blocksToBeFinalized := make([]*model.Block, l) - for qc.View > lastFinalized.View { - b, ok := f.GetBlock(qc.BlockID) - if !ok { - return nil, fmt.Errorf("failed to get block (view=%d, blockID=%x) for finalization", qc.View, qc.BlockID) - } - l-- - blocksToBeFinalized[l] = b - qc = b.QC // move to parent - } - // Now, `l` is the index where we stored the oldest block that should be finalized. Note that `l` - // might be larger than zero, if some views have no finalized blocks. Hence, `blocksToBeFinalized` - // might start with nil entries, which we remove: - blocksToBeFinalized = blocksToBeFinalized[l:] - - // qc should now point to the latest finalized block. Otherwise, the - // consensus committee is compromised (or we have a critical internal bug). - if qc.View < lastFinalized.View { - return nil, model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( - "finalizing block with view %d which is lower than previously finalized block at view %d", - qc.View, lastFinalized.View, - )} - } - if qc.View == lastFinalized.View && lastFinalized.BlockID != qc.BlockID { - return nil, model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( - "finalizing blocks with view %d at conflicting forks: %x and %x", - qc.View, qc.BlockID, lastFinalized.BlockID, - )} - } - - return blocksToBeFinalized, nil -} diff --git a/consensus/hotstuff/forks/forks2_test.go b/consensus/hotstuff/forks/forks2_test.go deleted file mode 100644 index 88641c87357..00000000000 --- a/consensus/hotstuff/forks/forks2_test.go +++ /dev/null @@ -1,960 +0,0 @@ -package forks - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/consensus/hotstuff/mocks" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" - mockmodule "github.com/onflow/flow-go/module/mock" -) - -/***************************************************************************** - * NOTATION: * - * A block is denoted as [(◄) ]. * - * For example, [(◄1) 2] means: a block of view 2 that has a QC for view 1. * - *****************************************************************************/ - -// TestInitialization verifies that at initialization, Forks reports: -// - the root / genesis block as finalized -// - it has no finalization proof for the root / genesis block (block and its finaization is trusted) -func TestInitialization(t *testing.T) { - forks, _ := newForks(t) - requireOnlyGenesisBlockFinalized(t, forks) - _, hasProof := forks.FinalityProof() - require.False(t, hasProof) -} - -// TestFinalize_Direct1Chain tests adding a direct 1-chain on top of the genesis block: -// - receives [(◄1) 2] [(◄2) 5] -// -// Expected behaviour: -// - On the one hand, Forks should not finalize any _additional_ blocks, because there is -// no finalizable 2-chain for [(◄1) 2]. Hence, finalization no events should be emitted. -// - On the other hand, after adding the two blocks, Forks has enough knowledge to construct -// a FinalityProof for the genesis block. -func TestFinalize_Direct1Chain(t *testing.T) { - builder := NewBlockBuilder(). - Add(1, 2). - Add(2, 3) - blocks, err := builder.Blocks() - require.Nil(t, err) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - - // adding block [(◄1) 2] should not finalize anything - // as the genesis block is trusted, there should be no FinalityProof available for it - require.NoError(t, forks.AddProposal(blocks[0].Block)) - requireOnlyGenesisBlockFinalized(t, forks) - _, hasProof := forks.FinalityProof() - require.False(t, hasProof) - - // After adding block [(◄2) 3], Forks has enough knowledge to construct a FinalityProof for the - // genesis block. However, finalization remains at the genesis block, so no events should be emitted. - expectedFinalityProof := makeFinalityProof(t, builder.GenesisBlock().Block, blocks[0].Block, blocks[1].Block.QC) - require.NoError(t, forks.AddProposal(blocks[1].Block)) - requireLatestFinalizedBlock(t, forks, builder.GenesisBlock().Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - - // After adding CertifiedBlock [(◄1) 2] (◄2), Forks has enough knowledge to construct a FinalityProof for - // the genesis block. However, finalization remains at the genesis block, so no events should be emitted. - expectedFinalityProof := makeFinalityProof(t, builder.GenesisBlock().Block, blocks[0].Block, blocks[1].Block.QC) - c, err := model.NewCertifiedBlock(blocks[0].Block, blocks[1].Block.QC) - require.NoError(t, err) - - require.NoError(t, forks.AddCertifiedBlock(&c)) - requireLatestFinalizedBlock(t, forks, builder.GenesisBlock().Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) -} - -// TestFinalize_Direct2Chain tests adding a direct 1-chain on a direct 1-chain (direct 2-chain). -// - receives [(◄1) 2] [(◄2) 3] [(◄3) 4] -// - Forks should finalize [(◄1) 2] -func TestFinalize_Direct2Chain(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). - Add(2, 3). - Add(3, 4). - Blocks() - require.Nil(t, err) - expectedFinalityProof := makeFinalityProof(t, blocks[0].Block, blocks[1].Block, blocks[2].Block.QC) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addProposalsToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[0].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[0].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) -} - -// TestFinalize_DirectIndirect2Chain tests adding an indirect 1-chain on a direct 1-chain. -// receives [(◄1) 2] [(◄2) 3] [(◄3) 5] -// it should finalize [(◄1) 2] -func TestFinalize_DirectIndirect2Chain(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). - Add(2, 3). - Add(3, 5). - Blocks() - require.Nil(t, err) - expectedFinalityProof := makeFinalityProof(t, blocks[0].Block, blocks[1].Block, blocks[2].Block.QC) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addProposalsToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[0].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[0].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) -} - -// TestFinalize_IndirectDirect2Chain tests adding a direct 1-chain on an indirect 1-chain. -// - Forks receives [(◄1) 3] [(◄3) 5] [(◄7) 7] -// - it should not finalize any blocks because there is no finalizable 2-chain. -func TestFinalize_IndirectDirect2Chain(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 3). - Add(3, 5). - Add(5, 7). - Blocks() - require.Nil(t, err) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addProposalsToForks(forks, blocks)) - - requireOnlyGenesisBlockFinalized(t, forks) - _, hasProof := forks.FinalityProof() - require.False(t, hasProof) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) - - requireOnlyGenesisBlockFinalized(t, forks) - _, hasProof := forks.FinalityProof() - require.False(t, hasProof) - }) -} - -// TestFinalize_Direct2ChainOnIndirect tests adding a direct 2-chain on an indirect 2-chain: -// - ingesting [(◄1) 3] [(◄3) 5] [(◄5) 6] [(◄6) 7] [(◄7) 8] -// - should result in finalization of [(◄5) 6] -func TestFinalize_Direct2ChainOnIndirect(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 3). - Add(3, 5). - Add(5, 6). - Add(6, 7). - Add(7, 8). - Blocks() - require.Nil(t, err) - expectedFinalityProof := makeFinalityProof(t, blocks[2].Block, blocks[3].Block, blocks[4].Block.QC) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addProposalsToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[2].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[2].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) -} - -// TestFinalize_Direct2ChainOnDirect tests adding a sequence of direct 2-chains: -// - ingesting [(◄1) 2] [(◄2) 3] [(◄3) 4] [(◄4) 5] [(◄5) 6] -// - should result in finalization of [(◄3) 4] -func TestFinalize_Direct2ChainOnDirect(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). - Add(2, 3). - Add(3, 4). - Add(4, 5). - Add(5, 6). - Blocks() - require.Nil(t, err) - expectedFinalityProof := makeFinalityProof(t, blocks[2].Block, blocks[3].Block, blocks[4].Block.QC) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addProposalsToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[2].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[2].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) -} - -// TestFinalize_Multiple2Chains tests the case where a block can be finalized by different 2-chains. -// - ingesting [(◄1) 2] [(◄2) 3] [(◄3) 5] [(◄3) 6] [(◄3) 7] -// - should result in finalization of [(◄1) 2] -func TestFinalize_Multiple2Chains(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). - Add(2, 3). - Add(3, 5). - Add(3, 6). - Add(3, 7). - Blocks() - require.Nil(t, err) - expectedFinalityProof := makeFinalityProof(t, blocks[0].Block, blocks[1].Block, blocks[2].Block.QC) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addProposalsToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[0].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[0].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) -} - -// TestFinalize_OrphanedFork tests that we can finalize a block which causes a conflicting fork to be orphaned. -// We ingest the the following block tree: -// -// [(◄1) 2] [(◄2) 3] -// [(◄2) 4] [(◄4) 5] [(◄5) 6] -// -// which should result in finalization of [(◄2) 4] and pruning of [(◄2) 3] -func TestFinalize_OrphanedFork(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). // [(◄1) 2] - Add(2, 3). // [(◄2) 3], should eventually be pruned - Add(2, 4). // [(◄2) 4], should eventually be finalized - Add(4, 5). // [(◄4) 5] - Add(5, 6). // [(◄5) 6] - Blocks() - require.Nil(t, err) - expectedFinalityProof := makeFinalityProof(t, blocks[2].Block, blocks[3].Block, blocks[4].Block.QC) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addProposalsToForks(forks, blocks)) - - require.False(t, forks.IsKnownBlock(blocks[1].Block.BlockID)) - requireLatestFinalizedBlock(t, forks, blocks[2].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) - - require.False(t, forks.IsKnownBlock(blocks[1].Block.BlockID)) - requireLatestFinalizedBlock(t, forks, blocks[2].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) -} - -// TestDuplication tests that delivering the same block/qc multiple times has -// the same end state as delivering the block/qc once. -// - Forks receives [(◄1) 2] [(◄2) 3] [(◄2) 3] [(◄3) 4] [(◄3) 4] [(◄4) 5] [(◄4) 5] -// - it should finalize [(◄2) 3] -func TestDuplication(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). - Add(2, 3). - Add(2, 3). - Add(3, 4). - Add(3, 4). - Add(4, 5). - Add(4, 5). - Blocks() - require.Nil(t, err) - expectedFinalityProof := makeFinalityProof(t, blocks[1].Block, blocks[3].Block, blocks[5].Block.QC) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addProposalsToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[1].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) - - requireLatestFinalizedBlock(t, forks, blocks[1].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) -} - -// TestIgnoreBlocksBelowFinalizedView tests that blocks below finalized view are ignored. -// - Forks receives [(◄1) 2] [(◄2) 3] [(◄3) 4] [(◄1) 5] -// - it should finalize [(◄1) 2] -func TestIgnoreBlocksBelowFinalizedView(t *testing.T) { - builder := NewBlockBuilder(). - Add(1, 2). // [(◄1) 2] - Add(2, 3). // [(◄2) 3] - Add(3, 4). // [(◄3) 4] - Add(1, 5) // [(◄1) 5] - blocks, err := builder.Blocks() - require.Nil(t, err) - expectedFinalityProof := makeFinalityProof(t, blocks[0].Block, blocks[1].Block, blocks[2].Block.QC) - - t.Run("ingest proposals", func(t *testing.T) { - // initialize forks and add first 3 blocks: - // * block [(◄1) 2] should then be finalized - // * and block [1] should be pruned - forks, _ := newForks(t) - require.Nil(t, addProposalsToForks(forks, blocks[:3])) - - // sanity checks to confirm correct test setup - requireLatestFinalizedBlock(t, forks, blocks[0].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - require.False(t, forks.IsKnownBlock(builder.GenesisBlock().ID())) - - // adding block [(◄1) 5]: note that QC is _below_ the pruning threshold, i.e. cannot resolve the parent - // * Forks should store block, despite the parent already being pruned - // * finalization should not change - orphanedBlock := blocks[3].Block - require.Nil(t, forks.AddProposal(orphanedBlock)) - require.True(t, forks.IsKnownBlock(orphanedBlock.BlockID)) - requireLatestFinalizedBlock(t, forks, blocks[0].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - // initialize forks and add first 3 blocks: - // * block [(◄1) 2] should then be finalized - // * and block [1] should be pruned - forks, _ := newForks(t) - require.Nil(t, addCertifiedBlocksToForks(forks, blocks[:3])) - // sanity checks to confirm correct test setup - requireLatestFinalizedBlock(t, forks, blocks[0].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - require.False(t, forks.IsKnownBlock(builder.GenesisBlock().ID())) - - // adding block [(◄1) 5]: note that QC is _below_ the pruning threshold, i.e. cannot resolve the parent - // * Forks should store block, despite the parent already being pruned - // * finalization should not change - certBlockWithUnknownParent := toCertifiedBlock(t, blocks[3].Block) - require.Nil(t, forks.AddCertifiedBlock(certBlockWithUnknownParent)) - require.True(t, forks.IsKnownBlock(certBlockWithUnknownParent.Block.BlockID)) - requireLatestFinalizedBlock(t, forks, blocks[0].Block) - requireFinalityProof(t, forks, expectedFinalityProof) - }) -} - -// TestDoubleProposal tests that the DoubleProposal notification is emitted when two different -// proposals for the same view are added. We ingest the the following block tree: -// -// / [(◄1) 2] -// [1] -// \ [(◄1) 2'] -// -// which should result in a DoubleProposal event referencing the blocks [(◄1) 2] and [(◄1) 2'] -func TestDoubleProposal(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). // [(◄1) 2] - AddVersioned(1, 2, 0, 1). // [(◄1) 2'] - Blocks() - require.Nil(t, err) - - t.Run("ingest proposals", func(t *testing.T) { - forks, notifier := newForks(t) - notifier.On("OnDoubleProposeDetected", blocks[1].Block, blocks[0].Block).Once() - - err = addProposalsToForks(forks, blocks) - require.Nil(t, err) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, notifier := newForks(t) - notifier.On("OnDoubleProposeDetected", blocks[1].Block, blocks[0].Block).Once() - - err = forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[0].Block)) // add [(◄1) 2] as certified block - require.Nil(t, err) - err = forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[1].Block)) // add [(◄1) 2'] as certified block - require.Nil(t, err) - }) -} - -// TestConflictingQCs checks that adding 2 conflicting QCs should return model.ByzantineThresholdExceededError -// We ingest the the following block tree: -// -// [(◄1) 2] [(◄2) 3] [(◄3) 4] [(◄4) 6] -// [(◄2) 3'] [(◄3') 5] -// -// which should result in a `ByzantineThresholdExceededError`, because conflicting blocks 3 and 3' both have QCs -func TestConflictingQCs(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). // [(◄1) 2] - Add(2, 3). // [(◄2) 3] - AddVersioned(2, 3, 0, 1). // [(◄2) 3'] - Add(3, 4). // [(◄3) 4] - Add(4, 6). // [(◄4) 6] - AddVersioned(3, 5, 1, 0). // [(◄3') 5] - Blocks() - require.Nil(t, err) - - t.Run("ingest proposals", func(t *testing.T) { - forks, notifier := newForks(t) - notifier.On("OnDoubleProposeDetected", blocks[2].Block, blocks[1].Block).Return(nil) - - err = addProposalsToForks(forks, blocks) - assert.True(t, model.IsByzantineThresholdExceededError(err)) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, notifier := newForks(t) - notifier.On("OnDoubleProposeDetected", blocks[2].Block, blocks[1].Block).Return(nil) - - // As [(◄3') 5] is not certified, it will not be added to Forks. However, its QC (◄3') is - // delivered to Forks as part of the *certified* block [(◄2) 3']. - err = addCertifiedBlocksToForks(forks, blocks) - assert.True(t, model.IsByzantineThresholdExceededError(err)) - }) -} - -// TestConflictingFinalizedForks checks that finalizing 2 conflicting forks should return model.ByzantineThresholdExceededError -// We ingest the the following block tree: -// -// [(◄1) 2] [(◄2) 3] [(◄3) 4] [(◄4) 5] -// [(◄2) 6] [(◄6) 7] [(◄7) 8] -// -// Here, both blocks [(◄2) 3] and [(◄2) 6] satisfy the finalization condition, i.e. we have a fork -// in the finalized blocks, which should result in a model.ByzantineThresholdExceededError exception. -func TestConflictingFinalizedForks(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). - Add(2, 3). - Add(3, 4). - Add(4, 5). // finalizes [(◄2) 3] - Add(2, 6). - Add(6, 7). - Add(7, 8). // finalizes [(◄2) 6], conflicting with conflicts with [(◄2) 3] - Blocks() - require.Nil(t, err) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - err = addProposalsToForks(forks, blocks) - assert.True(t, model.IsByzantineThresholdExceededError(err)) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - err = addCertifiedBlocksToForks(forks, blocks) - assert.True(t, model.IsByzantineThresholdExceededError(err)) - }) -} - -// TestAddUnconnectedProposal checks that adding a proposal which does not connect to the -// latest finalized block returns a `model.MissingBlockError` -// - receives [(◄2) 3] -// - should return `model.MissingBlockError`, because the parent is above the pruning -// threshold, but Forks does not know its parent -func TestAddUnconnectedProposal(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). // we will skip this block [(◄1) 2] - Add(2, 3). // [(◄2) 3] - Blocks() - require.Nil(t, err) - - t.Run("ingest proposals", func(t *testing.T) { - forks, _ := newForks(t) - err := forks.AddProposal(blocks[1].Block) - require.Error(t, err) - assert.True(t, model.IsMissingBlockError(err)) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, _ := newForks(t) - err := forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[1].Block)) - require.Error(t, err) - assert.True(t, model.IsMissingBlockError(err)) - }) -} - -// TestGetProposal tests that we can retrieve stored proposals. Here, we test that -// attempting to retrieve nonexistent or pruned proposals fails without causing an exception. -// - Forks receives [(◄1) 2] [(◄2) 3] [(◄3) 4], then [(◄4) 5] -// - should finalize [(◄1) 2], then [(◄2) 3] -func TestGetProposal(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). // [(◄1) 2] - Add(2, 3). // [(◄2) 3] - Add(3, 4). // [(◄3) 4] - Add(4, 5). // [(◄4) 5] - Blocks() - require.Nil(t, err) - - t.Run("ingest proposals", func(t *testing.T) { - blocksAddedFirst := blocks[:3] // [(◄1) 2] [(◄2) 3] [(◄3) 4] - remainingBlock := blocks[3].Block // [(◄4) 5] - forks, _ := newForks(t) - - // should be unable to retrieve a block before it is added - _, ok := forks.GetBlock(blocks[0].Block.BlockID) - assert.False(t, ok) - - // add first 3 blocks - should finalize [(◄1) 2] - err = addProposalsToForks(forks, blocksAddedFirst) - require.Nil(t, err) - - // should be able to retrieve all stored blocks - for _, proposal := range blocksAddedFirst { - b, ok := forks.GetBlock(proposal.Block.BlockID) - assert.True(t, ok) - assert.Equal(t, proposal.Block, b) - } - - // add remaining block [(◄4) 5] - should finalize [(◄2) 3] and prune [(◄1) 2] - require.Nil(t, forks.AddProposal(remainingBlock)) - - // should be able to retrieve just added block - b, ok := forks.GetBlock(remainingBlock.BlockID) - assert.True(t, ok) - assert.Equal(t, remainingBlock, b) - - // should be unable to retrieve pruned block - _, ok = forks.GetBlock(blocksAddedFirst[0].Block.BlockID) - assert.False(t, ok) - }) - - // Caution: finalization is driven by QCs. Therefore, we include the QC for block 3 - // in the first batch of blocks that we add. This is analogous to previous test case, - // except that we are delivering the QC (◄3) as part of the certified block of view 2 - // [(◄2) 3] (◄3) - // while in the previous sub-test, the QC (◄3) was delivered as part of block [(◄3) 4] - t.Run("ingest certified blocks", func(t *testing.T) { - blocksAddedFirst := toCertifiedBlocks(t, toBlocks(blocks[:2])...) // [(◄1) 2] [(◄2) 3] (◄3) - remainingBlock := toCertifiedBlock(t, blocks[2].Block) // [(◄3) 4] (◄4) - forks, _ := newForks(t) - - // should be unable to retrieve a block before it is added - _, ok := forks.GetBlock(blocks[0].Block.BlockID) - assert.False(t, ok) - - // add first blocks - should finalize [(◄1) 2] - err := forks.AddCertifiedBlock(blocksAddedFirst[0]) - require.Nil(t, err) - err = forks.AddCertifiedBlock(blocksAddedFirst[1]) - require.Nil(t, err) - - // should be able to retrieve all stored blocks - for _, proposal := range blocksAddedFirst { - b, ok := forks.GetBlock(proposal.Block.BlockID) - assert.True(t, ok) - assert.Equal(t, proposal.Block, b) - } - - // add remaining block [(◄4) 5] - should finalize [(◄2) 3] and prune [(◄1) 2] - require.Nil(t, forks.AddCertifiedBlock(remainingBlock)) - - // should be able to retrieve just added block - b, ok := forks.GetBlock(remainingBlock.Block.BlockID) - assert.True(t, ok) - assert.Equal(t, remainingBlock.Block, b) - - // should be unable to retrieve pruned block - _, ok = forks.GetBlock(blocksAddedFirst[0].Block.BlockID) - assert.False(t, ok) - }) -} - -// TestGetProposalsForView tests retrieving proposals for a view (also including double proposals). -// - Forks receives [(◄1) 2] [(◄2) 4] [(◄2) 4'], -// where [(◄2) 4'] is a double proposal, because it has the same view as [(◄2) 4] -// -// Expected behaviour: -// - Forks should store all the blocks -// - Forks should emit a `OnDoubleProposeDetected` notification -// - we can retrieve all blocks, including the double proposal -func TestGetProposalsForView(t *testing.T) { - blocks, err := NewBlockBuilder(). - Add(1, 2). // [(◄1) 2] - Add(2, 4). // [(◄2) 4] - AddVersioned(2, 4, 0, 1). // [(◄2) 4'] - Blocks() - require.Nil(t, err) - - t.Run("ingest proposals", func(t *testing.T) { - forks, notifier := newForks(t) - notifier.On("OnDoubleProposeDetected", blocks[2].Block, blocks[1].Block).Once() - - err = addProposalsToForks(forks, blocks) - require.Nil(t, err) - - // expect 1 proposal at view 2 - proposals := forks.GetBlocksForView(2) - assert.Len(t, proposals, 1) - assert.Equal(t, blocks[0].Block, proposals[0]) - - // expect 2 proposals at view 4 - proposals = forks.GetBlocksForView(4) - assert.Len(t, proposals, 2) - assert.ElementsMatch(t, toBlocks(blocks[1:]), proposals) - - // expect 0 proposals at view 3 - proposals = forks.GetBlocksForView(3) - assert.Len(t, proposals, 0) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, notifier := newForks(t) - notifier.On("OnDoubleProposeDetected", blocks[2].Block, blocks[1].Block).Once() - - err := forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[0].Block)) - require.Nil(t, err) - err = forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[1].Block)) - require.Nil(t, err) - err = forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[2].Block)) - require.Nil(t, err) - - // expect 1 proposal at view 2 - proposals := forks.GetBlocksForView(2) - assert.Len(t, proposals, 1) - assert.Equal(t, blocks[0].Block, proposals[0]) - - // expect 2 proposals at view 4 - proposals = forks.GetBlocksForView(4) - assert.Len(t, proposals, 2) - assert.ElementsMatch(t, toBlocks(blocks[1:]), proposals) - - // expect 0 proposals at view 3 - proposals = forks.GetBlocksForView(3) - assert.Len(t, proposals, 0) - }) -} - -// TestNotifications tests that Forks emits the expected events: -// - Forks receives [(◄1) 2] [(◄2) 3] [(◄3) 4] -// -// Expected Behaviour: -// - Each of the ingested blocks should result in an `OnBlockIncorporated` notification -// - Forks should finalize [(◄1) 2], resulting in a `MakeFinal` event and an `OnFinalizedBlock` event -func TestNotifications(t *testing.T) { - builder := NewBlockBuilder(). - Add(1, 2). - Add(2, 3). - Add(3, 4) - blocks, err := builder.Blocks() - require.Nil(t, err) - - t.Run("ingest proposals", func(t *testing.T) { - notifier := &mocks.Consumer{} - // 4 blocks including the genesis are incorporated - notifier.On("OnBlockIncorporated", mock.Anything).Return(nil).Times(4) - notifier.On("OnFinalizedBlock", blocks[0].Block).Once() - finalizationCallback := mockmodule.NewFinalizer(t) - finalizationCallback.On("MakeFinal", blocks[0].Block.BlockID).Return(nil).Once() - - forks, err := NewForks2(builder.GenesisBlock(), finalizationCallback, notifier) - require.NoError(t, err) - require.NoError(t, addProposalsToForks(forks, blocks)) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - notifier := &mocks.Consumer{} - // 4 blocks including the genesis are incorporated - notifier.On("OnBlockIncorporated", mock.Anything).Return(nil).Times(4) - notifier.On("OnFinalizedBlock", blocks[0].Block).Once() - finalizationCallback := mockmodule.NewFinalizer(t) - finalizationCallback.On("MakeFinal", blocks[0].Block.BlockID).Return(nil).Once() - - forks, err := NewForks2(builder.GenesisBlock(), finalizationCallback, notifier) - require.NoError(t, err) - require.NoError(t, addCertifiedBlocksToForks(forks, blocks)) - }) -} - -// TestFinalizingMultipleBlocks tests that `OnFinalizedBlock` notifications are emitted in correct order -// when there are multiple blocks finalized by adding a _single_ block. -// - receiving [(◄1) 3] [(◄3) 5] [(◄5) 7] [(◄7) 11] [(◄11) 12] should not finalize any blocks, -// because there is no 2-chain with the first chain link being a _direct_ 1-chain -// - adding [(◄12) 22] should finalize up to block [(◄6) 11] -// -// This test verifies the following expected properties: -// 1. Safety under reentrancy: -// While Forks is single-threaded, there is still the possibility of reentrancy. Specifically, the -// consumers of our finalization events are served by the goroutine executing Forks. It is conceivable -// that a consumer might access Forks and query the latest finalization proof. This would be legal, if -// the component supplying the goroutine to Forks also consumes the notifications. Therefore, for API -// safety, we require forks to _first update_ its `FinalityProof()` before it emits _any_ events. -// 2. For each finalized block, `finalizationCallback` event is executed _before_ `OnFinalizedBlock` notifications. -// 3. Blocks are finalized in order of increasing height (without skipping any blocks). -func TestFinalizingMultipleBlocks(t *testing.T) { - builder := NewBlockBuilder(). - Add(1, 3). // index 0: [(◄1) 2] - Add(3, 5). // index 1: [(◄2) 4] - Add(5, 7). // index 2: [(◄4) 6] - Add(7, 11). // index 3: [(◄6) 11] -- expected to be finalized - Add(11, 12). // index 4: [(◄11) 12] - Add(12, 22) // index 5: [(◄12) 22] - blocks, err := builder.Blocks() - require.Nil(t, err) - - // The Finality Proof should right away point to the _latest_ finalized block. Subsequently emitting - // Finalization events for lower blocks is fine, because notifications are guaranteed to be - // _eventually_ arriving. I.e. consumers expect notifications / events to be potentially lagging behind. - expectedFinalityProof := makeFinalityProof(t, blocks[3].Block, blocks[4].Block, blocks[5].Block.QC) - - setupForksAndAssertions := func() (*Forks2, *mockmodule.Finalizer, *mocks.Consumer) { - // initialize Forks with custom event consumers so we can check order of emitted events - notifier := &mocks.Consumer{} - finalizationCallback := mockmodule.NewFinalizer(t) - notifier.On("OnBlockIncorporated", mock.Anything).Return(nil) - forks, err := NewForks2(builder.GenesisBlock(), finalizationCallback, notifier) - require.NoError(t, err) - - // expecting finalization of [(◄1) 2] [(◄2) 4] [(◄4) 6] [(◄6) 11] in this order - blocksAwaitingFinalization := toBlockAwaitingFinalization(toBlocks(blocks[:4])) - - finalizationCallback.On("MakeFinal", mock.Anything).Run(func(args mock.Arguments) { - requireFinalityProof(t, forks, expectedFinalityProof) // Requirement 1: forks should _first update_ its `FinalityProof()` before it emits _any_ events - - // Requirement 3: finalized in order of increasing height (without skipping any blocks). - expectedNextFinalizationEvents := blocksAwaitingFinalization[0] - require.Equal(t, expectedNextFinalizationEvents.Block.BlockID, args[0]) - - // Requirement 2: finalized block, `finalizationCallback` event is executed _before_ `OnFinalizedBlock` notifications. - // no duplication of events under normal operations expected - require.False(t, expectedNextFinalizationEvents.MakeFinalCalled) - require.False(t, expectedNextFinalizationEvents.OnFinalizedBlockEmitted) - expectedNextFinalizationEvents.MakeFinalCalled = true - }).Return(nil).Times(4) - - notifier.On("OnFinalizedBlock", mock.Anything).Run(func(args mock.Arguments) { - requireFinalityProof(t, forks, expectedFinalityProof) // Requirement 1: forks should _first update_ its `FinalityProof()` before it emits _any_ events - - // Requirement 3: finalized in order of increasing height (without skipping any blocks). - expectedNextFinalizationEvents := blocksAwaitingFinalization[0] - require.Equal(t, expectedNextFinalizationEvents.Block, args[0]) - - // Requirement 2: finalized block, `finalizationCallback` event is executed _before_ `OnFinalizedBlock` notifications. - // no duplication of events under normal operations expected - require.True(t, expectedNextFinalizationEvents.MakeFinalCalled) - require.False(t, expectedNextFinalizationEvents.OnFinalizedBlockEmitted) - expectedNextFinalizationEvents.OnFinalizedBlockEmitted = true - - // At this point, `MakeFinal` and `OnFinalizedBlock` have both been emitted for the block, so we are done with it - blocksAwaitingFinalization = blocksAwaitingFinalization[1:] - }).Times(4) - - return forks, finalizationCallback, notifier - } - - t.Run("ingest proposals", func(t *testing.T) { - forks, finalizationCallback, notifier := setupForksAndAssertions() - err = addProposalsToForks(forks, blocks[:5]) // adding [(◄1) 2] [(◄2) 4] [(◄4) 6] [(◄6) 11] [(◄11) 12] - require.Nil(t, err) - requireOnlyGenesisBlockFinalized(t, forks) // finalization should still be at the genesis block - - require.NoError(t, forks.AddProposal(blocks[5].Block)) // adding [(◄12) 22] should trigger finalization events - requireFinalityProof(t, forks, expectedFinalityProof) - finalizationCallback.AssertExpectations(t) - notifier.AssertExpectations(t) - }) - - t.Run("ingest certified blocks", func(t *testing.T) { - forks, finalizationCallback, notifier := setupForksAndAssertions() - // adding [(◄1) 2] [(◄2) 4] [(◄4) 6] [(◄6) 11] (◄11) - require.NoError(t, forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[0].Block))) - require.NoError(t, forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[1].Block))) - require.NoError(t, forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[2].Block))) - require.NoError(t, forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[3].Block))) - require.Nil(t, err) - requireOnlyGenesisBlockFinalized(t, forks) // finalization should still be at the genesis block - - // adding certified block [(◄11) 12] (◄12) should trigger finalization events - require.NoError(t, forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[4].Block))) - requireFinalityProof(t, forks, expectedFinalityProof) - finalizationCallback.AssertExpectations(t) - notifier.AssertExpectations(t) - }) -} - -//* ************************************* internal functions ************************************* */ - -func newForks(t *testing.T) (*Forks2, *mocks.Consumer) { - notifier := mocks.NewConsumer(t) - notifier.On("OnBlockIncorporated", mock.Anything).Return(nil).Maybe() - notifier.On("OnFinalizedBlock", mock.Anything).Maybe() - finalizationCallback := mockmodule.NewFinalizer(t) - finalizationCallback.On("MakeFinal", mock.Anything).Return(nil).Maybe() - - genesisBQ := makeGenesis() - - forks, err := NewForks2(genesisBQ, finalizationCallback, notifier) - - require.Nil(t, err) - return forks, notifier -} - -// addProposalsToForks adds all the given blocks to Forks, in order. -// If any errors occur, returns the first one. -func addProposalsToForks(forks *Forks2, proposals []*model.Proposal) error { - for _, proposal := range proposals { - err := forks.AddProposal(proposal.Block) - if err != nil { - return fmt.Errorf("test failed to add proposal for view %d: %w", proposal.Block.View, err) - } - } - return nil -} - -// addCertifiedBlocksToForks iterates over all proposals, caches them locally in a map, -// constructs certified blocks whenever possible and adds the certified blocks to forks, -// Note: if proposals is a single fork, the _last block_ in the slice will not be added, -// -// because there is no qc for it -// -// If any errors occur, returns the first one. -func addCertifiedBlocksToForks(forks *Forks2, proposals []*model.Proposal) error { - uncertifiedBlocks := make(map[flow.Identifier]*model.Block) - for _, proposal := range proposals { - uncertifiedBlocks[proposal.Block.BlockID] = proposal.Block - parentID := proposal.Block.QC.BlockID - parent, found := uncertifiedBlocks[parentID] - if !found { - continue - } - delete(uncertifiedBlocks, parentID) - - certParent, err := model.NewCertifiedBlock(parent, proposal.Block.QC) - if err != nil { - return fmt.Errorf("test failed to creat certified block for view %d: %w", certParent.Block.View, err) - } - err = forks.AddCertifiedBlock(&certParent) - if err != nil { - return fmt.Errorf("test failed to add certified block for view %d: %w", certParent.Block.View, err) - } - } - - return nil -} - -// requireLatestFinalizedBlock asserts that the latest finalized block has the given view and qc view. -func requireLatestFinalizedBlock(t *testing.T, forks *Forks2, expectedFinalized *model.Block) { - require.Equal(t, expectedFinalized, forks.FinalizedBlock(), "finalized block is not as expected") - require.Equal(t, forks.FinalizedView(), uint64(expectedFinalized.View), "FinalizedView returned wrong value") -} - -// requireOnlyGenesisBlockFinalized asserts that no blocks have been finalized beyond the genesis block. -// Caution: does not inspect output of `forks.FinalityProof()` -func requireOnlyGenesisBlockFinalized(t *testing.T, forks *Forks2) { - genesis := makeGenesis() - require.Equal(t, forks.FinalizedBlock(), genesis.Block, "finalized block is not the genesis block") - require.Equal(t, forks.FinalizedBlock().View, genesis.Block.View) - require.Equal(t, forks.FinalizedBlock().View, genesis.CertifyingQC.View) - require.Equal(t, forks.FinalizedView(), genesis.Block.View, "finalized block has wrong qc") - - finalityProof, isKnown := forks.FinalityProof() - require.Nil(t, finalityProof, "expecting finality proof to be nil for genesis block at initialization") - require.False(t, isKnown, "no finality proof should be known for genesis block at initialization") -} - -// requireNoBlocksFinalized asserts that no blocks have been finalized (genesis is latest finalized block). -func requireFinalityProof(t *testing.T, forks *Forks2, expectedFinalityProof *FinalityProof) { - finalityProof, isKnown := forks.FinalityProof() - require.True(t, isKnown) - require.Equal(t, expectedFinalityProof, finalityProof) - require.Equal(t, forks.FinalizedBlock(), expectedFinalityProof.Block) - require.Equal(t, forks.FinalizedView(), expectedFinalityProof.Block.View) -} - -// toBlocks converts the given proposals to slice of blocks -// TODO: change `BlockBuilder` to generate model.Blocks instead of model.Proposals and then remove this method -func toBlocks(proposals []*model.Proposal) []*model.Block { - blocks := make([]*model.Block, 0, len(proposals)) - for _, b := range proposals { - blocks = append(blocks, b.Block) - } - return blocks -} - -// toCertifiedBlock generates a QC for the given block and returns their combination as a certified block -func toCertifiedBlock(t *testing.T, block *model.Block) *model.CertifiedBlock { - qc := &flow.QuorumCertificate{ - View: block.View, - BlockID: block.BlockID, - } - cb, err := model.NewCertifiedBlock(block, qc) - require.Nil(t, err) - return &cb -} - -// toCertifiedBlocks generates a QC for the given block and returns their combination as a certified blocks -func toCertifiedBlocks(t *testing.T, blocks ...*model.Block) []*model.CertifiedBlock { - certBlocks := make([]*model.CertifiedBlock, 0, len(blocks)) - for _, b := range blocks { - certBlocks = append(certBlocks, toCertifiedBlock(t, b)) - } - return certBlocks -} - -func makeFinalityProof(t *testing.T, block *model.Block, directChild *model.Block, qcCertifyingChild *flow.QuorumCertificate) *FinalityProof { - c, err := model.NewCertifiedBlock(directChild, qcCertifyingChild) // certified child of FinalizedBlock - require.NoError(t, err) - return &FinalityProof{block, c} -} - -// blockAwaitingFinalization is intended for tracking finalization events and their order for a specific block -type blockAwaitingFinalization struct { - Block *model.Block - MakeFinalCalled bool // indicates whether `Finalizer.MakeFinal` was called - OnFinalizedBlockEmitted bool // indicates whether `OnFinalizedBlockCalled` notification was emitted -} - -// toBlockAwaitingFinalization creates a `blockAwaitingFinalization` tracker for each input block -func toBlockAwaitingFinalization(blocks []*model.Block) []*blockAwaitingFinalization { - trackers := make([]*blockAwaitingFinalization, 0, len(blocks)) - for _, b := range blocks { - tracker := &blockAwaitingFinalization{b, false, false} - trackers = append(trackers, tracker) - } - return trackers -} diff --git a/consensus/hotstuff/forks/forks_test.go b/consensus/hotstuff/forks/forks_test.go index d8aaa8bec3f..9662533dd0d 100644 --- a/consensus/hotstuff/forks/forks_test.go +++ b/consensus/hotstuff/forks/forks_test.go @@ -1,4 +1,4 @@ -package forks_test +package forks import ( "fmt" @@ -8,480 +8,870 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/consensus/hotstuff/forks" - "github.com/onflow/flow-go/consensus/hotstuff/helper" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" mockmodule "github.com/onflow/flow-go/module/mock" ) -/* *************************************************************************************************** - * TO BE REMOVED: I have moved the tests for the prior version of Forks to this file for reference. - *************************************************************************************************** */ +/***************************************************************************** + * NOTATION: * + * A block is denoted as [◄() ]. * + * For example, [◄(1) 2] means: a block of view 2 that has a QC for view 1. * + *****************************************************************************/ -// NOTATION: -// A block is denoted as [, ]. -// For example, [1,2] means: a block of view 2 has a QC for view 1. +// TestInitialization verifies that at initialization, Forks reports: +// - the root / genesis block as finalized +// - it has no finalization proof for the root / genesis block (block and its finalization is trusted) +func TestInitialization(t *testing.T) { + forks, _ := newForks(t) + requireOnlyGenesisBlockFinalized(t, forks) + _, hasProof := forks.FinalityProof() + require.False(t, hasProof) +} -// TestFinalize_Direct1Chain tests adding a direct 1-chain. -// receives [1,2] [2,3] -// it should not finalize any block because there is no finalizable 2-chain. +// TestFinalize_Direct1Chain tests adding a direct 1-chain on top of the genesis block: +// - receives [◄(1) 2] [◄(2) 5] +// +// Expected behaviour: +// - On the one hand, Forks should not finalize any _additional_ blocks, because there is +// no finalizable 2-chain for [◄(1) 2]. Hence, finalization no events should be emitted. +// - On the other hand, after adding the two blocks, Forks has enough knowledge to construct +// a FinalityProof for the genesis block. func TestFinalize_Direct1Chain(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - + builder := NewBlockBuilder(). + Add(1, 2). + Add(2, 3) blocks, err := builder.Blocks() require.Nil(t, err) - forks, _ := newForks(t) - - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) - - requireNoBlocksFinalized(t, forks) + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + + // adding block [◄(1) 2] should not finalize anything + // as the genesis block is trusted, there should be no FinalityProof available for it + require.NoError(t, forks.AddValidatedBlock(blocks[0])) + requireOnlyGenesisBlockFinalized(t, forks) + _, hasProof := forks.FinalityProof() + require.False(t, hasProof) + + // After adding block [◄(2) 3], Forks has enough knowledge to construct a FinalityProof for the + // genesis block. However, finalization remains at the genesis block, so no events should be emitted. + expectedFinalityProof := makeFinalityProof(t, builder.GenesisBlock().Block, blocks[0], blocks[1].QC) + require.NoError(t, forks.AddValidatedBlock(blocks[1])) + requireLatestFinalizedBlock(t, forks, builder.GenesisBlock().Block) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + + // After adding CertifiedBlock [◄(1) 2] ◄(2), Forks has enough knowledge to construct a FinalityProof for + // the genesis block. However, finalization remains at the genesis block, so no events should be emitted. + expectedFinalityProof := makeFinalityProof(t, builder.GenesisBlock().Block, blocks[0], blocks[1].QC) + c, err := model.NewCertifiedBlock(blocks[0], blocks[1].QC) + require.NoError(t, err) + + require.NoError(t, forks.AddCertifiedBlock(&c)) + requireLatestFinalizedBlock(t, forks, builder.GenesisBlock().Block) + requireFinalityProof(t, forks, expectedFinalityProof) + }) } // TestFinalize_Direct2Chain tests adding a direct 1-chain on a direct 1-chain (direct 2-chain). -// receives [1,2] [2,3] [3,4] -// it should finalize [1,2] +// - receives [◄(1) 2] [◄(2) 3] [◄(3) 4] +// - Forks should finalize [◄(1) 2] func TestFinalize_Direct2Chain(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - - blocks, err := builder.Blocks() + blocks, err := NewBlockBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 4). + Blocks() require.Nil(t, err) + expectedFinalityProof := makeFinalityProof(t, blocks[0], blocks[1], blocks[2].QC) - forks, _ := newForks(t) + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedBlockToForks(forks, blocks)) - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) + requireLatestFinalizedBlock(t, forks, blocks[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) - requireLatestFinalizedBlock(t, forks, 1, 2) + requireLatestFinalizedBlock(t, forks, blocks[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) } // TestFinalize_DirectIndirect2Chain tests adding an indirect 1-chain on a direct 1-chain. -// receives [1,2] [2,3] [3,5] -// it should finalize [1,2] +// receives [◄(1) 2] [◄(2) 3] [◄(3) 5] +// it should finalize [◄(1) 2] func TestFinalize_DirectIndirect2Chain(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 5) - - blocks, err := builder.Blocks() + blocks, err := NewBlockBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 5). + Blocks() require.Nil(t, err) + expectedFinalityProof := makeFinalityProof(t, blocks[0], blocks[1], blocks[2].QC) - forks, _ := newForks(t) + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedBlockToForks(forks, blocks)) - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) + requireLatestFinalizedBlock(t, forks, blocks[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) - requireLatestFinalizedBlock(t, forks, 1, 2) + requireLatestFinalizedBlock(t, forks, blocks[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) } // TestFinalize_IndirectDirect2Chain tests adding a direct 1-chain on an indirect 1-chain. -// receives [1,2] [2,4] [4,5] -// it should not finalize any blocks because there is no finalizable 2-chain. +// - Forks receives [◄(1) 3] [◄(3) 5] [◄(7) 7] +// - it should not finalize any blocks because there is no finalizable 2-chain. func TestFinalize_IndirectDirect2Chain(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 4) - builder.Add(4, 5) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - forks, _ := newForks(t) - - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) - - requireNoBlocksFinalized(t, forks) + blocks, err := NewBlockBuilder(). + Add(1, 3). + Add(3, 5). + Add(5, 7). + Blocks() + require.Nil(t, err) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedBlockToForks(forks, blocks)) + + requireOnlyGenesisBlockFinalized(t, forks) + _, hasProof := forks.FinalityProof() + require.False(t, hasProof) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) + + requireOnlyGenesisBlockFinalized(t, forks) + _, hasProof := forks.FinalityProof() + require.False(t, hasProof) + }) } -// TestFinalize_Direct2ChainOnIndirect tests adding a direct 2-chain on an indirect 2-chain. -// The head of highest 2-chain should be finalized. -// receives [1,3] [3,5] [5,6] [6,7] [7,8] -// it should finalize [5,6] +// TestFinalize_Direct2ChainOnIndirect tests adding a direct 2-chain on an indirect 2-chain: +// - ingesting [◄(1) 3] [◄(3) 5] [◄(5) 6] [◄(6) 7] [◄(7) 8] +// - should result in finalization of [◄(5) 6] func TestFinalize_Direct2ChainOnIndirect(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 3) - builder.Add(3, 5) - builder.Add(5, 6) - builder.Add(6, 7) - builder.Add(7, 8) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - forks, _ := newForks(t) - - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) - - requireLatestFinalizedBlock(t, forks, 5, 6) + blocks, err := NewBlockBuilder(). + Add(1, 3). + Add(3, 5). + Add(5, 6). + Add(6, 7). + Add(7, 8). + Blocks() + require.Nil(t, err) + expectedFinalityProof := makeFinalityProof(t, blocks[2], blocks[3], blocks[4].QC) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedBlockToForks(forks, blocks)) + + requireLatestFinalizedBlock(t, forks, blocks[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) + + requireLatestFinalizedBlock(t, forks, blocks[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) } -// TestFinalize_Direct2ChainOnDirect tests adding a sequence of direct 2-chains. -// The head of highest 2-chain should be finalized. -// receives [1,2] [2,3] [3,4] [4,5] [5,6] -// it should finalize [3,4] +// TestFinalize_Direct2ChainOnDirect tests adding a sequence of direct 2-chains: +// - ingesting [◄(1) 2] [◄(2) 3] [◄(3) 4] [◄(4) 5] [◄(5) 6] +// - should result in finalization of [◄(3) 4] func TestFinalize_Direct2ChainOnDirect(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(5, 6) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - forks, _ := newForks(t) - - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) - - requireLatestFinalizedBlock(t, forks, 3, 4) + blocks, err := NewBlockBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 4). + Add(4, 5). + Add(5, 6). + Blocks() + require.Nil(t, err) + expectedFinalityProof := makeFinalityProof(t, blocks[2], blocks[3], blocks[4].QC) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedBlockToForks(forks, blocks)) + + requireLatestFinalizedBlock(t, forks, blocks[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) + + requireLatestFinalizedBlock(t, forks, blocks[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) } -// TestFinalize_Multiple2Chains tests the case where a block can be finalized -// by different 2-chains. -// receives [1,2] [2,3] [3,5] [3,6] [3,7] -// it should finalize [1,2] +// TestFinalize_Multiple2Chains tests the case where a block can be finalized by different 2-chains. +// - ingesting [◄(1) 2] [◄(2) 3] [◄(3) 5] [◄(3) 6] [◄(3) 7] +// - should result in finalization of [◄(1) 2] func TestFinalize_Multiple2Chains(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 5) - builder.Add(3, 6) - builder.Add(3, 7) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - forks, _ := newForks(t) - - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) - - requireLatestFinalizedBlock(t, forks, 1, 2) + blocks, err := NewBlockBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 5). + Add(3, 6). + Add(3, 7). + Blocks() + require.Nil(t, err) + expectedFinalityProof := makeFinalityProof(t, blocks[0], blocks[1], blocks[2].QC) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedBlockToForks(forks, blocks)) + + requireLatestFinalizedBlock(t, forks, blocks[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) + + requireLatestFinalizedBlock(t, forks, blocks[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) } -// TestFinalize_OrphanedFork tests that we can finalize a block which causes -// a conflicting fork to be orphaned. -// receives [1,2] [2,3] [2,4] [4,5] [5,6] -// it should finalize [2,4] +// TestFinalize_OrphanedFork tests that we can finalize a block which causes a conflicting fork to be orphaned. +// We ingest the the following block tree: +// +// [◄(1) 2] [◄(2) 3] +// [◄(2) 4] [◄(4) 5] [◄(5) 6] +// +// which should result in finalization of [◄(2) 4] and pruning of [◄(2) 3] func TestFinalize_OrphanedFork(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(2, 4) - builder.Add(4, 5) - builder.Add(5, 6) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - forks, _ := newForks(t) - - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) - - requireLatestFinalizedBlock(t, forks, 2, 4) + blocks, err := NewBlockBuilder(). + Add(1, 2). // [◄(1) 2] + Add(2, 3). // [◄(2) 3], should eventually be pruned + Add(2, 4). // [◄(2) 4], should eventually be finalized + Add(4, 5). // [◄(4) 5] + Add(5, 6). // [◄(5) 6] + Blocks() + require.Nil(t, err) + expectedFinalityProof := makeFinalityProof(t, blocks[2], blocks[3], blocks[4].QC) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedBlockToForks(forks, blocks)) + + require.False(t, forks.IsKnownBlock(blocks[1].BlockID)) + requireLatestFinalizedBlock(t, forks, blocks[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) + + require.False(t, forks.IsKnownBlock(blocks[1].BlockID)) + requireLatestFinalizedBlock(t, forks, blocks[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) } // TestDuplication tests that delivering the same block/qc multiple times has // the same end state as delivering the block/qc once. -// receives [1,2] [2,3] [2,3] [3,4] [3,4] [4,5] [4,5] -// it should finalize [2,3] +// - Forks receives [◄(1) 2] [◄(2) 3] [◄(2) 3] [◄(3) 4] [◄(3) 4] [◄(4) 5] [◄(4) 5] +// - it should finalize [◄(2) 3] func TestDuplication(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(4, 5) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - forks, _ := newForks(t) - - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) - - requireLatestFinalizedBlock(t, forks, 2, 3) + blocks, err := NewBlockBuilder(). + Add(1, 2). + Add(2, 3). + Add(2, 3). + Add(3, 4). + Add(3, 4). + Add(4, 5). + Add(4, 5). + Blocks() + require.Nil(t, err) + expectedFinalityProof := makeFinalityProof(t, blocks[1], blocks[3], blocks[5].QC) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedBlockToForks(forks, blocks)) + + requireLatestFinalizedBlock(t, forks, blocks[1]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedBlocksToForks(forks, blocks)) + + requireLatestFinalizedBlock(t, forks, blocks[1]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) } // TestIgnoreBlocksBelowFinalizedView tests that blocks below finalized view are ignored. -// receives [1,2] [2,3] [3,4] [1,5] -// it should finalize [1,2] +// - Forks receives [◄(1) 2] [◄(2) 3] [◄(3) 4] [◄(1) 5] +// - it should finalize [◄(1) 2] func TestIgnoreBlocksBelowFinalizedView(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(1, 5) - + builder := NewBlockBuilder(). + Add(1, 2). // [◄(1) 2] + Add(2, 3). // [◄(2) 3] + Add(3, 4). // [◄(3) 4] + Add(1, 5) // [◄(1) 5] blocks, err := builder.Blocks() require.Nil(t, err) - - forks, _ := newForks(t) - - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) - - requireLatestFinalizedBlock(t, forks, 1, 2) + expectedFinalityProof := makeFinalityProof(t, blocks[0], blocks[1], blocks[2].QC) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + // initialize forks and add first 3 blocks: + // * block [◄(1) 2] should then be finalized + // * and block [1] should be pruned + forks, _ := newForks(t) + require.Nil(t, addValidatedBlockToForks(forks, blocks[:3])) + + // sanity checks to confirm correct test setup + requireLatestFinalizedBlock(t, forks, blocks[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + require.False(t, forks.IsKnownBlock(builder.GenesisBlock().ID())) + + // adding block [◄(1) 5]: note that QC is _below_ the pruning threshold, i.e. cannot resolve the parent + // * Forks should store block, despite the parent already being pruned + // * finalization should not change + orphanedBlock := blocks[3] + require.Nil(t, forks.AddValidatedBlock(orphanedBlock)) + require.True(t, forks.IsKnownBlock(orphanedBlock.BlockID)) + requireLatestFinalizedBlock(t, forks, blocks[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + // initialize forks and add first 3 blocks: + // * block [◄(1) 2] should then be finalized + // * and block [1] should be pruned + forks, _ := newForks(t) + require.Nil(t, addCertifiedBlocksToForks(forks, blocks[:3])) + // sanity checks to confirm correct test setup + requireLatestFinalizedBlock(t, forks, blocks[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + require.False(t, forks.IsKnownBlock(builder.GenesisBlock().ID())) + + // adding block [◄(1) 5]: note that QC is _below_ the pruning threshold, i.e. cannot resolve the parent + // * Forks should store block, despite the parent already being pruned + // * finalization should not change + certBlockWithUnknownParent := toCertifiedBlock(t, blocks[3]) + require.Nil(t, forks.AddCertifiedBlock(certBlockWithUnknownParent)) + require.True(t, forks.IsKnownBlock(certBlockWithUnknownParent.Block.BlockID)) + requireLatestFinalizedBlock(t, forks, blocks[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) } // TestDoubleProposal tests that the DoubleProposal notification is emitted when two different -// proposals for the same view are added. -// receives [1,2] [2,3] [3,4] [4,5] [3,5'] -// it should finalize block [2,3], and emits an DoubleProposal event with ([3,5'], [4,5]) +// blocks for the same view are added. We ingest the the following block tree: +// +// / [◄(1) 2] +// [1] +// \ [◄(1) 2'] +// +// which should result in a DoubleProposal event referencing the blocks [◄(1) 2] and [◄(1) 2'] func TestDoubleProposal(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.AddVersioned(3, 5, 0, 1) - - blocks, err := builder.Blocks() + blocks, err := NewBlockBuilder(). + Add(1, 2). // [◄(1) 2] + AddVersioned(1, 2, 0, 1). // [◄(1) 2'] + Blocks() require.Nil(t, err) - forks, notifier := newForks(t) - notifier.On("OnDoubleProposeDetected", blocks[4].Block, blocks[3].Block).Once() + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", blocks[1], blocks[0]).Once() - err = addBlocksToForks(forks, blocks) - require.Nil(t, err) + err = addValidatedBlockToForks(forks, blocks) + require.Nil(t, err) + }) - requireLatestFinalizedBlock(t, forks, 2, 3) + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", blocks[1], blocks[0]).Once() + + err = forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[0])) // add [◄(1) 2] as certified block + require.Nil(t, err) + err = forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[1])) // add [◄(1) 2'] as certified block + require.Nil(t, err) + }) } // TestConflictingQCs checks that adding 2 conflicting QCs should return model.ByzantineThresholdExceededError -// receives [1,2] [2,3] [2,3'] [3,4] [3',5] -// it should return fatal error, because conflicting blocks 3 and 3' both received enough votes for QC +// We ingest the following block tree: +// +// [◄(1) 2] [◄(2) 3] [◄(3) 4] [◄(4) 6] +// [◄(2) 3'] [◄(3') 5] +// +// which should result in a `ByzantineThresholdExceededError`, because conflicting blocks 3 and 3' both have QCs func TestConflictingQCs(t *testing.T) { - builder := forks.NewBlockBuilder() - - builder.Add(1, 2) - builder.Add(2, 3) - builder.AddVersioned(2, 3, 0, 1) // make a conflicting proposal at view 3 - builder.Add(3, 4) // creates a QC for 3 - builder.AddVersioned(3, 5, 1, 0) // creates a QC for 3' - - blocks, err := builder.Blocks() - require.Nil(t, err) - - forks, notifier := newForks(t) - notifier.On("OnDoubleProposeDetected", blocks[2].Block, blocks[1].Block).Return(nil) - - err = addBlocksToForks(forks, blocks) - require.NotNil(t, err) - assert.True(t, model.IsByzantineThresholdExceededError(err)) + blocks, err := NewBlockBuilder(). + Add(1, 2). // [◄(1) 2] + Add(2, 3). // [◄(2) 3] + AddVersioned(2, 3, 0, 1). // [◄(2) 3'] + Add(3, 4). // [◄(3) 4] + Add(4, 6). // [◄(4) 6] + AddVersioned(3, 5, 1, 0). // [◄(3') 5] + Blocks() + require.Nil(t, err) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", blocks[2], blocks[1]).Return(nil) + + err = addValidatedBlockToForks(forks, blocks) + assert.True(t, model.IsByzantineThresholdExceededError(err)) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", blocks[2], blocks[1]).Return(nil) + + // As [◄(3') 5] is not certified, it will not be added to Forks. However, its QC ◄(3') is + // delivered to Forks as part of the *certified* block [◄(2) 3']. + err = addCertifiedBlocksToForks(forks, blocks) + assert.True(t, model.IsByzantineThresholdExceededError(err)) + }) } // TestConflictingFinalizedForks checks that finalizing 2 conflicting forks should return model.ByzantineThresholdExceededError -// receives [1,2] [2,3] [2,6] [3,4] [4,5] [6,7] [7,8] -// It should return fatal error, because 2 conflicting forks were finalized +// We ingest the the following block tree: +// +// [◄(1) 2] [◄(2) 3] [◄(3) 4] [◄(4) 5] +// [◄(2) 6] [◄(6) 7] [◄(7) 8] +// +// Here, both blocks [◄(2) 3] and [◄(2) 6] satisfy the finalization condition, i.e. we have a fork +// in the finalized blocks, which should result in a model.ByzantineThresholdExceededError exception. func TestConflictingFinalizedForks(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) // finalizes (2,3) - builder.Add(2, 6) - builder.Add(6, 7) - builder.Add(7, 8) // finalizes (2,6) conflicts with (2,3) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - forks, _ := newForks(t) - - err = addBlocksToForks(forks, blocks) - require.Error(t, err) - assert.True(t, model.IsByzantineThresholdExceededError(err)) + blocks, err := NewBlockBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 4). + Add(4, 5). // finalizes [◄(2) 3] + Add(2, 6). + Add(6, 7). + Add(7, 8). // finalizes [◄(2) 6], conflicting with conflicts with [◄(2) 3] + Blocks() + require.Nil(t, err) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + err = addValidatedBlockToForks(forks, blocks) + assert.True(t, model.IsByzantineThresholdExceededError(err)) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + err = addCertifiedBlocksToForks(forks, blocks) + assert.True(t, model.IsByzantineThresholdExceededError(err)) + }) } -// TestAddUnconnectedProposal checks that adding a proposal which does not connect to the -// latest finalized block returns an exception. -// receives [2,3] -// should return fatal error, because the proposal is invalid for addition to Forks -func TestAddUnconnectedProposal(t *testing.T) { - unconnectedProposal := helper.MakeProposal( - helper.WithBlock(helper.MakeBlock( - helper.WithBlockView(3), - ))) - - forks, _ := newForks(t) - - err := forks.AddProposal(unconnectedProposal) - require.Error(t, err) - // adding a disconnected block is an internal error, should return generic error - assert.False(t, model.IsByzantineThresholdExceededError(err)) +// TestAddDisconnectedBlock checks that adding a block which does not connect to the +// latest finalized block returns a `model.MissingBlockError` +// - receives [◄(2) 3] +// - should return `model.MissingBlockError`, because the parent is above the pruning +// threshold, but Forks does not know its parent +func TestAddDisconnectedBlock(t *testing.T) { + blocks, err := NewBlockBuilder(). + Add(1, 2). // we will skip this block [◄(1) 2] + Add(2, 3). // [◄(2) 3] + Blocks() + require.Nil(t, err) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, _ := newForks(t) + err := forks.AddValidatedBlock(blocks[1]) + require.Error(t, err) + assert.True(t, model.IsMissingBlockError(err)) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, _ := newForks(t) + err := forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[1])) + require.Error(t, err) + assert.True(t, model.IsMissingBlockError(err)) + }) } -// TestGetProposal tests that we can retrieve stored proposals. -// Attempting to retrieve nonexistent or pruned proposals should fail. -// receives [1,2] [2,3] [3,4], then [4,5] -// should finalize [1,2], then [2,3] -func TestGetProposal(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - - blocks, err := builder.Blocks() - require.Nil(t, err) - blocksAddedFirst := blocks[:3] // [1,2] [2,3] [3,4] - blocksAddedSecond := blocks[3:] // [4,5] - - forks, _ := newForks(t) - - // should be unable to retrieve a block before it is added - _, ok := forks.GetProposal(blocks[0].Block.BlockID) - assert.False(t, ok) +// TestGetBlock tests that we can retrieve stored blocks. Here, we test that +// attempting to retrieve nonexistent or pruned blocks fails without causing an exception. +// - Forks receives [◄(1) 2] [◄(2) 3] [◄(3) 4], then [◄(4) 5] +// - should finalize [◄(1) 2], then [◄(2) 3] +func TestGetBlock(t *testing.T) { + blocks, err := NewBlockBuilder(). + Add(1, 2). // [◄(1) 2] + Add(2, 3). // [◄(2) 3] + Add(3, 4). // [◄(3) 4] + Add(4, 5). // [◄(4) 5] + Blocks() + require.Nil(t, err) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + blocksAddedFirst := blocks[:3] // [◄(1) 2] [◄(2) 3] [◄(3) 4] + remainingBlock := blocks[3] // [◄(4) 5] + forks, _ := newForks(t) + + // should be unable to retrieve a block before it is added + _, ok := forks.GetBlock(blocks[0].BlockID) + assert.False(t, ok) + + // add first 3 blocks - should finalize [◄(1) 2] + err = addValidatedBlockToForks(forks, blocksAddedFirst) + require.Nil(t, err) + + // should be able to retrieve all stored blocks + for _, block := range blocksAddedFirst { + b, ok := forks.GetBlock(block.BlockID) + assert.True(t, ok) + assert.Equal(t, block, b) + } - // add first blocks - should finalize [1,2] - err = addBlocksToForks(forks, blocksAddedFirst) - require.Nil(t, err) + // add remaining block [◄(4) 5] - should finalize [◄(2) 3] and prune [◄(1) 2] + require.Nil(t, forks.AddValidatedBlock(remainingBlock)) - // should be able to retrieve all stored blocks - for _, proposal := range blocksAddedFirst { - got, ok := forks.GetProposal(proposal.Block.BlockID) + // should be able to retrieve just added block + b, ok := forks.GetBlock(remainingBlock.BlockID) assert.True(t, ok) - assert.Equal(t, proposal, got) - } + assert.Equal(t, remainingBlock, b) + + // should be unable to retrieve pruned block + _, ok = forks.GetBlock(blocksAddedFirst[0].BlockID) + assert.False(t, ok) + }) + + // Caution: finalization is driven by QCs. Therefore, we include the QC for block 3 + // in the first batch of blocks that we add. This is analogous to previous test case, + // except that we are delivering the QC ◄(3) as part of the certified block of view 2 + // [◄(2) 3] ◄(3) + // while in the previous sub-test, the QC ◄(3) was delivered as part of block [◄(3) 4] + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + blocksAddedFirst := toCertifiedBlocks(t, blocks[:2]...) // [◄(1) 2] [◄(2) 3] ◄(3) + remainingBlock := toCertifiedBlock(t, blocks[2]) // [◄(3) 4] ◄(4) + forks, _ := newForks(t) + + // should be unable to retrieve a block before it is added + _, ok := forks.GetBlock(blocks[0].BlockID) + assert.False(t, ok) + + // add first blocks - should finalize [◄(1) 2] + err := forks.AddCertifiedBlock(blocksAddedFirst[0]) + require.Nil(t, err) + err = forks.AddCertifiedBlock(blocksAddedFirst[1]) + require.Nil(t, err) + + // should be able to retrieve all stored blocks + for _, block := range blocksAddedFirst { + b, ok := forks.GetBlock(block.Block.BlockID) + assert.True(t, ok) + assert.Equal(t, block.Block, b) + } - // add second blocks - should finalize [2,3] and prune [1,2] - err = addBlocksToForks(forks, blocksAddedSecond) - require.Nil(t, err) + // add remaining block [◄(4) 5] - should finalize [◄(2) 3] and prune [◄(1) 2] + require.Nil(t, forks.AddCertifiedBlock(remainingBlock)) - // should be able to retrieve just added block - got, ok := forks.GetProposal(blocksAddedSecond[0].Block.BlockID) - assert.True(t, ok) - assert.Equal(t, blocksAddedSecond[0], got) + // should be able to retrieve just added block + b, ok := forks.GetBlock(remainingBlock.Block.BlockID) + assert.True(t, ok) + assert.Equal(t, remainingBlock.Block, b) - // should be unable to retrieve pruned block - _, ok = forks.GetProposal(blocksAddedFirst[0].Block.BlockID) - assert.False(t, ok) + // should be unable to retrieve pruned block + _, ok = forks.GetBlock(blocksAddedFirst[0].Block.BlockID) + assert.False(t, ok) + }) } -// TestGetProposalsForView tests retrieving proposals for a view. -// receives [1,2] [2,4] [2,4'] -func TestGetProposalsForView(t *testing.T) { - - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 4) - builder.AddVersioned(2, 4, 0, 1) +// TestGetBlocksForView tests retrieving blocks for a view (also including double proposals). +// - Forks receives [◄(1) 2] [◄(2) 4] [◄(2) 4'], +// where [◄(2) 4'] is a double proposal, because it has the same view as [◄(2) 4] +// +// Expected behaviour: +// - Forks should store all the blocks +// - Forks should emit a `OnDoubleProposeDetected` notification +// - we can retrieve all blocks, including the double proposals +func TestGetBlocksForView(t *testing.T) { + blocks, err := NewBlockBuilder(). + Add(1, 2). // [◄(1) 2] + Add(2, 4). // [◄(2) 4] + AddVersioned(2, 4, 0, 1). // [◄(2) 4'] + Blocks() + require.Nil(t, err) + + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", blocks[2], blocks[1]).Once() + + err = addValidatedBlockToForks(forks, blocks) + require.Nil(t, err) + + // expect 1 block at view 2 + storedBlocks := forks.GetBlocksForView(2) + assert.Len(t, storedBlocks, 1) + assert.Equal(t, blocks[0], storedBlocks[0]) + + // expect 2 blocks at view 4 + storedBlocks = forks.GetBlocksForView(4) + assert.Len(t, storedBlocks, 2) + assert.ElementsMatch(t, blocks[1:], storedBlocks) + + // expect 0 blocks at view 3 + storedBlocks = forks.GetBlocksForView(3) + assert.Len(t, storedBlocks, 0) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", blocks[2], blocks[1]).Once() + + err := forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[0])) + require.Nil(t, err) + err = forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[1])) + require.Nil(t, err) + err = forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[2])) + require.Nil(t, err) + + // expect 1 block at view 2 + storedBlocks := forks.GetBlocksForView(2) + assert.Len(t, storedBlocks, 1) + assert.Equal(t, blocks[0], storedBlocks[0]) + + // expect 2 blocks at view 4 + storedBlocks = forks.GetBlocksForView(4) + assert.Len(t, storedBlocks, 2) + assert.ElementsMatch(t, blocks[1:], storedBlocks) + + // expect 0 blocks at view 3 + storedBlocks = forks.GetBlocksForView(3) + assert.Len(t, storedBlocks, 0) + }) +} +// TestNotifications tests that Forks emits the expected events: +// - Forks receives [◄(1) 2] [◄(2) 3] [◄(3) 4] +// +// Expected Behaviour: +// - Each of the ingested blocks should result in an `OnBlockIncorporated` notification +// - Forks should finalize [◄(1) 2], resulting in a `MakeFinal` event and an `OnFinalizedBlock` event +func TestNotifications(t *testing.T) { + builder := NewBlockBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 4) blocks, err := builder.Blocks() require.Nil(t, err) - forks, notifier := newForks(t) - notifier.On("OnDoubleProposeDetected", blocks[2].Block, blocks[1].Block).Once() + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + notifier := &mocks.Consumer{} + // 4 blocks including the genesis are incorporated + notifier.On("OnBlockIncorporated", mock.Anything).Return(nil).Times(4) + notifier.On("OnFinalizedBlock", blocks[0]).Once() + finalizationCallback := mockmodule.NewFinalizer(t) + finalizationCallback.On("MakeFinal", blocks[0].BlockID).Return(nil).Once() + + forks, err := New(builder.GenesisBlock(), finalizationCallback, notifier) + require.NoError(t, err) + require.NoError(t, addValidatedBlockToForks(forks, blocks)) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + notifier := &mocks.Consumer{} + // 4 blocks including the genesis are incorporated + notifier.On("OnBlockIncorporated", mock.Anything).Return(nil).Times(4) + notifier.On("OnFinalizedBlock", blocks[0]).Once() + finalizationCallback := mockmodule.NewFinalizer(t) + finalizationCallback.On("MakeFinal", blocks[0].BlockID).Return(nil).Once() + + forks, err := New(builder.GenesisBlock(), finalizationCallback, notifier) + require.NoError(t, err) + require.NoError(t, addCertifiedBlocksToForks(forks, blocks)) + }) +} - err = addBlocksToForks(forks, blocks) +// TestFinalizingMultipleBlocks tests that `OnFinalizedBlock` notifications are emitted in correct order +// when there are multiple blocks finalized by adding a _single_ block. +// - receiving [◄(1) 3] [◄(3) 5] [◄(5) 7] [◄(7) 11] [◄(11) 12] should not finalize any blocks, +// because there is no 2-chain with the first chain link being a _direct_ 1-chain +// - adding [◄(12) 22] should finalize up to block [◄(6) 11] +// +// This test verifies the following expected properties: +// 1. Safety under reentrancy: +// While Forks is single-threaded, there is still the possibility of reentrancy. Specifically, the +// consumers of our finalization events are served by the goroutine executing Forks. It is conceivable +// that a consumer might access Forks and query the latest finalization proof. This would be legal, if +// the component supplying the goroutine to Forks also consumes the notifications. Therefore, for API +// safety, we require forks to _first update_ its `FinalityProof()` before it emits _any_ events. +// 2. For each finalized block, `finalizationCallback` event is executed _before_ `OnFinalizedBlock` notifications. +// 3. Blocks are finalized in order of increasing height (without skipping any blocks). +func TestFinalizingMultipleBlocks(t *testing.T) { + builder := NewBlockBuilder(). + Add(1, 3). // index 0: [◄(1) 2] + Add(3, 5). // index 1: [◄(2) 4] + Add(5, 7). // index 2: [◄(4) 6] + Add(7, 11). // index 3: [◄(6) 11] -- expected to be finalized + Add(11, 12). // index 4: [◄(11) 12] + Add(12, 22) // index 5: [◄(12) 22] + blocks, err := builder.Blocks() require.Nil(t, err) - // 1 proposal at view 2 - proposals := forks.GetProposalsForView(2) - assert.Len(t, proposals, 1) - assert.Equal(t, blocks[0], proposals[0]) + // The Finality Proof should right away point to the _latest_ finalized block. Subsequently emitting + // Finalization events for lower blocks is fine, because notifications are guaranteed to be + // _eventually_ arriving. I.e. consumers expect notifications / events to be potentially lagging behind. + expectedFinalityProof := makeFinalityProof(t, blocks[3], blocks[4], blocks[5].QC) - // 2 proposals at view 4 - proposals = forks.GetProposalsForView(4) - assert.Len(t, proposals, 2) - assert.ElementsMatch(t, blocks[1:], proposals) + setupForksAndAssertions := func() (*Forks, *mockmodule.Finalizer, *mocks.Consumer) { + // initialize Forks with custom event consumers so we can check order of emitted events + notifier := &mocks.Consumer{} + finalizationCallback := mockmodule.NewFinalizer(t) + notifier.On("OnBlockIncorporated", mock.Anything).Return(nil) + forks, err := New(builder.GenesisBlock(), finalizationCallback, notifier) + require.NoError(t, err) - // 0 proposals at view 3 - proposals = forks.GetProposalsForView(3) - assert.Len(t, proposals, 0) -} - -// TestNotification tests that notifier gets correct notifications when incorporating block as well as finalization events. -// receives [1,2] [2,3] [3,4] -// should finalize [1,2] -func TestNotification(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - - blocks, err := builder.Blocks() - require.Nil(t, err) + // expecting finalization of [◄(1) 2] [◄(2) 4] [◄(4) 6] [◄(6) 11] in this order + blocksAwaitingFinalization := toBlockAwaitingFinalization(blocks[:4]) - notifier := &mocks.Consumer{} - // 4 blocks including the genesis are incorporated - notifier.On("OnBlockIncorporated", mock.Anything).Return(nil).Times(4) - notifier.On("OnFinalizedBlock", blocks[0].Block).Return(nil).Once() - finalizationCallback := mockmodule.NewFinalizer(t) - finalizationCallback.On("MakeFinal", blocks[0].Block.BlockID).Return(nil).Once() + finalizationCallback.On("MakeFinal", mock.Anything).Run(func(args mock.Arguments) { + requireFinalityProof(t, forks, expectedFinalityProof) // Requirement 1: forks should _first update_ its `FinalityProof()` before it emits _any_ events - forks, err := forks.New(builder.GenesisBlock(), finalizationCallback, notifier) - require.NoError(t, err) + // Requirement 3: finalized in order of increasing height (without skipping any blocks). + expectedNextFinalizationEvents := blocksAwaitingFinalization[0] + require.Equal(t, expectedNextFinalizationEvents.Block.BlockID, args[0]) - err = addBlocksToForks(forks, blocks) - require.NoError(t, err) -} + // Requirement 2: finalized block, `finalizationCallback` event is executed _before_ `OnFinalizedBlock` notifications. + // no duplication of events under normal operations expected + require.False(t, expectedNextFinalizationEvents.MakeFinalCalled) + require.False(t, expectedNextFinalizationEvents.OnFinalizedBlockEmitted) + expectedNextFinalizationEvents.MakeFinalCalled = true + }).Return(nil).Times(4) -// TestNewestView tests that Forks tracks the newest block view seen in received blocks. -// receives [1,2] [2,3] [3,4] -func TestNewestView(t *testing.T) { - builder := forks.NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) + notifier.On("OnFinalizedBlock", mock.Anything).Run(func(args mock.Arguments) { + requireFinalityProof(t, forks, expectedFinalityProof) // Requirement 1: forks should _first update_ its `FinalityProof()` before it emits _any_ events - blocks, err := builder.Blocks() - require.Nil(t, err) + // Requirement 3: finalized in order of increasing height (without skipping any blocks). + expectedNextFinalizationEvents := blocksAwaitingFinalization[0] + require.Equal(t, expectedNextFinalizationEvents.Block, args[0]) - forks, _ := newForks(t) + // Requirement 2: finalized block, `finalizationCallback` event is executed _before_ `OnFinalizedBlock` notifications. + // no duplication of events under normal operations expected + require.True(t, expectedNextFinalizationEvents.MakeFinalCalled) + require.False(t, expectedNextFinalizationEvents.OnFinalizedBlockEmitted) + expectedNextFinalizationEvents.OnFinalizedBlockEmitted = true - genesis := builder.GenesisBlock() + // At this point, `MakeFinal` and `OnFinalizedBlock` have both been emitted for the block, so we are done with it + blocksAwaitingFinalization = blocksAwaitingFinalization[1:] + }).Times(4) - // initially newest view should be genesis block view - require.Equal(t, forks.NewestView(), genesis.Block.View) + return forks, finalizationCallback, notifier + } - err = addBlocksToForks(forks, blocks) - require.NoError(t, err) - // after inserting new blocks, newest view should be greatest view of all added blocks - require.Equal(t, forks.NewestView(), uint64(4)) + t.Run("consensus participant mode: ingest validated blocks", func(t *testing.T) { + forks, finalizationCallback, notifier := setupForksAndAssertions() + err = addValidatedBlockToForks(forks, blocks[:5]) // adding [◄(1) 2] [◄(2) 4] [◄(4) 6] [◄(6) 11] [◄(11) 12] + require.Nil(t, err) + requireOnlyGenesisBlockFinalized(t, forks) // finalization should still be at the genesis block + + require.NoError(t, forks.AddValidatedBlock(blocks[5])) // adding [◄(12) 22] should trigger finalization events + requireFinalityProof(t, forks, expectedFinalityProof) + finalizationCallback.AssertExpectations(t) + notifier.AssertExpectations(t) + }) + + t.Run("consensus follower mode: ingest certified blocks", func(t *testing.T) { + forks, finalizationCallback, notifier := setupForksAndAssertions() + // adding [◄(1) 2] [◄(2) 4] [◄(4) 6] [◄(6) 11] ◄(11) + require.NoError(t, forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[0]))) + require.NoError(t, forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[1]))) + require.NoError(t, forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[2]))) + require.NoError(t, forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[3]))) + require.Nil(t, err) + requireOnlyGenesisBlockFinalized(t, forks) // finalization should still be at the genesis block + + // adding certified block [◄(11) 12] ◄(12) should trigger finalization events + require.NoError(t, forks.AddCertifiedBlock(toCertifiedBlock(t, blocks[4]))) + requireFinalityProof(t, forks, expectedFinalityProof) + finalizationCallback.AssertExpectations(t) + notifier.AssertExpectations(t) + }) } -// ========== internal functions =============== +//* ************************************* internal functions ************************************* */ -func newForks(t *testing.T) (*forks.Forks, *mocks.Consumer) { +func newForks(t *testing.T) (*Forks, *mocks.Consumer) { notifier := mocks.NewConsumer(t) notifier.On("OnBlockIncorporated", mock.Anything).Return(nil).Maybe() - notifier.On("OnFinalizedBlock", mock.Anything).Return(nil).Maybe() + notifier.On("OnFinalizedBlock", mock.Anything).Maybe() finalizationCallback := mockmodule.NewFinalizer(t) finalizationCallback.On("MakeFinal", mock.Anything).Return(nil).Maybe() - genesisBQ := forks.NewBlockBuilder().GenesisBlock() + genesisBQ := makeGenesis() - forks, err := forks.New(genesisBQ, finalizationCallback, notifier) + forks, err := New(genesisBQ, finalizationCallback, notifier) require.Nil(t, err) return forks, notifier } -// addBlocksToForks adds all the given blocks to Forks, in order. +// addValidatedBlockToForks adds all the given blocks to Forks, in order. // If any errors occur, returns the first one. -func addBlocksToForks(forks *forks.Forks, proposals []*model.Proposal) error { - for _, proposal := range proposals { - err := forks.AddProposal(proposal) +func addValidatedBlockToForks(forks *Forks, blocks []*model.Block) error { + for _, block := range blocks { + err := forks.AddValidatedBlock(block) if err != nil { - return fmt.Errorf("test case failed at adding proposal: %v: %w", proposal.Block.View, err) + return fmt.Errorf("test failed to add block for view %d: %w", block.View, err) + } + } + return nil +} + +// addCertifiedBlocksToForks iterates over all blocks, caches them locally in a map, +// constructs certified blocks whenever possible and adds the certified blocks to forks, +// Note: if blocks is a single fork, the _last block_ in the slice will not be added, +// +// because there is no qc for it +// +// If any errors occur, returns the first one. +func addCertifiedBlocksToForks(forks *Forks, blocks []*model.Block) error { + uncertifiedBlocks := make(map[flow.Identifier]*model.Block) + for _, b := range blocks { + uncertifiedBlocks[b.BlockID] = b + parentID := b.QC.BlockID + parent, found := uncertifiedBlocks[parentID] + if !found { + continue + } + delete(uncertifiedBlocks, parentID) + + certParent, err := model.NewCertifiedBlock(parent, b.QC) + if err != nil { + return fmt.Errorf("test failed to creat certified block for view %d: %w", certParent.Block.View, err) + } + err = forks.AddCertifiedBlock(&certParent) + if err != nil { + return fmt.Errorf("test failed to add certified block for view %d: %w", certParent.Block.View, err) } } @@ -489,14 +879,73 @@ func addBlocksToForks(forks *forks.Forks, proposals []*model.Proposal) error { } // requireLatestFinalizedBlock asserts that the latest finalized block has the given view and qc view. -func requireLatestFinalizedBlock(t *testing.T, forks *forks.Forks, qcView int, view int) { - require.Equal(t, forks.FinalizedBlock().View, uint64(view), "finalized block has wrong view") - require.Equal(t, forks.FinalizedBlock().QC.View, uint64(qcView), "finalized block has wrong qc") +func requireLatestFinalizedBlock(t *testing.T, forks *Forks, expectedFinalized *model.Block) { + require.Equal(t, expectedFinalized, forks.FinalizedBlock(), "finalized block is not as expected") + require.Equal(t, forks.FinalizedView(), expectedFinalized.View, "FinalizedView returned wrong value") +} + +// requireOnlyGenesisBlockFinalized asserts that no blocks have been finalized beyond the genesis block. +// Caution: does not inspect output of `forks.FinalityProof()` +func requireOnlyGenesisBlockFinalized(t *testing.T, forks *Forks) { + genesis := makeGenesis() + require.Equal(t, forks.FinalizedBlock(), genesis.Block, "finalized block is not the genesis block") + require.Equal(t, forks.FinalizedBlock().View, genesis.Block.View) + require.Equal(t, forks.FinalizedBlock().View, genesis.CertifyingQC.View) + require.Equal(t, forks.FinalizedView(), genesis.Block.View, "finalized block has wrong qc") + + finalityProof, isKnown := forks.FinalityProof() + require.Nil(t, finalityProof, "expecting finality proof to be nil for genesis block at initialization") + require.False(t, isKnown, "no finality proof should be known for genesis block at initialization") } // requireNoBlocksFinalized asserts that no blocks have been finalized (genesis is latest finalized block). -func requireNoBlocksFinalized(t *testing.T, f *forks.Forks) { - genesis := forks.NewBlockBuilder().GenesisBlock() - require.Equal(t, f.FinalizedBlock().View, genesis.Block.View) - require.Equal(t, f.FinalizedBlock().View, genesis.CertifyingQC.View) +func requireFinalityProof(t *testing.T, forks *Forks, expectedFinalityProof *hotstuff.FinalityProof) { + finalityProof, isKnown := forks.FinalityProof() + require.True(t, isKnown) + require.Equal(t, expectedFinalityProof, finalityProof) + require.Equal(t, forks.FinalizedBlock(), expectedFinalityProof.Block) + require.Equal(t, forks.FinalizedView(), expectedFinalityProof.Block.View) +} + +// toCertifiedBlock generates a QC for the given block and returns their combination as a certified block +func toCertifiedBlock(t *testing.T, block *model.Block) *model.CertifiedBlock { + qc := &flow.QuorumCertificate{ + View: block.View, + BlockID: block.BlockID, + } + cb, err := model.NewCertifiedBlock(block, qc) + require.Nil(t, err) + return &cb +} + +// toCertifiedBlocks generates a QC for the given block and returns their combination as a certified blocks +func toCertifiedBlocks(t *testing.T, blocks ...*model.Block) []*model.CertifiedBlock { + certBlocks := make([]*model.CertifiedBlock, 0, len(blocks)) + for _, b := range blocks { + certBlocks = append(certBlocks, toCertifiedBlock(t, b)) + } + return certBlocks +} + +func makeFinalityProof(t *testing.T, block *model.Block, directChild *model.Block, qcCertifyingChild *flow.QuorumCertificate) *hotstuff.FinalityProof { + c, err := model.NewCertifiedBlock(directChild, qcCertifyingChild) // certified child of FinalizedBlock + require.NoError(t, err) + return &hotstuff.FinalityProof{Block: block, CertifiedChild: c} +} + +// blockAwaitingFinalization is intended for tracking finalization events and their order for a specific block +type blockAwaitingFinalization struct { + Block *model.Block + MakeFinalCalled bool // indicates whether `Finalizer.MakeFinal` was called + OnFinalizedBlockEmitted bool // indicates whether `OnFinalizedBlockCalled` notification was emitted +} + +// toBlockAwaitingFinalization creates a `blockAwaitingFinalization` tracker for each input block +func toBlockAwaitingFinalization(blocks []*model.Block) []*blockAwaitingFinalization { + trackers := make([]*blockAwaitingFinalization, 0, len(blocks)) + for _, b := range blocks { + tracker := &blockAwaitingFinalization{b, false, false} + trackers = append(trackers, tracker) + } + return trackers } diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index b6d3ae27ec9..bf12244c099 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -32,8 +32,8 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/voteaggregator" "github.com/onflow/flow-go/consensus/hotstuff/votecollector" "github.com/onflow/flow-go/crypto" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/counters" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" @@ -84,7 +84,8 @@ type Instance struct { } type MockedCommunicatorConsumer struct { - notifications.NoopPartialConsumer + notifications.NoopProposalViolationConsumer + notifications.NoopParticipantConsumer notifications.NoopFinalizationConsumer *mocks.CommunicatorConsumer } @@ -390,7 +391,7 @@ func NewInstance(t *testing.T, options ...Option) *Instance { // initialize the pacemaker controller := timeout.NewController(cfg.Timeouts) - in.pacemaker, err = pacemaker.New(controller, notifier, in.persist) + in.pacemaker, err = pacemaker.New(controller, pacemaker.NoProposalDelay(), notifier, in.persist) require.NoError(t, err) // initialize the forks handler @@ -431,7 +432,8 @@ func NewInstance(t *testing.T, options ...Option) *Instance { ) }, nil).Maybe() - createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) + voteAggregationDistributor := pubsub.NewVoteAggregationDistributor() + createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, voteAggregationDistributor, voteProcessorFactory.Create) voteCollectors := voteaggregator.NewVoteCollectors(log, livenessData.CurrentView, workerpool.New(2), createCollectorFactoryMethod) metricsCollector := metrics.NewNoopCollector() @@ -442,14 +444,14 @@ func NewInstance(t *testing.T, options ...Option) *Instance { metricsCollector, metricsCollector, metricsCollector, - notifier, + voteAggregationDistributor, livenessData.CurrentView, voteCollectors, ) require.NoError(t, err) // initialize factories for timeout collector and timeout processor - collectorDistributor := pubsub.NewTimeoutCollectorDistributor() + timeoutAggregationDistributor := pubsub.NewTimeoutAggregationDistributor() timeoutProcessorFactory := mocks.NewTimeoutProcessorFactory(t) timeoutProcessorFactory.On("Create", mock.Anything).Return( func(view uint64) hotstuff.TimeoutProcessor { @@ -490,18 +492,22 @@ func NewInstance(t *testing.T, options ...Option) *Instance { in.committee, in.validator, aggregator, - collectorDistributor, + timeoutAggregationDistributor, ) require.NoError(t, err) return p }, nil).Maybe() timeoutCollectorFactory := timeoutcollector.NewTimeoutCollectorFactory( unittest.Logger(), - notifier, - collectorDistributor, + timeoutAggregationDistributor, timeoutProcessorFactory, ) - timeoutCollectors := timeoutaggregator.NewTimeoutCollectors(log, livenessData.CurrentView, timeoutCollectorFactory) + timeoutCollectors := timeoutaggregator.NewTimeoutCollectors( + log, + metricsCollector, + livenessData.CurrentView, + timeoutCollectorFactory, + ) // initialize the timeout aggregator in.timeoutAggregator, err = timeoutaggregator.NewTimeoutAggregator( @@ -509,7 +515,6 @@ func NewInstance(t *testing.T, options ...Option) *Instance { metricsCollector, metricsCollector, metricsCollector, - notifier, livenessData.CurrentView, timeoutCollectors, ) @@ -538,8 +543,10 @@ func NewInstance(t *testing.T, options ...Option) *Instance { ) require.NoError(t, err) - collectorDistributor.AddConsumer(logConsumer) - collectorDistributor.AddConsumer(&in) + timeoutAggregationDistributor.AddTimeoutCollectorConsumer(logConsumer) + timeoutAggregationDistributor.AddTimeoutCollectorConsumer(&in) + + voteAggregationDistributor.AddVoteCollectorConsumer(logConsumer) return &in } @@ -664,3 +671,5 @@ func (in *Instance) OnNewQcDiscovered(qc *flow.QuorumCertificate) { func (in *Instance) OnNewTcDiscovered(tc *flow.TimeoutCertificate) { in.queue <- tc } + +func (in *Instance) OnTimeoutProcessed(*model.TimeoutObject) {} diff --git a/consensus/hotstuff/integration/integration_test.go b/consensus/hotstuff/integration/integration_test.go index e2929777dee..e4f2e588ba9 100644 --- a/consensus/hotstuff/integration/integration_test.go +++ b/consensus/hotstuff/integration/integration_test.go @@ -52,7 +52,7 @@ func TestThreeInstances(t *testing.T) { // generate three hotstuff participants participants := unittest.IdentityListFixture(num) root := DefaultRoot() - timeouts, err := timeout.NewConfig(safeTimeout, safeTimeout, 1.5, happyPathMaxRoundFailures, 0, safeTimeout) + timeouts, err := timeout.NewConfig(safeTimeout, safeTimeout, 1.5, happyPathMaxRoundFailures, safeTimeout) require.NoError(t, err) // set up three instances that are exactly the same @@ -116,7 +116,7 @@ func TestSevenInstances(t *testing.T) { participants := unittest.IdentityListFixture(numPass + numFail) instances := make([]*Instance, 0, numPass+numFail) root := DefaultRoot() - timeouts, err := timeout.NewConfig(safeTimeout, safeTimeout, 1.5, happyPathMaxRoundFailures, 0, safeTimeout) + timeouts, err := timeout.NewConfig(safeTimeout, safeTimeout, 1.5, happyPathMaxRoundFailures, safeTimeout) require.NoError(t, err) // set up five instances that work fully diff --git a/consensus/hotstuff/integration/liveness_test.go b/consensus/hotstuff/integration/liveness_test.go index b9eca3cf005..109bf3b967f 100644 --- a/consensus/hotstuff/integration/liveness_test.go +++ b/consensus/hotstuff/integration/liveness_test.go @@ -36,7 +36,7 @@ func Test2TimeoutOutof7Instances(t *testing.T) { participants := unittest.IdentityListFixture(healthyReplicas + notVotingReplicas) instances := make([]*Instance, 0, healthyReplicas+notVotingReplicas) root := DefaultRoot() - timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, happyPathMaxRoundFailures, 0, maxTimeoutRebroadcast) + timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, happyPathMaxRoundFailures, maxTimeoutRebroadcast) require.NoError(t, err) // set up five instances that work fully @@ -103,8 +103,7 @@ func Test2TimeoutOutof4Instances(t *testing.T) { participants := unittest.IdentityListFixture(healthyReplicas + replicasDroppingHappyPathMsgs) instances := make([]*Instance, 0, healthyReplicas+replicasDroppingHappyPathMsgs) root := DefaultRoot() - timeouts, err := timeout.NewConfig( - 10*time.Millisecond, 50*time.Millisecond, 1.5, happyPathMaxRoundFailures, 0, maxTimeoutRebroadcast) + timeouts, err := timeout.NewConfig(10*time.Millisecond, 50*time.Millisecond, 1.5, happyPathMaxRoundFailures, maxTimeoutRebroadcast) require.NoError(t, err) // set up two instances that work fully @@ -173,7 +172,7 @@ func Test1TimeoutOutof5Instances(t *testing.T) { participants := unittest.IdentityListFixture(healthyReplicas + blockedReplicas) instances := make([]*Instance, 0, healthyReplicas+blockedReplicas) root := DefaultRoot() - timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, happyPathMaxRoundFailures, 0, maxTimeoutRebroadcast) + timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, happyPathMaxRoundFailures, maxTimeoutRebroadcast) require.NoError(t, err) // set up instances that work fully @@ -220,12 +219,11 @@ func Test1TimeoutOutof5Instances(t *testing.T) { t.Logf("dumping state of system:") for i, inst := range instances { t.Logf( - "instance %d: %d %d %d %d", + "instance %d: %d %d %d", i, inst.pacemaker.CurView(), inst.pacemaker.NewestQC().View, inst.forks.FinalizedBlock().View, - inst.forks.NewestView(), ) } } @@ -271,7 +269,7 @@ func TestBlockDelayIsHigherThanTimeout(t *testing.T) { instances := make([]*Instance, 0, healthyReplicas+replicasNotGeneratingTimeouts) root := DefaultRoot() // set block rate delay to be bigger than minimal timeout - timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, happyPathMaxRoundFailures, pmTimeout*2, maxTimeoutRebroadcast) + timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, happyPathMaxRoundFailures, maxTimeoutRebroadcast) require.NoError(t, err) // set up 2 instances that fully work (incl. sending TimeoutObjects) @@ -354,7 +352,7 @@ func TestAsyncClusterStartup(t *testing.T) { instances := make([]*Instance, 0, replicas) root := DefaultRoot() // set block rate delay to be bigger than minimal timeout - timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, 6, 0, maxTimeoutRebroadcast) + timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, 6, maxTimeoutRebroadcast) require.NoError(t, err) // set up instances that work fully diff --git a/consensus/hotstuff/mocks/block_signer.go b/consensus/hotstuff/mocks/block_signer.go deleted file mode 100644 index 16abe4ceb61..00000000000 --- a/consensus/hotstuff/mocks/block_signer.go +++ /dev/null @@ -1,51 +0,0 @@ -// Code generated by mockery v2.13.1. DO NOT EDIT. - -package mocks - -import ( - model "github.com/onflow/flow-go/consensus/hotstuff/model" - mock "github.com/stretchr/testify/mock" -) - -// BlockSigner is an autogenerated mock type for the BlockSigner type -type BlockSigner struct { - mock.Mock -} - -// CreateVote provides a mock function with given fields: _a0 -func (_m *BlockSigner) CreateVote(_a0 *model.Block) (*model.Vote, error) { - ret := _m.Called(_a0) - - var r0 *model.Vote - if rf, ok := ret.Get(0).(func(*model.Block) *model.Vote); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.Vote) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(*model.Block) error); ok { - r1 = rf(_a0) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type mockConstructorTestingTNewBlockSigner interface { - mock.TestingT - Cleanup(func()) -} - -// NewBlockSigner creates a new instance of BlockSigner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewBlockSigner(t mockConstructorTestingTNewBlockSigner) *BlockSigner { - mock := &BlockSigner{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/mocks/committee.go b/consensus/hotstuff/mocks/committee.go deleted file mode 100644 index 69385de999f..00000000000 --- a/consensus/hotstuff/mocks/committee.go +++ /dev/null @@ -1,138 +0,0 @@ -// Code generated by mockery v2.13.1. DO NOT EDIT. - -package mocks - -import ( - hotstuff "github.com/onflow/flow-go/consensus/hotstuff" - flow "github.com/onflow/flow-go/model/flow" - - mock "github.com/stretchr/testify/mock" -) - -// Committee is an autogenerated mock type for the Committee type -type Committee struct { - mock.Mock -} - -// DKG provides a mock function with given fields: blockID -func (_m *Committee) DKG(blockID flow.Identifier) (hotstuff.DKG, error) { - ret := _m.Called(blockID) - - var r0 hotstuff.DKG - if rf, ok := ret.Get(0).(func(flow.Identifier) hotstuff.DKG); ok { - r0 = rf(blockID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(hotstuff.DKG) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { - r1 = rf(blockID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Identities provides a mock function with given fields: blockID -func (_m *Committee) Identities(blockID flow.Identifier) (flow.IdentityList, error) { - ret := _m.Called(blockID) - - var r0 flow.IdentityList - if rf, ok := ret.Get(0).(func(flow.Identifier) flow.IdentityList); ok { - r0 = rf(blockID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.IdentityList) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { - r1 = rf(blockID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Identity provides a mock function with given fields: blockID, participantID -func (_m *Committee) Identity(blockID flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) { - ret := _m.Called(blockID, participantID) - - var r0 *flow.Identity - if rf, ok := ret.Get(0).(func(flow.Identifier, flow.Identifier) *flow.Identity); ok { - r0 = rf(blockID, participantID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.Identity) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(flow.Identifier, flow.Identifier) error); ok { - r1 = rf(blockID, participantID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// LeaderForView provides a mock function with given fields: view -func (_m *Committee) LeaderForView(view uint64) (flow.Identifier, error) { - ret := _m.Called(view) - - var r0 flow.Identifier - if rf, ok := ret.Get(0).(func(uint64) flow.Identifier); ok { - r0 = rf(view) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.Identifier) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(uint64) error); ok { - r1 = rf(view) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Self provides a mock function with given fields: -func (_m *Committee) Self() flow.Identifier { - ret := _m.Called() - - var r0 flow.Identifier - if rf, ok := ret.Get(0).(func() flow.Identifier); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.Identifier) - } - } - - return r0 -} - -type mockConstructorTestingTNewCommittee interface { - mock.TestingT - Cleanup(func()) -} - -// NewCommittee creates a new instance of Committee. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewCommittee(t mockConstructorTestingTNewCommittee) *Committee { - mock := &Committee{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/mocks/consumer.go b/consensus/hotstuff/mocks/consumer.go index ee991cee08e..cdd1ebe72cd 100644 --- a/consensus/hotstuff/mocks/consumer.go +++ b/consensus/hotstuff/mocks/consumer.go @@ -33,16 +33,6 @@ func (_m *Consumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) _m.Called(_a0, _a1) } -// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 -func (_m *Consumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { - _m.Called(_a0, _a1) -} - -// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 -func (_m *Consumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { - _m.Called(_a0, _a1) -} - // OnEventProcessed provides a mock function with given fields: func (_m *Consumer) OnEventProcessed() { _m.Called() @@ -53,13 +43,8 @@ func (_m *Consumer) OnFinalizedBlock(_a0 *model.Block) { _m.Called(_a0) } -// OnInvalidTimeoutDetected provides a mock function with given fields: err -func (_m *Consumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { - _m.Called(err) -} - -// OnInvalidVoteDetected provides a mock function with given fields: err -func (_m *Consumer) OnInvalidVoteDetected(err model.InvalidVoteError) { +// OnInvalidBlockDetected provides a mock function with given fields: err +func (_m *Consumer) OnInvalidBlockDetected(err flow.Slashable[model.InvalidProposalError]) { _m.Called(err) } @@ -123,26 +108,11 @@ func (_m *Consumer) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc * _m.Called(oldView, newView, tc) } -// OnTimeoutProcessed provides a mock function with given fields: timeout -func (_m *Consumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { - _m.Called(timeout) -} - // OnViewChange provides a mock function with given fields: oldView, newView func (_m *Consumer) OnViewChange(oldView uint64, newView uint64) { _m.Called(oldView, newView) } -// OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal -func (_m *Consumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { - _m.Called(vote, invalidProposal) -} - -// OnVoteProcessed provides a mock function with given fields: vote -func (_m *Consumer) OnVoteProcessed(vote *model.Vote) { - _m.Called(vote) -} - type mockConstructorTestingTNewConsumer interface { mock.TestingT Cleanup(func()) diff --git a/consensus/hotstuff/mocks/event_loop.go b/consensus/hotstuff/mocks/event_loop.go index 3a15f4a4331..a1425da0629 100644 --- a/consensus/hotstuff/mocks/event_loop.go +++ b/consensus/hotstuff/mocks/event_loop.go @@ -58,6 +58,16 @@ func (_m *EventLoop) OnTcConstructedFromTimeouts(certificate *flow.TimeoutCertif _m.Called(certificate) } +// OnTimeoutProcessed provides a mock function with given fields: timeout +func (_m *EventLoop) OnTimeoutProcessed(timeout *model.TimeoutObject) { + _m.Called(timeout) +} + +// OnVoteProcessed provides a mock function with given fields: vote +func (_m *EventLoop) OnVoteProcessed(vote *model.Vote) { + _m.Called(vote) +} + // Ready provides a mock function with given fields: func (_m *EventLoop) Ready() <-chan struct{} { ret := _m.Called() diff --git a/consensus/hotstuff/mocks/finalization_consumer.go b/consensus/hotstuff/mocks/finalization_consumer.go index 5c5a5f4b922..7780a5e1c79 100644 --- a/consensus/hotstuff/mocks/finalization_consumer.go +++ b/consensus/hotstuff/mocks/finalization_consumer.go @@ -17,11 +17,6 @@ func (_m *FinalizationConsumer) OnBlockIncorporated(_a0 *model.Block) { _m.Called(_a0) } -// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 -func (_m *FinalizationConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { - _m.Called(_a0, _a1) -} - // OnFinalizedBlock provides a mock function with given fields: _a0 func (_m *FinalizationConsumer) OnFinalizedBlock(_a0 *model.Block) { _m.Called(_a0) diff --git a/consensus/hotstuff/mocks/follower_consumer.go b/consensus/hotstuff/mocks/follower_consumer.go new file mode 100644 index 00000000000..4906eefacb7 --- /dev/null +++ b/consensus/hotstuff/mocks/follower_consumer.go @@ -0,0 +1,51 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// FollowerConsumer is an autogenerated mock type for the FollowerConsumer type +type FollowerConsumer struct { + mock.Mock +} + +// OnBlockIncorporated provides a mock function with given fields: _a0 +func (_m *FollowerConsumer) OnBlockIncorporated(_a0 *model.Block) { + _m.Called(_a0) +} + +// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 +func (_m *FollowerConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { + _m.Called(_a0, _a1) +} + +// OnFinalizedBlock provides a mock function with given fields: _a0 +func (_m *FollowerConsumer) OnFinalizedBlock(_a0 *model.Block) { + _m.Called(_a0) +} + +// OnInvalidBlockDetected provides a mock function with given fields: err +func (_m *FollowerConsumer) OnInvalidBlockDetected(err flow.Slashable[model.InvalidProposalError]) { + _m.Called(err) +} + +type mockConstructorTestingTNewFollowerConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewFollowerConsumer creates a new instance of FollowerConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewFollowerConsumer(t mockConstructorTestingTNewFollowerConsumer) *FollowerConsumer { + mock := &FollowerConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/follower_logic.go b/consensus/hotstuff/mocks/follower_logic.go deleted file mode 100644 index 9b978ea5b27..00000000000 --- a/consensus/hotstuff/mocks/follower_logic.go +++ /dev/null @@ -1,58 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mocks - -import ( - model "github.com/onflow/flow-go/consensus/hotstuff/model" - mock "github.com/stretchr/testify/mock" -) - -// FollowerLogic is an autogenerated mock type for the FollowerLogic type -type FollowerLogic struct { - mock.Mock -} - -// AddBlock provides a mock function with given fields: proposal -func (_m *FollowerLogic) AddBlock(proposal *model.Proposal) error { - ret := _m.Called(proposal) - - var r0 error - if rf, ok := ret.Get(0).(func(*model.Proposal) error); ok { - r0 = rf(proposal) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// FinalizedBlock provides a mock function with given fields: -func (_m *FollowerLogic) FinalizedBlock() *model.Block { - ret := _m.Called() - - var r0 *model.Block - if rf, ok := ret.Get(0).(func() *model.Block); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.Block) - } - } - - return r0 -} - -type mockConstructorTestingTNewFollowerLogic interface { - mock.TestingT - Cleanup(func()) -} - -// NewFollowerLogic creates a new instance of FollowerLogic. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewFollowerLogic(t mockConstructorTestingTNewFollowerLogic) *FollowerLogic { - mock := &FollowerLogic{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/mocks/forks.go b/consensus/hotstuff/mocks/forks.go index 063b7b9f551..c14ece84bc5 100644 --- a/consensus/hotstuff/mocks/forks.go +++ b/consensus/hotstuff/mocks/forks.go @@ -3,6 +3,7 @@ package mocks import ( + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" @@ -15,12 +16,26 @@ type Forks struct { mock.Mock } -// AddProposal provides a mock function with given fields: proposal -func (_m *Forks) AddProposal(proposal *model.Proposal) error { +// AddCertifiedBlock provides a mock function with given fields: certifiedBlock +func (_m *Forks) AddCertifiedBlock(certifiedBlock *model.CertifiedBlock) error { + ret := _m.Called(certifiedBlock) + + var r0 error + if rf, ok := ret.Get(0).(func(*model.CertifiedBlock) error); ok { + r0 = rf(certifiedBlock) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// AddValidatedBlock provides a mock function with given fields: proposal +func (_m *Forks) AddValidatedBlock(proposal *model.Block) error { ret := _m.Called(proposal) var r0 error - if rf, ok := ret.Get(0).(func(*model.Proposal) error); ok { + if rf, ok := ret.Get(0).(func(*model.Block) error); ok { r0 = rf(proposal) } else { r0 = ret.Error(0) @@ -29,6 +44,32 @@ func (_m *Forks) AddProposal(proposal *model.Proposal) error { return r0 } +// FinalityProof provides a mock function with given fields: +func (_m *Forks) FinalityProof() (*hotstuff.FinalityProof, bool) { + ret := _m.Called() + + var r0 *hotstuff.FinalityProof + var r1 bool + if rf, ok := ret.Get(0).(func() (*hotstuff.FinalityProof, bool)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *hotstuff.FinalityProof); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*hotstuff.FinalityProof) + } + } + + if rf, ok := ret.Get(1).(func() bool); ok { + r1 = rf() + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + // FinalizedBlock provides a mock function with given fields: func (_m *Forks) FinalizedBlock() *model.Block { ret := _m.Called() @@ -59,25 +100,25 @@ func (_m *Forks) FinalizedView() uint64 { return r0 } -// GetProposal provides a mock function with given fields: id -func (_m *Forks) GetProposal(id flow.Identifier) (*model.Proposal, bool) { - ret := _m.Called(id) +// GetBlock provides a mock function with given fields: blockID +func (_m *Forks) GetBlock(blockID flow.Identifier) (*model.Block, bool) { + ret := _m.Called(blockID) - var r0 *model.Proposal + var r0 *model.Block var r1 bool - if rf, ok := ret.Get(0).(func(flow.Identifier) (*model.Proposal, bool)); ok { - return rf(id) + if rf, ok := ret.Get(0).(func(flow.Identifier) (*model.Block, bool)); ok { + return rf(blockID) } - if rf, ok := ret.Get(0).(func(flow.Identifier) *model.Proposal); ok { - r0 = rf(id) + if rf, ok := ret.Get(0).(func(flow.Identifier) *model.Block); ok { + r0 = rf(blockID) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.Proposal) + r0 = ret.Get(0).(*model.Block) } } if rf, ok := ret.Get(1).(func(flow.Identifier) bool); ok { - r1 = rf(id) + r1 = rf(blockID) } else { r1 = ret.Get(1).(bool) } @@ -85,36 +126,22 @@ func (_m *Forks) GetProposal(id flow.Identifier) (*model.Proposal, bool) { return r0, r1 } -// GetProposalsForView provides a mock function with given fields: view -func (_m *Forks) GetProposalsForView(view uint64) []*model.Proposal { +// GetBlocksForView provides a mock function with given fields: view +func (_m *Forks) GetBlocksForView(view uint64) []*model.Block { ret := _m.Called(view) - var r0 []*model.Proposal - if rf, ok := ret.Get(0).(func(uint64) []*model.Proposal); ok { + var r0 []*model.Block + if rf, ok := ret.Get(0).(func(uint64) []*model.Block); ok { r0 = rf(view) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]*model.Proposal) + r0 = ret.Get(0).([]*model.Block) } } return r0 } -// NewestView provides a mock function with given fields: -func (_m *Forks) NewestView() uint64 { - ret := _m.Called() - - var r0 uint64 - if rf, ok := ret.Get(0).(func() uint64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(uint64) - } - - return r0 -} - type mockConstructorTestingTNewForks interface { mock.TestingT Cleanup(func()) diff --git a/consensus/hotstuff/mocks/forks_reader.go b/consensus/hotstuff/mocks/forks_reader.go deleted file mode 100644 index b9ba2848a33..00000000000 --- a/consensus/hotstuff/mocks/forks_reader.go +++ /dev/null @@ -1,114 +0,0 @@ -// Code generated by mockery v2.13.1. DO NOT EDIT. - -package mocks - -import ( - flow "github.com/onflow/flow-go/model/flow" - - mock "github.com/stretchr/testify/mock" - - model "github.com/onflow/flow-go/consensus/hotstuff/model" -) - -// ForksReader is an autogenerated mock type for the ForksReader type -type ForksReader struct { - mock.Mock -} - -// FinalizedBlock provides a mock function with given fields: -func (_m *ForksReader) FinalizedBlock() *model.Block { - ret := _m.Called() - - var r0 *model.Block - if rf, ok := ret.Get(0).(func() *model.Block); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.Block) - } - } - - return r0 -} - -// FinalizedView provides a mock function with given fields: -func (_m *ForksReader) FinalizedView() uint64 { - ret := _m.Called() - - var r0 uint64 - if rf, ok := ret.Get(0).(func() uint64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(uint64) - } - - return r0 -} - -// GetBlock provides a mock function with given fields: id -func (_m *ForksReader) GetBlock(id flow.Identifier) (*model.Block, bool) { - ret := _m.Called(id) - - var r0 *model.Block - if rf, ok := ret.Get(0).(func(flow.Identifier) *model.Block); ok { - r0 = rf(id) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.Block) - } - } - - var r1 bool - if rf, ok := ret.Get(1).(func(flow.Identifier) bool); ok { - r1 = rf(id) - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 -} - -// GetBlocksForView provides a mock function with given fields: view -func (_m *ForksReader) GetBlocksForView(view uint64) []*model.Block { - ret := _m.Called(view) - - var r0 []*model.Block - if rf, ok := ret.Get(0).(func(uint64) []*model.Block); ok { - r0 = rf(view) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*model.Block) - } - } - - return r0 -} - -// IsSafeBlock provides a mock function with given fields: block -func (_m *ForksReader) IsSafeBlock(block *model.Block) bool { - ret := _m.Called(block) - - var r0 bool - if rf, ok := ret.Get(0).(func(*model.Block) bool); ok { - r0 = rf(block) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -type mockConstructorTestingTNewForksReader interface { - mock.TestingT - Cleanup(func()) -} - -// NewForksReader creates a new instance of ForksReader. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewForksReader(t mockConstructorTestingTNewForksReader) *ForksReader { - mock := &ForksReader{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/mocks/pace_maker.go b/consensus/hotstuff/mocks/pace_maker.go index 1ec28cf7d34..236726efc9f 100644 --- a/consensus/hotstuff/mocks/pace_maker.go +++ b/consensus/hotstuff/mocks/pace_maker.go @@ -19,20 +19,6 @@ type PaceMaker struct { mock.Mock } -// BlockRateDelay provides a mock function with given fields: -func (_m *PaceMaker) BlockRateDelay() time.Duration { - ret := _m.Called() - - var r0 time.Duration - if rf, ok := ret.Get(0).(func() time.Duration); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(time.Duration) - } - - return r0 -} - // CurView provides a mock function with given fields: func (_m *PaceMaker) CurView() uint64 { ret := _m.Called() @@ -136,6 +122,20 @@ func (_m *PaceMaker) Start(ctx context.Context) { _m.Called(ctx) } +// TargetPublicationTime provides a mock function with given fields: proposalView, timeViewEntered, parentBlockId +func (_m *PaceMaker) TargetPublicationTime(proposalView uint64, timeViewEntered time.Time, parentBlockId flow.Identifier) time.Time { + ret := _m.Called(proposalView, timeViewEntered, parentBlockId) + + var r0 time.Time + if rf, ok := ret.Get(0).(func(uint64, time.Time, flow.Identifier) time.Time); ok { + r0 = rf(proposalView, timeViewEntered, parentBlockId) + } else { + r0 = ret.Get(0).(time.Time) + } + + return r0 +} + // TimeoutChannel provides a mock function with given fields: func (_m *PaceMaker) TimeoutChannel() <-chan time.Time { ret := _m.Called() diff --git a/consensus/hotstuff/mocks/participant_consumer.go b/consensus/hotstuff/mocks/participant_consumer.go new file mode 100644 index 00000000000..2d2b4141093 --- /dev/null +++ b/consensus/hotstuff/mocks/participant_consumer.go @@ -0,0 +1,92 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// ParticipantConsumer is an autogenerated mock type for the ParticipantConsumer type +type ParticipantConsumer struct { + mock.Mock +} + +// OnCurrentViewDetails provides a mock function with given fields: currentView, finalizedView, currentLeader +func (_m *ParticipantConsumer) OnCurrentViewDetails(currentView uint64, finalizedView uint64, currentLeader flow.Identifier) { + _m.Called(currentView, finalizedView, currentLeader) +} + +// OnEventProcessed provides a mock function with given fields: +func (_m *ParticipantConsumer) OnEventProcessed() { + _m.Called() +} + +// OnLocalTimeout provides a mock function with given fields: currentView +func (_m *ParticipantConsumer) OnLocalTimeout(currentView uint64) { + _m.Called(currentView) +} + +// OnPartialTc provides a mock function with given fields: currentView, partialTc +func (_m *ParticipantConsumer) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { + _m.Called(currentView, partialTc) +} + +// OnQcTriggeredViewChange provides a mock function with given fields: oldView, newView, qc +func (_m *ParticipantConsumer) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { + _m.Called(oldView, newView, qc) +} + +// OnReceiveProposal provides a mock function with given fields: currentView, proposal +func (_m *ParticipantConsumer) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { + _m.Called(currentView, proposal) +} + +// OnReceiveQc provides a mock function with given fields: currentView, qc +func (_m *ParticipantConsumer) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { + _m.Called(currentView, qc) +} + +// OnReceiveTc provides a mock function with given fields: currentView, tc +func (_m *ParticipantConsumer) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { + _m.Called(currentView, tc) +} + +// OnStart provides a mock function with given fields: currentView +func (_m *ParticipantConsumer) OnStart(currentView uint64) { + _m.Called(currentView) +} + +// OnStartingTimeout provides a mock function with given fields: _a0 +func (_m *ParticipantConsumer) OnStartingTimeout(_a0 model.TimerInfo) { + _m.Called(_a0) +} + +// OnTcTriggeredViewChange provides a mock function with given fields: oldView, newView, tc +func (_m *ParticipantConsumer) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { + _m.Called(oldView, newView, tc) +} + +// OnViewChange provides a mock function with given fields: oldView, newView +func (_m *ParticipantConsumer) OnViewChange(oldView uint64, newView uint64) { + _m.Called(oldView, newView) +} + +type mockConstructorTestingTNewParticipantConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewParticipantConsumer creates a new instance of ParticipantConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewParticipantConsumer(t mockConstructorTestingTNewParticipantConsumer) *ParticipantConsumer { + mock := &ParticipantConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/proposal_duration_provider.go b/consensus/hotstuff/mocks/proposal_duration_provider.go new file mode 100644 index 00000000000..2d45f75d409 --- /dev/null +++ b/consensus/hotstuff/mocks/proposal_duration_provider.go @@ -0,0 +1,45 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + time "time" +) + +// ProposalDurationProvider is an autogenerated mock type for the ProposalDurationProvider type +type ProposalDurationProvider struct { + mock.Mock +} + +// TargetPublicationTime provides a mock function with given fields: proposalView, timeViewEntered, parentBlockId +func (_m *ProposalDurationProvider) TargetPublicationTime(proposalView uint64, timeViewEntered time.Time, parentBlockId flow.Identifier) time.Time { + ret := _m.Called(proposalView, timeViewEntered, parentBlockId) + + var r0 time.Time + if rf, ok := ret.Get(0).(func(uint64, time.Time, flow.Identifier) time.Time); ok { + r0 = rf(proposalView, timeViewEntered, parentBlockId) + } else { + r0 = ret.Get(0).(time.Time) + } + + return r0 +} + +type mockConstructorTestingTNewProposalDurationProvider interface { + mock.TestingT + Cleanup(func()) +} + +// NewProposalDurationProvider creates a new instance of ProposalDurationProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewProposalDurationProvider(t mockConstructorTestingTNewProposalDurationProvider) *ProposalDurationProvider { + mock := &ProposalDurationProvider{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/proposal_violation_consumer.go b/consensus/hotstuff/mocks/proposal_violation_consumer.go new file mode 100644 index 00000000000..bb8735a1ca1 --- /dev/null +++ b/consensus/hotstuff/mocks/proposal_violation_consumer.go @@ -0,0 +1,41 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// ProposalViolationConsumer is an autogenerated mock type for the ProposalViolationConsumer type +type ProposalViolationConsumer struct { + mock.Mock +} + +// OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 +func (_m *ProposalViolationConsumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { + _m.Called(_a0, _a1) +} + +// OnInvalidBlockDetected provides a mock function with given fields: err +func (_m *ProposalViolationConsumer) OnInvalidBlockDetected(err flow.Slashable[model.InvalidProposalError]) { + _m.Called(err) +} + +type mockConstructorTestingTNewProposalViolationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewProposalViolationConsumer creates a new instance of ProposalViolationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewProposalViolationConsumer(t mockConstructorTestingTNewProposalViolationConsumer) *ProposalViolationConsumer { + mock := &ProposalViolationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/qc_created_consumer.go b/consensus/hotstuff/mocks/qc_created_consumer.go deleted file mode 100644 index e20bd948fb5..00000000000 --- a/consensus/hotstuff/mocks/qc_created_consumer.go +++ /dev/null @@ -1,34 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mocks - -import ( - flow "github.com/onflow/flow-go/model/flow" - - mock "github.com/stretchr/testify/mock" -) - -// QCCreatedConsumer is an autogenerated mock type for the QCCreatedConsumer type -type QCCreatedConsumer struct { - mock.Mock -} - -// OnQcConstructedFromVotes provides a mock function with given fields: _a0 -func (_m *QCCreatedConsumer) OnQcConstructedFromVotes(_a0 *flow.QuorumCertificate) { - _m.Called(_a0) -} - -type mockConstructorTestingTNewQCCreatedConsumer interface { - mock.TestingT - Cleanup(func()) -} - -// NewQCCreatedConsumer creates a new instance of QCCreatedConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewQCCreatedConsumer(t mockConstructorTestingTNewQCCreatedConsumer) *QCCreatedConsumer { - mock := &QCCreatedConsumer{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/mocks/timeout_aggregation_consumer.go b/consensus/hotstuff/mocks/timeout_aggregation_consumer.go new file mode 100644 index 00000000000..c123201f956 --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_aggregation_consumer.go @@ -0,0 +1,66 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// TimeoutAggregationConsumer is an autogenerated mock type for the TimeoutAggregationConsumer type +type TimeoutAggregationConsumer struct { + mock.Mock +} + +// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 +func (_m *TimeoutAggregationConsumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { + _m.Called(_a0, _a1) +} + +// OnInvalidTimeoutDetected provides a mock function with given fields: err +func (_m *TimeoutAggregationConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + _m.Called(err) +} + +// OnNewQcDiscovered provides a mock function with given fields: certificate +func (_m *TimeoutAggregationConsumer) OnNewQcDiscovered(certificate *flow.QuorumCertificate) { + _m.Called(certificate) +} + +// OnNewTcDiscovered provides a mock function with given fields: certificate +func (_m *TimeoutAggregationConsumer) OnNewTcDiscovered(certificate *flow.TimeoutCertificate) { + _m.Called(certificate) +} + +// OnPartialTcCreated provides a mock function with given fields: view, newestQC, lastViewTC +func (_m *TimeoutAggregationConsumer) OnPartialTcCreated(view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) { + _m.Called(view, newestQC, lastViewTC) +} + +// OnTcConstructedFromTimeouts provides a mock function with given fields: certificate +func (_m *TimeoutAggregationConsumer) OnTcConstructedFromTimeouts(certificate *flow.TimeoutCertificate) { + _m.Called(certificate) +} + +// OnTimeoutProcessed provides a mock function with given fields: timeout +func (_m *TimeoutAggregationConsumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { + _m.Called(timeout) +} + +type mockConstructorTestingTNewTimeoutAggregationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutAggregationConsumer creates a new instance of TimeoutAggregationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutAggregationConsumer(t mockConstructorTestingTNewTimeoutAggregationConsumer) *TimeoutAggregationConsumer { + mock := &TimeoutAggregationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_aggregation_violation_consumer.go b/consensus/hotstuff/mocks/timeout_aggregation_violation_consumer.go new file mode 100644 index 00000000000..552f8650f9f --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_aggregation_violation_consumer.go @@ -0,0 +1,38 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// TimeoutAggregationViolationConsumer is an autogenerated mock type for the TimeoutAggregationViolationConsumer type +type TimeoutAggregationViolationConsumer struct { + mock.Mock +} + +// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 +func (_m *TimeoutAggregationViolationConsumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { + _m.Called(_a0, _a1) +} + +// OnInvalidTimeoutDetected provides a mock function with given fields: err +func (_m *TimeoutAggregationViolationConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + _m.Called(err) +} + +type mockConstructorTestingTNewTimeoutAggregationViolationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutAggregationViolationConsumer creates a new instance of TimeoutAggregationViolationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutAggregationViolationConsumer(t mockConstructorTestingTNewTimeoutAggregationViolationConsumer) *TimeoutAggregationViolationConsumer { + mock := &TimeoutAggregationViolationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_collector_consumer.go b/consensus/hotstuff/mocks/timeout_collector_consumer.go index 459cfb8dd14..629f33f9a14 100644 --- a/consensus/hotstuff/mocks/timeout_collector_consumer.go +++ b/consensus/hotstuff/mocks/timeout_collector_consumer.go @@ -6,6 +6,8 @@ import ( flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" ) // TimeoutCollectorConsumer is an autogenerated mock type for the TimeoutCollectorConsumer type @@ -33,6 +35,11 @@ func (_m *TimeoutCollectorConsumer) OnTcConstructedFromTimeouts(certificate *flo _m.Called(certificate) } +// OnTimeoutProcessed provides a mock function with given fields: timeout +func (_m *TimeoutCollectorConsumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { + _m.Called(timeout) +} + type mockConstructorTestingTNewTimeoutCollectorConsumer interface { mock.TestingT Cleanup(func()) diff --git a/consensus/hotstuff/mocks/vote_aggregation_consumer.go b/consensus/hotstuff/mocks/vote_aggregation_consumer.go new file mode 100644 index 00000000000..0ab7b7f53aa --- /dev/null +++ b/consensus/hotstuff/mocks/vote_aggregation_consumer.go @@ -0,0 +1,56 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// VoteAggregationConsumer is an autogenerated mock type for the VoteAggregationConsumer type +type VoteAggregationConsumer struct { + mock.Mock +} + +// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 +func (_m *VoteAggregationConsumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { + _m.Called(_a0, _a1) +} + +// OnInvalidVoteDetected provides a mock function with given fields: err +func (_m *VoteAggregationConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { + _m.Called(err) +} + +// OnQcConstructedFromVotes provides a mock function with given fields: _a0 +func (_m *VoteAggregationConsumer) OnQcConstructedFromVotes(_a0 *flow.QuorumCertificate) { + _m.Called(_a0) +} + +// OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal +func (_m *VoteAggregationConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + _m.Called(vote, invalidProposal) +} + +// OnVoteProcessed provides a mock function with given fields: vote +func (_m *VoteAggregationConsumer) OnVoteProcessed(vote *model.Vote) { + _m.Called(vote) +} + +type mockConstructorTestingTNewVoteAggregationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewVoteAggregationConsumer creates a new instance of VoteAggregationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewVoteAggregationConsumer(t mockConstructorTestingTNewVoteAggregationConsumer) *VoteAggregationConsumer { + mock := &VoteAggregationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/vote_aggregation_violation_consumer.go b/consensus/hotstuff/mocks/vote_aggregation_violation_consumer.go new file mode 100644 index 00000000000..c27e40c1513 --- /dev/null +++ b/consensus/hotstuff/mocks/vote_aggregation_violation_consumer.go @@ -0,0 +1,43 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// VoteAggregationViolationConsumer is an autogenerated mock type for the VoteAggregationViolationConsumer type +type VoteAggregationViolationConsumer struct { + mock.Mock +} + +// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 +func (_m *VoteAggregationViolationConsumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { + _m.Called(_a0, _a1) +} + +// OnInvalidVoteDetected provides a mock function with given fields: err +func (_m *VoteAggregationViolationConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { + _m.Called(err) +} + +// OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal +func (_m *VoteAggregationViolationConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + _m.Called(vote, invalidProposal) +} + +type mockConstructorTestingTNewVoteAggregationViolationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewVoteAggregationViolationConsumer creates a new instance of VoteAggregationViolationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewVoteAggregationViolationConsumer(t mockConstructorTestingTNewVoteAggregationViolationConsumer) *VoteAggregationViolationConsumer { + mock := &VoteAggregationViolationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/vote_collector_consumer.go b/consensus/hotstuff/mocks/vote_collector_consumer.go new file mode 100644 index 00000000000..5c5b064e975 --- /dev/null +++ b/consensus/hotstuff/mocks/vote_collector_consumer.go @@ -0,0 +1,41 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// VoteCollectorConsumer is an autogenerated mock type for the VoteCollectorConsumer type +type VoteCollectorConsumer struct { + mock.Mock +} + +// OnQcConstructedFromVotes provides a mock function with given fields: _a0 +func (_m *VoteCollectorConsumer) OnQcConstructedFromVotes(_a0 *flow.QuorumCertificate) { + _m.Called(_a0) +} + +// OnVoteProcessed provides a mock function with given fields: vote +func (_m *VoteCollectorConsumer) OnVoteProcessed(vote *model.Vote) { + _m.Called(vote) +} + +type mockConstructorTestingTNewVoteCollectorConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewVoteCollectorConsumer creates a new instance of VoteCollectorConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewVoteCollectorConsumer(t mockConstructorTestingTNewVoteCollectorConsumer) *VoteCollectorConsumer { + mock := &VoteCollectorConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/voter.go b/consensus/hotstuff/mocks/voter.go deleted file mode 100644 index 92536db5553..00000000000 --- a/consensus/hotstuff/mocks/voter.go +++ /dev/null @@ -1,51 +0,0 @@ -// Code generated by mockery v2.13.1. DO NOT EDIT. - -package mocks - -import ( - model "github.com/onflow/flow-go/consensus/hotstuff/model" - mock "github.com/stretchr/testify/mock" -) - -// Voter is an autogenerated mock type for the Voter type -type Voter struct { - mock.Mock -} - -// ProduceVoteIfVotable provides a mock function with given fields: block, curView -func (_m *Voter) ProduceVoteIfVotable(block *model.Block, curView uint64) (*model.Vote, error) { - ret := _m.Called(block, curView) - - var r0 *model.Vote - if rf, ok := ret.Get(0).(func(*model.Block, uint64) *model.Vote); ok { - r0 = rf(block, curView) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.Vote) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(*model.Block, uint64) error); ok { - r1 = rf(block, curView) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type mockConstructorTestingTNewVoter interface { - mock.TestingT - Cleanup(func()) -} - -// NewVoter creates a new instance of Voter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewVoter(t mockConstructorTestingTNewVoter) *Voter { - mock := &Voter{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/model/block.go b/consensus/hotstuff/model/block.go index b221b3ecc00..6c682514dfc 100644 --- a/consensus/hotstuff/model/block.go +++ b/consensus/hotstuff/model/block.go @@ -72,10 +72,10 @@ func NewCertifiedBlock(block *Block, qc *flow.QuorumCertificate) (CertifiedBlock // ID returns unique identifier for the block. // To avoid repeated computation, we use value from the QC. func (b *CertifiedBlock) ID() flow.Identifier { - return b.CertifyingQC.BlockID + return b.Block.BlockID } // View returns view where the block was proposed. func (b *CertifiedBlock) View() uint64 { - return b.CertifyingQC.View + return b.Block.View } diff --git a/consensus/hotstuff/model/errors.go b/consensus/hotstuff/model/errors.go index bbb95ef17b8..4244d0ac531 100644 --- a/consensus/hotstuff/model/errors.go +++ b/consensus/hotstuff/model/errors.go @@ -163,20 +163,69 @@ func (e InvalidTCError) Unwrap() error { return e.Err } -// InvalidBlockError indicates that the block with identifier `BlockID` is invalid +// InvalidProposalError indicates that the proposal is invalid +type InvalidProposalError struct { + InvalidProposal *Proposal + Err error +} + +func NewInvalidProposalErrorf(proposal *Proposal, msg string, args ...interface{}) error { + return InvalidProposalError{ + InvalidProposal: proposal, + Err: fmt.Errorf(msg, args...), + } +} + +func (e InvalidProposalError) Error() string { + return fmt.Sprintf( + "invalid proposal %x at view %d: %s", + e.InvalidProposal.Block.BlockID, + e.InvalidProposal.Block.View, + e.Err.Error(), + ) +} + +func (e InvalidProposalError) Unwrap() error { + return e.Err +} + +// IsInvalidProposalError returns whether an error is InvalidProposalError +func IsInvalidProposalError(err error) bool { + var e InvalidProposalError + return errors.As(err, &e) +} + +// AsInvalidProposalError determines whether the given error is a InvalidProposalError +// (potentially wrapped). It follows the same semantics as a checked type cast. +func AsInvalidProposalError(err error) (*InvalidProposalError, bool) { + var e InvalidProposalError + ok := errors.As(err, &e) + if ok { + return &e, true + } + return nil, false +} + +// InvalidBlockError indicates that the block is invalid type InvalidBlockError struct { - BlockID flow.Identifier - View uint64 - Err error + InvalidBlock *Block + Err error } -// NewInvalidBlockError instantiates an `InvalidBlockError`. Input `err` cannot be nil. -func NewInvalidBlockError(blockID flow.Identifier, view uint64, err error) error { - return InvalidBlockError{BlockID: blockID, View: view, Err: err} +func NewInvalidBlockErrorf(block *Block, msg string, args ...interface{}) error { + return InvalidBlockError{ + InvalidBlock: block, + Err: fmt.Errorf(msg, args...), + } } func (e InvalidBlockError) Error() string { - return fmt.Sprintf("invalid block %x at view %d: %s", e.BlockID, e.View, e.Err.Error()) + return fmt.Sprintf( + "invalid block %x at view %d: %s", + e.InvalidBlock.BlockID, + e.InvalidBlock.View, + e.Err.Error(), + ) } // IsInvalidBlockError returns whether an error is InvalidBlockError @@ -185,6 +234,17 @@ func IsInvalidBlockError(err error) bool { return errors.As(err, &e) } +// AsInvalidBlockError determines whether the given error is a InvalidProposalError +// (potentially wrapped). It follows the same semantics as a checked type cast. +func AsInvalidBlockError(err error) (*InvalidBlockError, bool) { + var e InvalidBlockError + ok := errors.As(err, &e) + if ok { + return &e, true + } + return nil, false +} + func (e InvalidBlockError) Unwrap() error { return e.Err } diff --git a/consensus/hotstuff/model/proposal.go b/consensus/hotstuff/model/proposal.go index 538190906dd..6566de09a97 100644 --- a/consensus/hotstuff/model/proposal.go +++ b/consensus/hotstuff/model/proposal.go @@ -25,15 +25,11 @@ func (p *Proposal) ProposerVote() *Vote { // ProposalFromFlow turns a flow header into a hotstuff block type. func ProposalFromFlow(header *flow.Header) *Proposal { - - block := BlockFromFlow(header) - proposal := Proposal{ - Block: block, + Block: BlockFromFlow(header), SigData: header.ProposerSigData, LastViewTC: header.LastViewTC, } - return &proposal } diff --git a/consensus/hotstuff/model/signature_data.go b/consensus/hotstuff/model/signature_data.go index 0eb6c0741ff..cb6cb5217b3 100644 --- a/consensus/hotstuff/model/signature_data.go +++ b/consensus/hotstuff/model/signature_data.go @@ -2,9 +2,11 @@ package model import ( "bytes" + "fmt" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/encoding/rlp" + "github.com/onflow/flow-go/model/flow" ) // SigDataPacker implements logic for encoding/decoding SignatureData using RLP encoding. @@ -45,13 +47,17 @@ func (p *SigDataPacker) Decode(data []byte) (*SignatureData, error) { return &sigData, nil } -// UnpackRandomBeaconSig takes sigData previously packed by packer, -// decodes it and extracts random beacon signature. -// This function is side-effect free. It only ever returns a -// model.InvalidFormatError, which indicates an invalid encoding. -func UnpackRandomBeaconSig(sigData []byte) (crypto.Signature, error) { - // decode into typed data +// BeaconSignature extracts the source of randomness from the QC sigData. +// +// The sigData is an RLP encoded structure that is part of QuorumCertificate. +// The function only ever returns a model.InvalidFormatError, which indicates an +// invalid encoding. +func BeaconSignature(qc *flow.QuorumCertificate) ([]byte, error) { + // unpack sig data to extract random beacon signature packer := SigDataPacker{} - sig, err := packer.Decode(sigData) - return sig.ReconstructedRandomBeaconSig, err + sigData, err := packer.Decode(qc.SigData) + if err != nil { + return nil, fmt.Errorf("could not unpack block signature: %w", err) + } + return sigData.ReconstructedRandomBeaconSig, nil } diff --git a/consensus/hotstuff/notifications/log_consumer.go b/consensus/hotstuff/notifications/log_consumer.go index 0f3329c356d..f8baea639dc 100644 --- a/consensus/hotstuff/notifications/log_consumer.go +++ b/consensus/hotstuff/notifications/log_consumer.go @@ -18,7 +18,8 @@ type LogConsumer struct { } var _ hotstuff.Consumer = (*LogConsumer)(nil) -var _ hotstuff.TimeoutCollectorConsumer = (*LogConsumer)(nil) +var _ hotstuff.TimeoutAggregationConsumer = (*LogConsumer)(nil) +var _ hotstuff.VoteAggregationConsumer = (*LogConsumer)(nil) func NewLogConsumer(log zerolog.Logger) *LogConsumer { lc := &LogConsumer{ @@ -45,8 +46,22 @@ func (lc *LogConsumer) OnFinalizedBlock(block *model.Block) { Msg("block finalized") } +func (lc *LogConsumer) OnInvalidBlockDetected(err flow.Slashable[model.InvalidProposalError]) { + invalidBlock := err.Message.InvalidProposal.Block + lc.log.Warn(). + Str(logging.KeySuspicious, "true"). + Hex("origin_id", err.OriginID[:]). + Uint64("block_view", invalidBlock.View). + Hex("proposer_id", invalidBlock.ProposerID[:]). + Hex("block_id", invalidBlock.BlockID[:]). + Uint64("qc_block_view", invalidBlock.QC.View). + Hex("qc_block_id", invalidBlock.QC.BlockID[:]). + Msgf("invalid block detected: %s", err.Message.Error()) +} + func (lc *LogConsumer) OnDoubleProposeDetected(block *model.Block, alt *model.Block) { lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("block_view", block.View). Hex("block_id", block.BlockID[:]). Hex("alt_id", alt.BlockID[:]). @@ -165,6 +180,7 @@ func (lc *LogConsumer) OnCurrentViewDetails(currentView, finalizedView uint64, c func (lc *LogConsumer) OnDoubleVotingDetected(vote *model.Vote, alt *model.Vote) { lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("vote_view", vote.View). Hex("voted_block_id", vote.BlockID[:]). Hex("alt_id", alt.BlockID[:]). @@ -174,6 +190,7 @@ func (lc *LogConsumer) OnDoubleVotingDetected(vote *model.Vote, alt *model.Vote) func (lc *LogConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("vote_view", err.Vote.View). Hex("voted_block_id", err.Vote.BlockID[:]). Hex("voter_id", err.Vote.SignerID[:]). @@ -182,6 +199,7 @@ func (lc *LogConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { func (lc *LogConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, proposal *model.Proposal) { lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("vote_view", vote.View). Hex("voted_block_id", vote.BlockID[:]). Hex("voter_id", vote.SignerID[:]). @@ -191,6 +209,7 @@ func (lc *LogConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, proposal func (lc *LogConsumer) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, alt *model.TimeoutObject) { lc.log.Warn(). + Str(logging.KeySuspicious, "true"). Uint64("timeout_view", timeout.View). Hex("signer_id", logging.ID(timeout.SignerID)). Hex("timeout_id", logging.ID(timeout.ID())). @@ -200,7 +219,9 @@ func (lc *LogConsumer) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, alt func (lc *LogConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { log := err.Timeout.LogContext(lc.log).Logger() - log.Warn().Msgf("invalid timeout detected: %s", err.Error()) + log.Warn(). + Str(logging.KeySuspicious, "true"). + Msgf("invalid timeout detected: %s", err.Error()) } func (lc *LogConsumer) logBasicBlockData(loggerEvent *zerolog.Event, block *model.Block) *zerolog.Event { @@ -273,3 +294,10 @@ func (lc *LogConsumer) OnOwnProposal(header *flow.Header, targetPublicationTime Time("target_publication_time", targetPublicationTime). Msg("publishing HotStuff block proposal") } + +func (lc *LogConsumer) OnQcConstructedFromVotes(qc *flow.QuorumCertificate) { + lc.log.Info(). + Uint64("view", qc.View). + Hex("block_id", qc.BlockID[:]). + Msg("QC constructed from votes") +} diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index b5d980acdd3..d8ad3e66e4f 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -11,8 +11,9 @@ import ( // NoopConsumer is an implementation of the notifications consumer that // doesn't do anything. type NoopConsumer struct { + NoopProposalViolationConsumer NoopFinalizationConsumer - NoopPartialConsumer + NoopParticipantConsumer NoopCommunicatorConsumer } @@ -25,45 +26,31 @@ func NewNoopConsumer() *NoopConsumer { // no-op implementation of hotstuff.Consumer(but not nested interfaces) -type NoopPartialConsumer struct{} +type NoopParticipantConsumer struct{} -func (*NoopPartialConsumer) OnEventProcessed() {} +func (*NoopParticipantConsumer) OnEventProcessed() {} -func (*NoopPartialConsumer) OnStart(uint64) {} +func (*NoopParticipantConsumer) OnStart(uint64) {} -func (*NoopPartialConsumer) OnReceiveProposal(uint64, *model.Proposal) {} +func (*NoopParticipantConsumer) OnReceiveProposal(uint64, *model.Proposal) {} -func (*NoopPartialConsumer) OnReceiveQc(uint64, *flow.QuorumCertificate) {} +func (*NoopParticipantConsumer) OnReceiveQc(uint64, *flow.QuorumCertificate) {} -func (*NoopPartialConsumer) OnReceiveTc(uint64, *flow.TimeoutCertificate) {} +func (*NoopParticipantConsumer) OnReceiveTc(uint64, *flow.TimeoutCertificate) {} -func (*NoopPartialConsumer) OnPartialTc(uint64, *hotstuff.PartialTcCreated) {} +func (*NoopParticipantConsumer) OnPartialTc(uint64, *hotstuff.PartialTcCreated) {} -func (*NoopPartialConsumer) OnLocalTimeout(uint64) {} +func (*NoopParticipantConsumer) OnLocalTimeout(uint64) {} -func (*NoopPartialConsumer) OnViewChange(uint64, uint64) {} +func (*NoopParticipantConsumer) OnViewChange(uint64, uint64) {} -func (*NoopPartialConsumer) OnQcTriggeredViewChange(uint64, uint64, *flow.QuorumCertificate) {} +func (*NoopParticipantConsumer) OnQcTriggeredViewChange(uint64, uint64, *flow.QuorumCertificate) {} -func (*NoopPartialConsumer) OnTcTriggeredViewChange(uint64, uint64, *flow.TimeoutCertificate) {} +func (*NoopParticipantConsumer) OnTcTriggeredViewChange(uint64, uint64, *flow.TimeoutCertificate) {} -func (*NoopPartialConsumer) OnStartingTimeout(model.TimerInfo) {} +func (*NoopParticipantConsumer) OnStartingTimeout(model.TimerInfo) {} -func (*NoopPartialConsumer) OnVoteProcessed(*model.Vote) {} - -func (*NoopPartialConsumer) OnTimeoutProcessed(*model.TimeoutObject) {} - -func (*NoopPartialConsumer) OnCurrentViewDetails(uint64, uint64, flow.Identifier) {} - -func (*NoopPartialConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} - -func (*NoopPartialConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} - -func (*NoopPartialConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} - -func (*NoopPartialConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) {} - -func (*NoopPartialConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} +func (*NoopParticipantConsumer) OnCurrentViewDetails(uint64, uint64, flow.Identifier) {} // no-op implementation of hotstuff.FinalizationConsumer @@ -75,8 +62,6 @@ func (*NoopFinalizationConsumer) OnBlockIncorporated(*model.Block) {} func (*NoopFinalizationConsumer) OnFinalizedBlock(*model.Block) {} -func (*NoopFinalizationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} - // no-op implementation of hotstuff.TimeoutCollectorConsumer type NoopTimeoutCollectorConsumer struct{} @@ -92,6 +77,8 @@ func (*NoopTimeoutCollectorConsumer) OnNewQcDiscovered(*flow.QuorumCertificate) func (*NoopTimeoutCollectorConsumer) OnNewTcDiscovered(*flow.TimeoutCertificate) {} +func (*NoopTimeoutCollectorConsumer) OnTimeoutProcessed(*model.TimeoutObject) {} + // no-op implementation of hotstuff.CommunicatorConsumer type NoopCommunicatorConsumer struct{} @@ -104,10 +91,34 @@ func (*NoopCommunicatorConsumer) OnOwnTimeout(*model.TimeoutObject) {} func (*NoopCommunicatorConsumer) OnOwnProposal(*flow.Header, time.Time) {} -// no-op implementation of hotstuff.QCCreatedConsumer +// no-op implementation of hotstuff.VoteCollectorConsumer + +type NoopVoteCollectorConsumer struct{} + +var _ hotstuff.VoteCollectorConsumer = (*NoopVoteCollectorConsumer)(nil) -type NoopQCCreatedConsumer struct{} +func (*NoopVoteCollectorConsumer) OnQcConstructedFromVotes(*flow.QuorumCertificate) {} -var _ hotstuff.QCCreatedConsumer = (*NoopQCCreatedConsumer)(nil) +func (*NoopVoteCollectorConsumer) OnVoteProcessed(*model.Vote) {} + +// no-op implementation of hotstuff.ProposalViolationConsumer + +type NoopProposalViolationConsumer struct{} + +var _ hotstuff.ProposalViolationConsumer = (*NoopProposalViolationConsumer)(nil) + +func (*NoopProposalViolationConsumer) OnInvalidBlockDetected(flow.Slashable[model.InvalidProposalError]) { +} + +func (*NoopProposalViolationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} + +func (*NoopProposalViolationConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} + +func (*NoopProposalViolationConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} + +func (*NoopProposalViolationConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} + +func (*NoopProposalViolationConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) { +} -func (*NoopQCCreatedConsumer) OnQcConstructedFromVotes(*flow.QuorumCertificate) {} +func (*NoopProposalViolationConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} diff --git a/consensus/hotstuff/notifications/pubsub/communicator_distributor.go b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go new file mode 100644 index 00000000000..5e0604fa83c --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/communicator_distributor.go @@ -0,0 +1,56 @@ +package pubsub + +import ( + "sync" + "time" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// CommunicatorDistributor ingests outbound consensus messages from HotStuff's core logic and +// distributes them to consumers. This logic only runs inside active consensus participants proposing +// blocks, voting, collecting + aggregating votes to QCs, and participating in the pacemaker (sending +// timeouts, collecting + aggregating timeouts to TCs). +// Concurrently safe. +type CommunicatorDistributor struct { + consumers []hotstuff.CommunicatorConsumer + lock sync.RWMutex +} + +var _ hotstuff.CommunicatorConsumer = (*CommunicatorDistributor)(nil) + +func NewCommunicatorDistributor() *CommunicatorDistributor { + return &CommunicatorDistributor{} +} + +func (d *CommunicatorDistributor) AddCommunicatorConsumer(consumer hotstuff.CommunicatorConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *CommunicatorDistributor) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, s := range d.consumers { + s.OnOwnVote(blockID, view, sigData, recipientID) + } +} + +func (d *CommunicatorDistributor) OnOwnTimeout(timeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, s := range d.consumers { + s.OnOwnTimeout(timeout) + } +} + +func (d *CommunicatorDistributor) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, s := range d.consumers { + s.OnOwnProposal(proposal, targetPublicationTime) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index d122ad8cde3..ea461a23742 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -1,231 +1,95 @@ package pubsub import ( - "sync" - "time" - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" ) -// Distributor distributes notifications to a list of subscribers (event consumers). +// Distributor distributes notifications to a list of consumers (event consumers). // // It allows thread-safe subscription of multiple consumers to events. type Distributor struct { - subscribers []hotstuff.Consumer - lock sync.RWMutex + *FollowerDistributor + *CommunicatorDistributor + *ParticipantDistributor } var _ hotstuff.Consumer = (*Distributor)(nil) -func (p *Distributor) OnEventProcessed() { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnEventProcessed() - } -} - func NewDistributor() *Distributor { - return &Distributor{} + return &Distributor{ + FollowerDistributor: NewFollowerDistributor(), + CommunicatorDistributor: NewCommunicatorDistributor(), + ParticipantDistributor: NewParticipantDistributor(), + } } // AddConsumer adds an event consumer to the Distributor func (p *Distributor) AddConsumer(consumer hotstuff.Consumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.subscribers = append(p.subscribers, consumer) -} - -func (p *Distributor) OnStart(currentView uint64) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnStart(currentView) - } -} - -func (p *Distributor) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnReceiveProposal(currentView, proposal) - } -} - -func (p *Distributor) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnReceiveQc(currentView, qc) - } -} - -func (p *Distributor) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnReceiveTc(currentView, tc) - } -} - -func (p *Distributor) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnPartialTc(currentView, partialTc) - } -} - -func (p *Distributor) OnLocalTimeout(currentView uint64) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnLocalTimeout(currentView) - } -} - -func (p *Distributor) OnViewChange(oldView, newView uint64) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnViewChange(oldView, newView) - } -} - -func (p *Distributor) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnQcTriggeredViewChange(oldView, newView, qc) - } -} - -func (p *Distributor) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnTcTriggeredViewChange(oldView, newView, tc) - } -} - -func (p *Distributor) OnStartingTimeout(timerInfo model.TimerInfo) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnStartingTimeout(timerInfo) - } + p.FollowerDistributor.AddFollowerConsumer(consumer) + p.CommunicatorDistributor.AddCommunicatorConsumer(consumer) + p.ParticipantDistributor.AddParticipantConsumer(consumer) } -func (p *Distributor) OnVoteProcessed(vote *model.Vote) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnVoteProcessed(vote) - } -} - -func (p *Distributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnTimeoutProcessed(timeout) - } -} - -func (p *Distributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnCurrentViewDetails(currentView, finalizedView, currentLeader) - } +// FollowerDistributor ingests consensus follower events and distributes it to consumers. +// It allows thread-safe subscription of multiple consumers to events. +type FollowerDistributor struct { + *ProposalViolationDistributor + *FinalizationDistributor } -func (p *Distributor) OnBlockIncorporated(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnBlockIncorporated(block) - } -} +var _ hotstuff.FollowerConsumer = (*FollowerDistributor)(nil) -func (p *Distributor) OnFinalizedBlock(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnFinalizedBlock(block) +func NewFollowerDistributor() *FollowerDistributor { + return &FollowerDistributor{ + ProposalViolationDistributor: NewProtocolViolationDistributor(), + FinalizationDistributor: NewFinalizationDistributor(), } } -func (p *Distributor) OnDoubleProposeDetected(block1, block2 *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnDoubleProposeDetected(block1, block2) - } +// AddFollowerConsumer registers the input `consumer` to be notified on `hotstuff.ConsensusFollowerConsumer` events. +func (d *FollowerDistributor) AddFollowerConsumer(consumer hotstuff.FollowerConsumer) { + d.FinalizationDistributor.AddFinalizationConsumer(consumer) + d.ProposalViolationDistributor.AddProposalViolationConsumer(consumer) } -func (p *Distributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnDoubleVotingDetected(vote1, vote2) - } +// TimeoutAggregationDistributor ingests timeout aggregation events and distributes it to consumers. +// It allows thread-safe subscription of multiple consumers to events. +type TimeoutAggregationDistributor struct { + *TimeoutAggregationViolationDistributor + *TimeoutCollectorDistributor } -func (p *Distributor) OnInvalidVoteDetected(err model.InvalidVoteError) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnInvalidVoteDetected(err) - } -} +var _ hotstuff.TimeoutAggregationConsumer = (*TimeoutAggregationDistributor)(nil) -func (p *Distributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) +func NewTimeoutAggregationDistributor() *TimeoutAggregationDistributor { + return &TimeoutAggregationDistributor{ + TimeoutAggregationViolationDistributor: NewTimeoutAggregationViolationDistributor(), + TimeoutCollectorDistributor: NewTimeoutCollectorDistributor(), } } -func (p *Distributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) - } +func (d *TimeoutAggregationDistributor) AddTimeoutAggregationConsumer(consumer hotstuff.TimeoutAggregationConsumer) { + d.TimeoutAggregationViolationDistributor.AddTimeoutAggregationViolationConsumer(consumer) + d.TimeoutCollectorDistributor.AddTimeoutCollectorConsumer(consumer) } -func (p *Distributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, subscriber := range p.subscribers { - subscriber.OnInvalidTimeoutDetected(err) - } +// VoteAggregationDistributor ingests vote aggregation events and distributes it to consumers. +// It allows thread-safe subscription of multiple consumers to events. +type VoteAggregationDistributor struct { + *VoteAggregationViolationDistributor + *VoteCollectorDistributor } -func (p *Distributor) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, s := range p.subscribers { - s.OnOwnVote(blockID, view, sigData, recipientID) - } -} +var _ hotstuff.VoteAggregationConsumer = (*VoteAggregationDistributor)(nil) -func (p *Distributor) OnOwnTimeout(timeout *model.TimeoutObject) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, s := range p.subscribers { - s.OnOwnTimeout(timeout) +func NewVoteAggregationDistributor() *VoteAggregationDistributor { + return &VoteAggregationDistributor{ + VoteAggregationViolationDistributor: NewVoteAggregationViolationDistributor(), + VoteCollectorDistributor: NewQCCreatedDistributor(), } } -func (p *Distributor) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, s := range p.subscribers { - s.OnOwnProposal(proposal, targetPublicationTime) - } +func (d *VoteAggregationDistributor) AddVoteAggregationConsumer(consumer hotstuff.VoteAggregationConsumer) { + d.VoteAggregationViolationDistributor.AddVoteAggregationViolationConsumer(consumer) + d.VoteCollectorDistributor.AddVoteCollectorConsumer(consumer) } diff --git a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go index 6d1c72ef8e6..e351575c122 100644 --- a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go @@ -5,75 +5,64 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/consensus/hotstuff/notifications" ) type OnBlockFinalizedConsumer = func(block *model.Block) type OnBlockIncorporatedConsumer = func(block *model.Block) -// FinalizationDistributor ingests finalization events from hotstuff and distributes it to subscribers. +// FinalizationDistributor ingests events from HotStuff's logic for tracking forks + finalization +// and distributes them to consumers. This logic generally runs inside all nodes (irrespectively whether +// they are active consensus participants or or only consensus followers). +// Concurrently safe. type FinalizationDistributor struct { - notifications.NoopConsumer - blockFinalizedConsumers []OnBlockFinalizedConsumer - blockIncorporatedConsumers []OnBlockIncorporatedConsumer - hotStuffFinalizationConsumers []hotstuff.FinalizationConsumer - lock sync.RWMutex + blockFinalizedConsumers []OnBlockFinalizedConsumer + blockIncorporatedConsumers []OnBlockIncorporatedConsumer + consumers []hotstuff.FinalizationConsumer + lock sync.RWMutex } -var _ hotstuff.Consumer = (*FinalizationDistributor)(nil) +var _ hotstuff.FinalizationConsumer = (*FinalizationDistributor)(nil) func NewFinalizationDistributor() *FinalizationDistributor { - return &FinalizationDistributor{ - blockFinalizedConsumers: make([]OnBlockFinalizedConsumer, 0), - blockIncorporatedConsumers: make([]OnBlockIncorporatedConsumer, 0), - lock: sync.RWMutex{}, - } + return &FinalizationDistributor{} } -func (p *FinalizationDistributor) AddOnBlockFinalizedConsumer(consumer OnBlockFinalizedConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.blockFinalizedConsumers = append(p.blockFinalizedConsumers, consumer) +func (d *FinalizationDistributor) AddOnBlockFinalizedConsumer(consumer OnBlockFinalizedConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.blockFinalizedConsumers = append(d.blockFinalizedConsumers, consumer) } -func (p *FinalizationDistributor) AddOnBlockIncorporatedConsumer(consumer OnBlockIncorporatedConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.blockIncorporatedConsumers = append(p.blockIncorporatedConsumers, consumer) +func (d *FinalizationDistributor) AddOnBlockIncorporatedConsumer(consumer OnBlockIncorporatedConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.blockIncorporatedConsumers = append(d.blockIncorporatedConsumers, consumer) } -func (p *FinalizationDistributor) AddConsumer(consumer hotstuff.FinalizationConsumer) { - p.lock.Lock() - defer p.lock.Unlock() - p.hotStuffFinalizationConsumers = append(p.hotStuffFinalizationConsumers, consumer) +func (d *FinalizationDistributor) AddFinalizationConsumer(consumer hotstuff.FinalizationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) } -func (p *FinalizationDistributor) OnBlockIncorporated(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.blockIncorporatedConsumers { +func (d *FinalizationDistributor) OnBlockIncorporated(block *model.Block) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.blockIncorporatedConsumers { consumer(block) } - for _, consumer := range p.hotStuffFinalizationConsumers { + for _, consumer := range d.consumers { consumer.OnBlockIncorporated(block) } } -func (p *FinalizationDistributor) OnFinalizedBlock(block *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.blockFinalizedConsumers { +func (d *FinalizationDistributor) OnFinalizedBlock(block *model.Block) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.blockFinalizedConsumers { consumer(block) } - for _, consumer := range p.hotStuffFinalizationConsumers { + for _, consumer := range d.consumers { consumer.OnFinalizedBlock(block) } } - -func (p *FinalizationDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { - p.lock.RLock() - defer p.lock.RUnlock() - for _, consumer := range p.hotStuffFinalizationConsumers { - consumer.OnDoubleProposeDetected(block1, block2) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/participant_distributor.go b/consensus/hotstuff/notifications/pubsub/participant_distributor.go new file mode 100644 index 00000000000..f5047cd7a53 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/participant_distributor.go @@ -0,0 +1,127 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// ParticipantDistributor ingests events from HotStuff's core logic and distributes them to +// consumers. This logic only runs inside active consensus participants proposing blocks, voting, +// collecting + aggregating votes to QCs, and participating in the pacemaker (sending timeouts, +// collecting + aggregating timeouts to TCs). +// Concurrently safe. +type ParticipantDistributor struct { + consumers []hotstuff.ParticipantConsumer + lock sync.RWMutex +} + +var _ hotstuff.ParticipantConsumer = (*ParticipantDistributor)(nil) + +func NewParticipantDistributor() *ParticipantDistributor { + return &ParticipantDistributor{} +} + +func (d *ParticipantDistributor) AddParticipantConsumer(consumer hotstuff.ParticipantConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *ParticipantDistributor) OnEventProcessed() { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnEventProcessed() + } +} + +func (d *ParticipantDistributor) OnStart(currentView uint64) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnStart(currentView) + } +} + +func (d *ParticipantDistributor) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnReceiveProposal(currentView, proposal) + } +} + +func (d *ParticipantDistributor) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnReceiveQc(currentView, qc) + } +} + +func (d *ParticipantDistributor) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnReceiveTc(currentView, tc) + } +} + +func (d *ParticipantDistributor) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnPartialTc(currentView, partialTc) + } +} + +func (d *ParticipantDistributor) OnLocalTimeout(currentView uint64) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnLocalTimeout(currentView) + } +} + +func (d *ParticipantDistributor) OnViewChange(oldView, newView uint64) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnViewChange(oldView, newView) + } +} + +func (d *ParticipantDistributor) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnQcTriggeredViewChange(oldView, newView, qc) + } +} + +func (d *ParticipantDistributor) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnTcTriggeredViewChange(oldView, newView, tc) + } +} + +func (d *ParticipantDistributor) OnStartingTimeout(timerInfo model.TimerInfo) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnStartingTimeout(timerInfo) + } +} + +func (d *ParticipantDistributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnCurrentViewDetails(currentView, finalizedView, currentLeader) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go b/consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go new file mode 100644 index 00000000000..7b974a3269c --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/proposal_violation_distributor.go @@ -0,0 +1,46 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// ProposalViolationDistributor ingests notifications about HotStuff-protocol violations and +// distributes them to consumers. Such notifications are produced by the active consensus +// participants and the consensus follower. +// Concurrently safe. +type ProposalViolationDistributor struct { + consumers []hotstuff.ProposalViolationConsumer + lock sync.RWMutex +} + +var _ hotstuff.ProposalViolationConsumer = (*ProposalViolationDistributor)(nil) + +func NewProtocolViolationDistributor() *ProposalViolationDistributor { + return &ProposalViolationDistributor{} +} + +func (d *ProposalViolationDistributor) AddProposalViolationConsumer(consumer hotstuff.ProposalViolationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *ProposalViolationDistributor) OnInvalidBlockDetected(err flow.Slashable[model.InvalidProposalError]) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnInvalidBlockDetected(err) + } +} + +func (d *ProposalViolationDistributor) OnDoubleProposeDetected(block1, block2 *model.Block) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnDoubleProposeDetected(block1, block2) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go deleted file mode 100644 index 166fa9cf757..00000000000 --- a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go +++ /dev/null @@ -1,39 +0,0 @@ -package pubsub - -import ( - "sync" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/model/flow" -) - -// QCCreatedDistributor ingests events about QC creation from hotstuff and distributes them to subscribers. -// Objects are concurrency safe. -// NOTE: it can be refactored to work without lock since usually we never subscribe after startup. Mostly -// list of observers is static. -type QCCreatedDistributor struct { - qcCreatedConsumers []hotstuff.QCCreatedConsumer - lock sync.RWMutex -} - -var _ hotstuff.QCCreatedConsumer = (*QCCreatedDistributor)(nil) - -func NewQCCreatedDistributor() *QCCreatedDistributor { - return &QCCreatedDistributor{ - qcCreatedConsumers: make([]hotstuff.QCCreatedConsumer, 0), - } -} - -func (d *QCCreatedDistributor) AddConsumer(consumer hotstuff.QCCreatedConsumer) { - d.lock.Lock() - defer d.lock.Unlock() - d.qcCreatedConsumers = append(d.qcCreatedConsumers, consumer) -} - -func (d *QCCreatedDistributor) OnQcConstructedFromVotes(qc *flow.QuorumCertificate) { - d.lock.RLock() - defer d.lock.RUnlock() - for _, consumer := range d.qcCreatedConsumers { - consumer.OnQcConstructedFromVotes(qc) - } -} diff --git a/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go b/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go new file mode 100644 index 00000000000..25458088f87 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/timeout_aggregation_violation_consumer.go @@ -0,0 +1,44 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// TimeoutAggregationViolationDistributor ingests notifications about timeout aggregation violations and +// distributes them to consumers. Such notifications are produced by the timeout aggregation logic. +// Concurrently safe. +type TimeoutAggregationViolationDistributor struct { + consumers []hotstuff.TimeoutAggregationViolationConsumer + lock sync.RWMutex +} + +var _ hotstuff.TimeoutAggregationViolationConsumer = (*TimeoutAggregationViolationDistributor)(nil) + +func NewTimeoutAggregationViolationDistributor() *TimeoutAggregationViolationDistributor { + return &TimeoutAggregationViolationDistributor{} +} + +func (d *TimeoutAggregationViolationDistributor) AddTimeoutAggregationViolationConsumer(consumer hotstuff.TimeoutAggregationViolationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *TimeoutAggregationViolationDistributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) + } +} + +func (d *TimeoutAggregationViolationDistributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnInvalidTimeoutDetected(err) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go index 8387fb81663..b2bfd6b235e 100644 --- a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go @@ -4,13 +4,13 @@ import ( "sync" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) -// TimeoutCollectorDistributor ingests events from hotstuff and distributes them to subscribers. -// Concurrently safe -// TODO: investigate if this can be updated using atomics to prevent locking on mutex since we always add all consumers -// before delivering events. +// TimeoutCollectorDistributor ingests notifications about timeout aggregation and +// distributes them to consumers. Such notifications are produced by the timeout aggregation logic. +// Concurrently safe. type TimeoutCollectorDistributor struct { lock sync.RWMutex consumers []hotstuff.TimeoutCollectorConsumer @@ -19,12 +19,10 @@ type TimeoutCollectorDistributor struct { var _ hotstuff.TimeoutCollectorConsumer = (*TimeoutCollectorDistributor)(nil) func NewTimeoutCollectorDistributor() *TimeoutCollectorDistributor { - return &TimeoutCollectorDistributor{ - consumers: make([]hotstuff.TimeoutCollectorConsumer, 0), - } + return &TimeoutCollectorDistributor{} } -func (d *TimeoutCollectorDistributor) AddConsumer(consumer hotstuff.TimeoutCollectorConsumer) { +func (d *TimeoutCollectorDistributor) AddTimeoutCollectorConsumer(consumer hotstuff.TimeoutCollectorConsumer) { d.lock.Lock() defer d.lock.Unlock() d.consumers = append(d.consumers, consumer) @@ -61,3 +59,11 @@ func (d *TimeoutCollectorDistributor) OnNewTcDiscovered(tc *flow.TimeoutCertific consumer.OnNewTcDiscovered(tc) } } + +func (d *TimeoutCollectorDistributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnTimeoutProcessed(timeout) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go b/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go new file mode 100644 index 00000000000..d9d1e9baa26 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/vote_aggregation_violation_consumer.go @@ -0,0 +1,52 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// VoteAggregationViolationDistributor ingests notifications about vote aggregation violations and +// distributes them to consumers. Such notifications are produced by the vote aggregation logic. +// Concurrently safe. +type VoteAggregationViolationDistributor struct { + consumers []hotstuff.VoteAggregationViolationConsumer + lock sync.RWMutex +} + +var _ hotstuff.VoteAggregationViolationConsumer = (*VoteAggregationViolationDistributor)(nil) + +func NewVoteAggregationViolationDistributor() *VoteAggregationViolationDistributor { + return &VoteAggregationViolationDistributor{} +} + +func (d *VoteAggregationViolationDistributor) AddVoteAggregationViolationConsumer(consumer hotstuff.VoteAggregationViolationConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *VoteAggregationViolationDistributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnDoubleVotingDetected(vote1, vote2) + } +} + +func (d *VoteAggregationViolationDistributor) OnInvalidVoteDetected(err model.InvalidVoteError) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnInvalidVoteDetected(err) + } +} + +func (d *VoteAggregationViolationDistributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go new file mode 100644 index 00000000000..c96631aed78 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/vote_collector_distributor.go @@ -0,0 +1,45 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// VoteCollectorDistributor ingests notifications about vote aggregation and +// distributes them to consumers. Such notifications are produced by the vote aggregation logic. +// Concurrently safe. +type VoteCollectorDistributor struct { + consumers []hotstuff.VoteCollectorConsumer + lock sync.RWMutex +} + +var _ hotstuff.VoteCollectorConsumer = (*VoteCollectorDistributor)(nil) + +func NewQCCreatedDistributor() *VoteCollectorDistributor { + return &VoteCollectorDistributor{} +} + +func (d *VoteCollectorDistributor) AddVoteCollectorConsumer(consumer hotstuff.VoteCollectorConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *VoteCollectorDistributor) OnQcConstructedFromVotes(qc *flow.QuorumCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.consumers { + consumer.OnQcConstructedFromVotes(qc) + } +} + +func (d *VoteCollectorDistributor) OnVoteProcessed(vote *model.Vote) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, subscriber := range d.consumers { + subscriber.OnVoteProcessed(vote) + } +} diff --git a/consensus/hotstuff/notifications/slashing_violation_consumer.go b/consensus/hotstuff/notifications/slashing_violation_consumer.go index fb80e15e522..c03347ece6f 100644 --- a/consensus/hotstuff/notifications/slashing_violation_consumer.go +++ b/consensus/hotstuff/notifications/slashing_violation_consumer.go @@ -3,22 +3,39 @@ package notifications import ( "github.com/rs/zerolog" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/logging" ) // SlashingViolationsConsumer is an implementation of the notifications consumer that logs a // message for any slashable offenses. type SlashingViolationsConsumer struct { - NoopConsumer log zerolog.Logger } +var _ hotstuff.ProposalViolationConsumer = (*SlashingViolationsConsumer)(nil) +var _ hotstuff.VoteAggregationViolationConsumer = (*SlashingViolationsConsumer)(nil) +var _ hotstuff.TimeoutAggregationViolationConsumer = (*SlashingViolationsConsumer)(nil) + func NewSlashingViolationsConsumer(log zerolog.Logger) *SlashingViolationsConsumer { return &SlashingViolationsConsumer{ log: log, } } +func (c *SlashingViolationsConsumer) OnInvalidBlockDetected(err flow.Slashable[model.InvalidProposalError]) { + block := err.Message.InvalidProposal.Block + c.log.Warn(). + Bool(logging.KeySuspicious, true). + Hex("origin_id", err.OriginID[:]). + Hex("proposer_id", block.ProposerID[:]). + Uint64("block_view", block.View). + Hex("block_id", block.BlockID[:]). + Hex("block_payloadhash", block.PayloadHash[:]). + Time("block_timestamp", block.Timestamp). + Msgf("OnInvalidBlockDetected: %s", err.Message.Error()) +} func (c *SlashingViolationsConsumer) OnDoubleVotingDetected(vote1 *model.Vote, vote2 *model.Vote) { c.log.Warn(). @@ -41,6 +58,16 @@ func (c *SlashingViolationsConsumer) OnInvalidVoteDetected(err model.InvalidVote Msg("OnInvalidVoteDetected") } +func (c *SlashingViolationsConsumer) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { + c.log.Warn(). + Bool(logging.KeySuspicious, true). + Hex("timeout_creator", timeout.SignerID[:]). + Uint64("timeout_view", timeout.View). + Hex("timeout_id1", logging.ID(timeout.ID())). + Hex("timeout_id2", logging.ID(altTimeout.ID())). + Msg("OnDoubleTimeoutDetected") +} + func (c *SlashingViolationsConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { timeout := err.Timeout c.log.Warn(). diff --git a/consensus/hotstuff/notifications/telemetry.go b/consensus/hotstuff/notifications/telemetry.go index 67f0ca1339a..7bbf57f79de 100644 --- a/consensus/hotstuff/notifications/telemetry.go +++ b/consensus/hotstuff/notifications/telemetry.go @@ -32,12 +32,15 @@ import ( // // Telemetry does NOT capture slashing notifications type TelemetryConsumer struct { - NoopConsumer + NoopTimeoutCollectorConsumer + NoopVoteCollectorConsumer pathHandler *PathHandler noPathLogger zerolog.Logger } -var _ hotstuff.Consumer = (*TelemetryConsumer)(nil) +var _ hotstuff.ParticipantConsumer = (*TelemetryConsumer)(nil) +var _ hotstuff.VoteCollectorConsumer = (*TelemetryConsumer)(nil) +var _ hotstuff.TimeoutCollectorConsumer = (*TelemetryConsumer)(nil) // NewTelemetryConsumer creates consumer that reports telemetry events using logger backend. // Logger MUST include `chain` parameter as part of log context with corresponding chain ID to correctly map telemetry events to chain. @@ -240,6 +243,13 @@ func (t *TelemetryConsumer) OnCurrentViewDetails(currentView, finalizedView uint Msg("OnCurrentViewDetails") } +func (t *TelemetryConsumer) OnViewChange(oldView, newView uint64) { + t.pathHandler.NextStep(). + Uint64("old_view", oldView). + Uint64("new_view", newView). + Msg("OnViewChange") +} + // PathHandler maintains a notion of the current path through the state machine. // It allows to close a path and open new path. Each path is identified by a unique // (randomly generated) uuid. Along each path, we can capture information about relevant diff --git a/consensus/hotstuff/pacemaker.go b/consensus/hotstuff/pacemaker.go index 66b8787b241..90020b58d1c 100644 --- a/consensus/hotstuff/pacemaker.go +++ b/consensus/hotstuff/pacemaker.go @@ -50,6 +50,7 @@ type LivenessData struct { // // Not concurrency safe. type PaceMaker interface { + ProposalDurationProvider // CurView returns the current view. CurView() uint64 @@ -81,8 +82,20 @@ type PaceMaker interface { // be executed by the same goroutine that also calls the other business logic // methods, or concurrency safety has to be implemented externally. Start(ctx context.Context) +} - // BlockRateDelay returns the minimal wait time for broadcasting a proposal, measured from - // the point in time when the primary (locally) enters the respective view. - BlockRateDelay() time.Duration +// ProposalDurationProvider generates the target publication time for block proposals. +type ProposalDurationProvider interface { + // TargetPublicationTime is intended to be called by the EventHandler, whenever it + // wants to publish a new proposal. The event handler inputs + // - proposalView: the view it is proposing for, + // - timeViewEntered: the time when the EventHandler entered this view + // - parentBlockId: the ID of the parent block , which the EventHandler is building on + // TargetPublicationTime returns the time stamp when the new proposal should be broadcasted. + // For a given view where we are the primary, suppose the actual time we are done building our proposal is P: + // - if P < TargetPublicationTime(..), then the EventHandler should wait until + // `TargetPublicationTime` to broadcast the proposal + // - if P >= TargetPublicationTime(..), then the EventHandler should immediately broadcast the proposal + // Concurrency safe. + TargetPublicationTime(proposalView uint64, timeViewEntered time.Time, parentBlockId flow.Identifier) time.Time } diff --git a/consensus/hotstuff/pacemaker/pacemaker.go b/consensus/hotstuff/pacemaker/pacemaker.go index 1e1959eeb60..fc3ba87dbe3 100644 --- a/consensus/hotstuff/pacemaker/pacemaker.go +++ b/consensus/hotstuff/pacemaker/pacemaker.go @@ -27,6 +27,8 @@ import ( // // Not concurrency safe. type ActivePaceMaker struct { + hotstuff.ProposalDurationProvider + ctx context.Context timeoutControl *timeout.Controller notifier hotstuff.Consumer @@ -35,6 +37,7 @@ type ActivePaceMaker struct { } var _ hotstuff.PaceMaker = (*ActivePaceMaker)(nil) +var _ hotstuff.ProposalDurationProvider = (*ActivePaceMaker)(nil) // New creates a new ActivePaceMaker instance // - startView is the view for the pacemaker to start with. @@ -45,6 +48,7 @@ var _ hotstuff.PaceMaker = (*ActivePaceMaker)(nil) // * model.ConfigurationError if initial LivenessData is invalid func New( timeoutController *timeout.Controller, + proposalDurationProvider hotstuff.ProposalDurationProvider, notifier hotstuff.Consumer, persist hotstuff.Persister, recovery ...recoveryInformation, @@ -55,10 +59,11 @@ func New( } pm := &ActivePaceMaker{ - timeoutControl: timeoutController, - notifier: notifier, - viewTracker: vt, - started: false, + ProposalDurationProvider: proposalDurationProvider, + timeoutControl: timeoutController, + notifier: notifier, + viewTracker: vt, + started: false, } for _, recoveryAction := range recovery { err = recoveryAction(pm) @@ -85,9 +90,6 @@ func (p *ActivePaceMaker) LastViewTC() *flow.TimeoutCertificate { return p.viewT // To get the timeout for the next timeout, you need to call TimeoutChannel() again. func (p *ActivePaceMaker) TimeoutChannel() <-chan time.Time { return p.timeoutControl.Channel() } -// BlockRateDelay returns the delay for broadcasting its own proposals. -func (p *ActivePaceMaker) BlockRateDelay() time.Duration { return p.timeoutControl.BlockRateDelay() } - // ProcessQC notifies the pacemaker with a new QC, which might allow pacemaker to // fast-forward its view. In contrast to `ProcessTC`, this function does _not_ handle `nil` inputs. // No errors are expected, any error should be treated as exception @@ -171,7 +173,7 @@ type recoveryInformation func(p *ActivePaceMaker) error // WithQCs informs the PaceMaker about the given QCs. Old and nil QCs are accepted (no-op). func WithQCs(qcs ...*flow.QuorumCertificate) recoveryInformation { - // To avoid excessive data base writes during initialization, we pre-filter the newest QC + // To avoid excessive database writes during initialization, we pre-filter the newest QC // here and only hand that one to the viewTracker. For recovery, we allow the special case // of nil QCs, because the genesis block has no QC. tracker := tracker.NewNewestQCTracker() diff --git a/consensus/hotstuff/pacemaker/pacemaker_test.go b/consensus/hotstuff/pacemaker/pacemaker_test.go index 58193e0bd50..7db14618460 100644 --- a/consensus/hotstuff/pacemaker/pacemaker_test.go +++ b/consensus/hotstuff/pacemaker/pacemaker_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -17,6 +18,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" ) const ( @@ -44,11 +46,12 @@ type ActivePaceMakerTestSuite struct { initialQC *flow.QuorumCertificate initialTC *flow.TimeoutCertificate - notifier *mocks.Consumer - persist *mocks.Persister - paceMaker *ActivePaceMaker - stop context.CancelFunc - timeoutConf timeout.Config + notifier *mocks.Consumer + proposalDurationProvider hotstuff.ProposalDurationProvider + persist *mocks.Persister + paceMaker *ActivePaceMaker + stop context.CancelFunc + timeoutConf timeout.Config } func (s *ActivePaceMakerTestSuite) SetupTest() { @@ -57,13 +60,7 @@ func (s *ActivePaceMakerTestSuite) SetupTest() { s.initialTC = nil var err error - s.timeoutConf, err = timeout.NewConfig( - time.Duration(minRepTimeout*1e6), - time.Duration(maxRepTimeout*1e6), - multiplicativeIncrease, - happyPathMaxRoundFailures, - 0, - time.Duration(maxRepTimeout*1e6)) + s.timeoutConf, err = timeout.NewConfig(time.Duration(minRepTimeout*1e6), time.Duration(maxRepTimeout*1e6), multiplicativeIncrease, happyPathMaxRoundFailures, time.Duration(maxRepTimeout*1e6)) require.NoError(s.T(), err) // init consumer for notifications emitted by PaceMaker @@ -82,7 +79,7 @@ func (s *ActivePaceMakerTestSuite) SetupTest() { s.persist.On("GetLivenessData").Return(livenessData, nil) // init PaceMaker and start - s.paceMaker, err = New(timeout.NewController(s.timeoutConf), s.notifier, s.persist) + s.paceMaker, err = New(timeout.NewController(s.timeoutConf), NoProposalDelay(), s.notifier, s.persist) require.NoError(s.T(), err) var ctx context.Context @@ -347,7 +344,7 @@ func (s *ActivePaceMakerTestSuite) Test_Initialization() { // test that the constructor finds the newest QC and TC s.Run("Random TCs and QCs combined", func() { pm, err := New( - timeout.NewController(s.timeoutConf), s.notifier, s.persist, + timeout.NewController(s.timeoutConf), NoProposalDelay(), s.notifier, s.persist, WithQCs(qcs...), WithTCs(tcs...), ) require.NoError(s.T(), err) @@ -367,7 +364,7 @@ func (s *ActivePaceMakerTestSuite) Test_Initialization() { tcs[45] = helper.MakeTC(helper.WithTCView(highestView+15), helper.WithTCNewestQC(QC(highestView+12))) pm, err := New( - timeout.NewController(s.timeoutConf), s.notifier, s.persist, + timeout.NewController(s.timeoutConf), NoProposalDelay(), s.notifier, s.persist, WithTCs(tcs...), WithQCs(qcs...), ) require.NoError(s.T(), err) @@ -387,7 +384,7 @@ func (s *ActivePaceMakerTestSuite) Test_Initialization() { tcs[45] = helper.MakeTC(helper.WithTCView(highestView+15), helper.WithTCNewestQC(QC(highestView+15))) pm, err := New( - timeout.NewController(s.timeoutConf), s.notifier, s.persist, + timeout.NewController(s.timeoutConf), NoProposalDelay(), s.notifier, s.persist, WithTCs(tcs...), WithQCs(qcs...), ) require.NoError(s.T(), err) @@ -403,11 +400,11 @@ func (s *ActivePaceMakerTestSuite) Test_Initialization() { // Verify that WithTCs still works correctly if no TCs are given: // the list of TCs is empty or all contained TCs are nil s.Run("Only nil TCs", func() { - pm, err := New(timeout.NewController(s.timeoutConf), s.notifier, s.persist, WithTCs()) + pm, err := New(timeout.NewController(s.timeoutConf), NoProposalDelay(), s.notifier, s.persist, WithTCs()) require.NoError(s.T(), err) require.Equal(s.T(), s.initialView, pm.CurView()) - pm, err = New(timeout.NewController(s.timeoutConf), s.notifier, s.persist, WithTCs(nil, nil, nil)) + pm, err = New(timeout.NewController(s.timeoutConf), NoProposalDelay(), s.notifier, s.persist, WithTCs(nil, nil, nil)) require.NoError(s.T(), err) require.Equal(s.T(), s.initialView, pm.CurView()) }) @@ -415,17 +412,29 @@ func (s *ActivePaceMakerTestSuite) Test_Initialization() { // Verify that WithQCs still works correctly if no QCs are given: // the list of QCs is empty or all contained QCs are nil s.Run("Only nil QCs", func() { - pm, err := New(timeout.NewController(s.timeoutConf), s.notifier, s.persist, WithQCs()) + pm, err := New(timeout.NewController(s.timeoutConf), NoProposalDelay(), s.notifier, s.persist, WithQCs()) require.NoError(s.T(), err) require.Equal(s.T(), s.initialView, pm.CurView()) - pm, err = New(timeout.NewController(s.timeoutConf), s.notifier, s.persist, WithQCs(nil, nil, nil)) + pm, err = New(timeout.NewController(s.timeoutConf), NoProposalDelay(), s.notifier, s.persist, WithQCs(nil, nil, nil)) require.NoError(s.T(), err) require.Equal(s.T(), s.initialView, pm.CurView()) }) } +// TestProposalDuration tests that the active pacemaker forwards proposal duration values from the provider. +func (s *ActivePaceMakerTestSuite) TestProposalDuration() { + proposalDurationProvider := NewStaticProposalDurationProvider(time.Millisecond * 500) + pm, err := New(timeout.NewController(s.timeoutConf), &proposalDurationProvider, s.notifier, s.persist) + require.NoError(s.T(), err) + + now := time.Now().UTC() + assert.Equal(s.T(), now.Add(time.Millisecond*500), pm.TargetPublicationTime(117, now, unittest.IdentifierFixture())) + proposalDurationProvider.dur = time.Second + assert.Equal(s.T(), now.Add(time.Second), pm.TargetPublicationTime(117, now, unittest.IdentifierFixture())) +} + func max(a uint64, values ...uint64) uint64 { for _, v := range values { if v > a { diff --git a/consensus/hotstuff/pacemaker/proposal_timing.go b/consensus/hotstuff/pacemaker/proposal_timing.go new file mode 100644 index 00000000000..7530b2aedcb --- /dev/null +++ b/consensus/hotstuff/pacemaker/proposal_timing.go @@ -0,0 +1,29 @@ +package pacemaker + +import ( + "time" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/model/flow" +) + +// StaticProposalDurationProvider is a hotstuff.ProposalDurationProvider which provides a static ProposalDuration. +// The constant dur represents the time to produce and broadcast the proposal (ProposalDuration), +// NOT the time for the entire view (ViewDuration). +type StaticProposalDurationProvider struct { + dur time.Duration +} + +var _ hotstuff.ProposalDurationProvider = (*StaticProposalDurationProvider)(nil) + +func NewStaticProposalDurationProvider(dur time.Duration) StaticProposalDurationProvider { + return StaticProposalDurationProvider{dur: dur} +} + +func (p StaticProposalDurationProvider) TargetPublicationTime(_ uint64, timeViewEntered time.Time, _ flow.Identifier) time.Time { + return timeViewEntered.Add(p.dur) +} + +func NoProposalDelay() StaticProposalDurationProvider { + return NewStaticProposalDurationProvider(0) +} diff --git a/consensus/hotstuff/pacemaker/timeout/config.go b/consensus/hotstuff/pacemaker/timeout/config.go index 6de384f92d3..a10f65b68a5 100644 --- a/consensus/hotstuff/pacemaker/timeout/config.go +++ b/consensus/hotstuff/pacemaker/timeout/config.go @@ -1,14 +1,9 @@ package timeout import ( - "fmt" "time" - "github.com/rs/zerolog/log" - "go.uber.org/atomic" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/module/updatable_configs" ) // Config contains the configuration parameters for a Truncated Exponential Backoff, @@ -16,6 +11,9 @@ import ( // - On timeout: increase timeout by multiplicative factor `TimeoutAdjustmentFactor`. This // results in exponentially growing timeout duration on multiple subsequent timeouts. // - On progress: decrease timeout by multiplicative factor `TimeoutAdjustmentFactor. +// +// Config is implemented such that it can be passed by value, while still supporting updates of +// `BlockRateDelayMS` at runtime (all configs share the same memory holding `BlockRateDelayMS`). type Config struct { // MinReplicaTimeout is the minimum the timeout can decrease to [MILLISECONDS] MinReplicaTimeout float64 @@ -27,8 +25,6 @@ type Config struct { // HappyPathMaxRoundFailures is the number of rounds without progress where we still consider being // on hot path of execution. After exceeding this value we will start increasing timeout values. HappyPathMaxRoundFailures uint64 - // BlockRateDelayMS is a delay to broadcast the proposal in order to control block production rate [MILLISECONDS] - BlockRateDelayMS *atomic.Float64 // MaxTimeoutObjectRebroadcastInterval is the maximum value for timeout object rebroadcast interval [MILLISECONDS] MaxTimeoutObjectRebroadcastInterval float64 } @@ -51,14 +47,7 @@ func NewDefaultConfig() Config { blockRateDelay := 0 * time.Millisecond maxRebroadcastInterval := 5 * time.Second - conf, err := NewConfig( - minReplicaTimeout+blockRateDelay, - maxReplicaTimeout, - timeoutAdjustmentFactorFactor, - happyPathMaxRoundFailures, - blockRateDelay, - maxRebroadcastInterval, - ) + conf, err := NewConfig(minReplicaTimeout+blockRateDelay, maxReplicaTimeout, timeoutAdjustmentFactorFactor, happyPathMaxRoundFailures, maxRebroadcastInterval) if err != nil { // we check in a unit test that this does not happen panic("Default config is not compliant with timeout Config requirements") @@ -79,14 +68,7 @@ func NewDefaultConfig() Config { // Consistency requirement: must be non-negative // // Returns `model.ConfigurationError` is any of the consistency requirements is violated. -func NewConfig( - minReplicaTimeout time.Duration, - maxReplicaTimeout time.Duration, - timeoutAdjustmentFactor float64, - happyPathMaxRoundFailures uint64, - blockRateDelay time.Duration, - maxRebroadcastInterval time.Duration, -) (Config, error) { +func NewConfig(minReplicaTimeout time.Duration, maxReplicaTimeout time.Duration, timeoutAdjustmentFactor float64, happyPathMaxRoundFailures uint64, maxRebroadcastInterval time.Duration) (Config, error) { if minReplicaTimeout <= 0 { return Config{}, model.NewConfigurationErrorf("minReplicaTimeout must be a positive number[milliseconds]") } @@ -96,9 +78,6 @@ func NewConfig( if timeoutAdjustmentFactor <= 1 { return Config{}, model.NewConfigurationErrorf("timeoutAdjustmentFactor must be strictly bigger than 1") } - if err := validBlockRateDelay(blockRateDelay); err != nil { - return Config{}, err - } if maxRebroadcastInterval <= 0 { return Config{}, model.NewConfigurationErrorf("maxRebroadcastInterval must be a positive number [milliseconds]") } @@ -109,43 +88,6 @@ func NewConfig( TimeoutAdjustmentFactor: timeoutAdjustmentFactor, HappyPathMaxRoundFailures: happyPathMaxRoundFailures, MaxTimeoutObjectRebroadcastInterval: float64(maxRebroadcastInterval.Milliseconds()), - BlockRateDelayMS: atomic.NewFloat64(float64(blockRateDelay.Milliseconds())), } return tc, nil } - -// validBlockRateDelay validates a block rate delay config. -// Returns model.ConfigurationError for invalid config inputs. -func validBlockRateDelay(blockRateDelay time.Duration) error { - if blockRateDelay < 0 { - return model.NewConfigurationErrorf("blockRateDelay must be must be non-negative") - } - return nil -} - -// GetBlockRateDelay returns the block rate delay as a Duration. This is used by -// the dyamic config manager. -func (c *Config) GetBlockRateDelay() time.Duration { - ms := c.BlockRateDelayMS.Load() - return time.Millisecond * time.Duration(ms) -} - -// SetBlockRateDelay sets the block rate delay. It is used to modify this config -// value while HotStuff is running. -// Returns updatable_configs.ValidationError if the new value is invalid. -func (c *Config) SetBlockRateDelay(delay time.Duration) error { - if err := validBlockRateDelay(delay); err != nil { - if model.IsConfigurationError(err) { - return updatable_configs.NewValidationErrorf("invalid block rate delay: %w", err) - } - return fmt.Errorf("unexpected error validating block rate delay: %w", err) - } - // sanity check: log a warning if we set block rate delay above min timeout - // it is valid to want to do this, to significantly slow the block rate, but - // only in edge cases - if c.MinReplicaTimeout < float64(delay.Milliseconds()) { - log.Warn().Msgf("CAUTION: setting block rate delay to %s, above min timeout %dms - this will degrade performance!", delay.String(), int64(c.MinReplicaTimeout)) - } - c.BlockRateDelayMS.Store(float64(delay.Milliseconds())) - return nil -} diff --git a/consensus/hotstuff/pacemaker/timeout/config_test.go b/consensus/hotstuff/pacemaker/timeout/config_test.go index 259b87727ed..fe758dbd70d 100644 --- a/consensus/hotstuff/pacemaker/timeout/config_test.go +++ b/consensus/hotstuff/pacemaker/timeout/config_test.go @@ -11,37 +11,34 @@ import ( // TestConstructor tests that constructor performs needed checks and returns expected values depending on different inputs. func TestConstructor(t *testing.T) { - c, err := NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, time.Second, 2000*time.Millisecond) + c, err := NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, 2000*time.Millisecond) require.NoError(t, err) require.Equal(t, float64(1200), c.MinReplicaTimeout) require.Equal(t, float64(2000), c.MaxReplicaTimeout) require.Equal(t, float64(1.5), c.TimeoutAdjustmentFactor) require.Equal(t, uint64(3), c.HappyPathMaxRoundFailures) - require.Equal(t, float64(1000), c.BlockRateDelayMS.Load()) require.Equal(t, float64(2000), c.MaxTimeoutObjectRebroadcastInterval) // should not allow negative minReplicaTimeout - c, err = NewConfig(-1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, time.Second, 2000*time.Millisecond) + c, err = NewConfig(-1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, 2000*time.Millisecond) require.True(t, model.IsConfigurationError(err)) // should not allow 0 minReplicaTimeout - c, err = NewConfig(0, 2000*time.Millisecond, 1.5, 3, time.Second, 2000*time.Millisecond) + c, err = NewConfig(0, 2000*time.Millisecond, 1.5, 3, 2000*time.Millisecond) require.True(t, model.IsConfigurationError(err)) // should not allow maxReplicaTimeout < minReplicaTimeout - c, err = NewConfig(1200*time.Millisecond, 1000*time.Millisecond, 1.5, 3, time.Second, 2000*time.Millisecond) + c, err = NewConfig(1200*time.Millisecond, 1000*time.Millisecond, 1.5, 3, 2000*time.Millisecond) require.True(t, model.IsConfigurationError(err)) // should not allow timeoutIncrease to be 1.0 or smaller - c, err = NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.0, 3, time.Second, 2000*time.Millisecond) + c, err = NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.0, 3, 2000*time.Millisecond) require.True(t, model.IsConfigurationError(err)) - // should not allow blockRateDelay to be zero negative - c, err = NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, -1*time.Nanosecond, 2000*time.Millisecond) + // should accept only positive values for maxRebroadcastInterval + c, err = NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, 0) require.True(t, model.IsConfigurationError(err)) - - // should not allow maxRebroadcastInterval to be smaller than minReplicaTimeout - c, err = NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, -1*time.Nanosecond, 1000*time.Millisecond) + c, err = NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, -1000*time.Millisecond) require.True(t, model.IsConfigurationError(err)) } @@ -52,5 +49,4 @@ func TestDefaultConfig(t *testing.T) { require.Equal(t, float64(3000), c.MinReplicaTimeout) require.Equal(t, 1.2, c.TimeoutAdjustmentFactor) require.Equal(t, uint64(6), c.HappyPathMaxRoundFailures) - require.Equal(t, float64(0), c.BlockRateDelayMS.Load()) } diff --git a/consensus/hotstuff/pacemaker/timeout/controller.go b/consensus/hotstuff/pacemaker/timeout/controller.go index 55c73137134..1b09cf8debf 100644 --- a/consensus/hotstuff/pacemaker/timeout/controller.go +++ b/consensus/hotstuff/pacemaker/timeout/controller.go @@ -38,7 +38,9 @@ type Controller struct { r uint64 // failed rounds counter, higher value results in longer round duration } -// NewController creates a new Controller. +// NewController creates a new Controller. Note that the input Config is implemented such that +// it can be passed by value, while still supporting updates of `BlockRateDelayMS` at runtime +// (all configs share the same memory holding `BlockRateDelayMS`). func NewController(timeoutConfig Config) *Controller { // the initial value for the timeout channel is a closed channel which returns immediately // this prevents indefinite blocking when no timeout has been started @@ -145,8 +147,3 @@ func (t *Controller) OnProgressBeforeTimeout() { t.r-- } } - -// BlockRateDelay is a delay to broadcast the proposal in order to control block production rate -func (t *Controller) BlockRateDelay() time.Duration { - return time.Duration(t.cfg.BlockRateDelayMS.Load() * float64(time.Millisecond)) -} diff --git a/consensus/hotstuff/pacemaker/timeout/controller_test.go b/consensus/hotstuff/pacemaker/timeout/controller_test.go index beb31f4eea9..be2b367f774 100644 --- a/consensus/hotstuff/pacemaker/timeout/controller_test.go +++ b/consensus/hotstuff/pacemaker/timeout/controller_test.go @@ -17,13 +17,7 @@ const ( ) func initTimeoutController(t *testing.T) *Controller { - tc, err := NewConfig( - time.Duration(minRepTimeout*1e6), - time.Duration(maxRepTimeout*1e6), - timeoutAdjustmentFactor, - happyPathMaxRoundFailures, - 0, - time.Duration(maxRepTimeout*1e6)) + tc, err := NewConfig(time.Duration(minRepTimeout*1e6), time.Duration(maxRepTimeout*1e6), timeoutAdjustmentFactor, happyPathMaxRoundFailures, time.Duration(maxRepTimeout*1e6)) if err != nil { t.Fail() } @@ -149,20 +143,3 @@ func Test_CombinedIncreaseDecreaseDynamics(t *testing.T) { testDynamicSequence([]bool{increase, decrease, increase, decrease, increase, decrease}) testDynamicSequence([]bool{increase, increase, increase, increase, increase, decrease}) } - -// Test_BlockRateDelay check that correct block rate delay is returned -func Test_BlockRateDelay(t *testing.T) { - - c, err := NewConfig( - time.Duration(minRepTimeout*float64(time.Millisecond)), - time.Duration(maxRepTimeout*float64(time.Millisecond)), - timeoutAdjustmentFactor, - happyPathMaxRoundFailures, - time.Second, - time.Duration(maxRepTimeout*float64(time.Millisecond))) - if err != nil { - t.Fail() - } - tc := NewController(c) - assert.Equal(t, time.Second, tc.BlockRateDelay()) -} diff --git a/consensus/hotstuff/signature/block_signer_decoder_test.go b/consensus/hotstuff/signature/block_signer_decoder_test.go index 78efb3005eb..c065e315add 100644 --- a/consensus/hotstuff/signature/block_signer_decoder_test.go +++ b/consensus/hotstuff/signature/block_signer_decoder_test.go @@ -140,7 +140,8 @@ func (s *blockSignerDecoderSuite) Test_EpochTransition() { blockView := s.block.Header.View parentView := s.block.Header.ParentView epoch1Committee := s.allConsensus - epoch2Committee := s.allConsensus.SamplePct(.8) + epoch2Committee, err := s.allConsensus.SamplePct(.8) + require.NoError(s.T(), err) *s.committee = *hotstuff.NewDynamicCommittee(s.T()) s.committee.On("IdentitiesByEpoch", parentView).Return(epoch1Committee, nil).Maybe() diff --git a/consensus/hotstuff/signature/randombeacon_inspector_test.go b/consensus/hotstuff/signature/randombeacon_inspector_test.go index 5784577f668..5df5b897289 100644 --- a/consensus/hotstuff/signature/randombeacon_inspector_test.go +++ b/consensus/hotstuff/signature/randombeacon_inspector_test.go @@ -2,10 +2,9 @@ package signature import ( "errors" - mrand "math/rand" + "math/rand" "sync" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,6 +23,7 @@ func TestRandomBeaconInspector(t *testing.T) { type randomBeaconSuite struct { suite.Suite + rng *rand.Rand n int threshold int kmac hash.Hasher @@ -39,9 +39,9 @@ func (rs *randomBeaconSuite) SetupTest() { rs.threshold = signature.RandomBeaconThreshold(rs.n) // generate threshold keys - mrand.Seed(time.Now().UnixNano()) + rs.rng = unittest.GetPRG(rs.T()) seed := make([]byte, crypto.SeedMinLenDKG) - _, err := mrand.Read(seed) + _, err := rs.rng.Read(seed) require.NoError(rs.T(), err) rs.skShares, rs.pkShares, rs.pkGroup, err = crypto.BLSThresholdKeyGen(rs.n, rs.threshold, seed) require.NoError(rs.T(), err) @@ -57,7 +57,7 @@ func (rs *randomBeaconSuite) SetupTest() { for i := 0; i < rs.n; i++ { rs.signers = append(rs.signers, i) } - mrand.Shuffle(rs.n, func(i, j int) { + rs.rng.Shuffle(rs.n, func(i, j int) { rs.signers[i], rs.signers[j] = rs.signers[j], rs.signers[i] }) } @@ -166,7 +166,7 @@ func (rs *randomBeaconSuite) TestInvalidSignerIndex() { func (rs *randomBeaconSuite) TestInvalidSignature() { follower, err := NewRandomBeaconInspector(rs.pkGroup, rs.pkShares, rs.threshold, rs.thresholdSignatureMessage) require.NoError(rs.T(), err) - index := mrand.Intn(rs.n) // random signer + index := rs.rng.Intn(rs.n) // random signer share, err := rs.skShares[index].Sign(rs.thresholdSignatureMessage, rs.kmac) require.NoError(rs.T(), err) diff --git a/consensus/hotstuff/signature/randombeacon_signer_store_test.go b/consensus/hotstuff/signature/randombeacon_signer_store_test.go index 87ceeb0a7fe..c578e1b2e97 100644 --- a/consensus/hotstuff/signature/randombeacon_signer_store_test.go +++ b/consensus/hotstuff/signature/randombeacon_signer_store_test.go @@ -4,7 +4,6 @@ import ( "errors" "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,7 +30,6 @@ func TestBeaconKeyStore(t *testing.T) { } func (suite *BeaconKeyStore) SetupTest() { - rand.Seed(time.Now().Unix()) suite.epochLookup = mockmodule.NewEpochLookup(suite.T()) suite.beaconKeys = mockstorage.NewSafeBeaconKeys(suite.T()) suite.store = NewEpochAwareRandomBeaconKeyStore(suite.epochLookup, suite.beaconKeys) diff --git a/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go b/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go index ae308c42048..f4125d96080 100644 --- a/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go +++ b/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go @@ -13,9 +13,9 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/notifications" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/common/fifoqueue" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/counters" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/metrics" @@ -35,7 +35,6 @@ type TimeoutAggregator struct { log zerolog.Logger hotstuffMetrics module.HotstuffMetrics engineMetrics module.EngineMetrics - notifier hotstuff.Consumer lowestRetainedView counters.StrictMonotonousCounter // lowest view, for which we still process timeouts collectors hotstuff.TimeoutCollectors queuedTimeoutsNotifier engine.Notifier @@ -52,7 +51,6 @@ func NewTimeoutAggregator(log zerolog.Logger, hotstuffMetrics module.HotstuffMetrics, engineMetrics module.EngineMetrics, mempoolMetrics module.MempoolMetrics, - notifier hotstuff.Consumer, lowestRetainedView uint64, collectors hotstuff.TimeoutCollectors, ) (*TimeoutAggregator, error) { @@ -66,7 +64,6 @@ func NewTimeoutAggregator(log zerolog.Logger, log: log.With().Str("component", "hotstuff.timeout_aggregator").Logger(), hotstuffMetrics: hotstuffMetrics, engineMetrics: engineMetrics, - notifier: notifier, lowestRetainedView: counters.NewMonotonousCounter(lowestRetainedView), collectors: collectors, queuedTimeoutsNotifier: engine.NewNotifier(), diff --git a/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go b/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go index fddce6bd717..e8fd19b1bb8 100644 --- a/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go +++ b/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go @@ -33,14 +33,12 @@ type TimeoutAggregatorTestSuite struct { highestKnownView uint64 aggregator *TimeoutAggregator collectors *mocks.TimeoutCollectors - consumer *mocks.Consumer stopAggregator context.CancelFunc } func (s *TimeoutAggregatorTestSuite) SetupTest() { var err error s.collectors = mocks.NewTimeoutCollectors(s.T()) - s.consumer = mocks.NewConsumer(s.T()) s.lowestRetainedView = 100 @@ -51,7 +49,6 @@ func (s *TimeoutAggregatorTestSuite) SetupTest() { metricsCollector, metricsCollector, metricsCollector, - s.consumer, s.lowestRetainedView, s.collectors, ) diff --git a/consensus/hotstuff/timeoutaggregator/timeout_collectors.go b/consensus/hotstuff/timeoutaggregator/timeout_collectors.go index 20369bc9485..31e83d10b21 100644 --- a/consensus/hotstuff/timeoutaggregator/timeout_collectors.go +++ b/consensus/hotstuff/timeoutaggregator/timeout_collectors.go @@ -7,6 +7,7 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/mempool" ) @@ -15,23 +16,26 @@ import ( // particular view is lazy (instances are created on demand). // This structure is concurrently safe. // TODO: once VoteCollectors gets updated to stop managing worker pool we can merge VoteCollectors and TimeoutCollectors using generics -// TODO(active-pacemaker): add metrics for tracking size of collectors and active range type TimeoutCollectors struct { - log zerolog.Logger - lock sync.RWMutex - lowestRetainedView uint64 // lowest view, for which we still retain a TimeoutCollector and process timeouts - collectors map[uint64]hotstuff.TimeoutCollector // view -> TimeoutCollector - collectorFactory hotstuff.TimeoutCollectorFactory // factor for creating collectors + log zerolog.Logger + metrics module.HotstuffMetrics + lock sync.RWMutex + lowestRetainedView uint64 // lowest view, for which we still retain a TimeoutCollector and process timeouts + newestViewCachedCollector uint64 // highest view, for which we have created a TimeoutCollector + collectors map[uint64]hotstuff.TimeoutCollector // view -> TimeoutCollector + collectorFactory hotstuff.TimeoutCollectorFactory // factor for creating collectors } var _ hotstuff.TimeoutCollectors = (*TimeoutCollectors)(nil) -func NewTimeoutCollectors(log zerolog.Logger, lowestRetainedView uint64, collectorFactory hotstuff.TimeoutCollectorFactory) *TimeoutCollectors { +func NewTimeoutCollectors(log zerolog.Logger, metrics module.HotstuffMetrics, lowestRetainedView uint64, collectorFactory hotstuff.TimeoutCollectorFactory) *TimeoutCollectors { return &TimeoutCollectors{ - log: log.With().Str("component", "timeout_collectors").Logger(), - lowestRetainedView: lowestRetainedView, - collectors: make(map[uint64]hotstuff.TimeoutCollector), - collectorFactory: collectorFactory, + log: log.With().Str("component", "timeout_collectors").Logger(), + metrics: metrics, + lowestRetainedView: lowestRetainedView, + newestViewCachedCollector: lowestRetainedView, + collectors: make(map[uint64]hotstuff.TimeoutCollector), + collectorFactory: collectorFactory, } } @@ -70,8 +74,15 @@ func (t *TimeoutCollectors) GetOrCreateCollector(view uint64) (hotstuff.TimeoutC return clr, false, nil } t.collectors[view] = collector + if t.newestViewCachedCollector < view { + t.newestViewCachedCollector = view + } + lowestRetainedView := t.lowestRetainedView + numCollectors := len(t.collectors) + newestViewCachedCollector := t.newestViewCachedCollector t.lock.Unlock() + t.metrics.TimeoutCollectorsRange(lowestRetainedView, newestViewCachedCollector, numCollectors) t.log.Info().Uint64("view", view).Msg("timeout collector has been created") return collector, true, nil } @@ -97,13 +108,14 @@ func (t *TimeoutCollectors) getCollector(view uint64) (hotstuff.TimeoutCollector // kept and the method call is a NoOp. func (t *TimeoutCollectors) PruneUpToView(lowestRetainedView uint64) { t.lock.Lock() - defer t.lock.Unlock() if t.lowestRetainedView >= lowestRetainedView { + t.lock.Unlock() return } sizeBefore := len(t.collectors) if sizeBefore == 0 { t.lowestRetainedView = lowestRetainedView + t.lock.Unlock() return } @@ -124,11 +136,15 @@ func (t *TimeoutCollectors) PruneUpToView(lowestRetainedView uint64) { } from := t.lowestRetainedView t.lowestRetainedView = lowestRetainedView + numCollectors := len(t.collectors) + newestViewCachedCollector := t.newestViewCachedCollector + t.lock.Unlock() + t.metrics.TimeoutCollectorsRange(lowestRetainedView, newestViewCachedCollector, numCollectors) t.log.Debug(). Uint64("prior_lowest_retained_view", from). Uint64("lowest_retained_view", lowestRetainedView). Int("prior_size", sizeBefore). - Int("size", len(t.collectors)). + Int("size", numCollectors). Msgf("pruned timeout collectors") } diff --git a/consensus/hotstuff/timeoutaggregator/timeout_collectors_test.go b/consensus/hotstuff/timeoutaggregator/timeout_collectors_test.go index 66252c6e065..ef19cfce01d 100644 --- a/consensus/hotstuff/timeoutaggregator/timeout_collectors_test.go +++ b/consensus/hotstuff/timeoutaggregator/timeout_collectors_test.go @@ -17,6 +17,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/module/mempool" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/utils/unittest" ) @@ -54,7 +55,7 @@ func (s *TimeoutCollectorsTestSuite) SetupTest() { } return fmt.Errorf("mocked collector %v not found: %w", view, factoryError) }).Maybe() - s.collectors = NewTimeoutCollectors(unittest.Logger(), s.lowestView, s.factoryMethod) + s.collectors = NewTimeoutCollectors(unittest.Logger(), metrics.NewNoopCollector(), s.lowestView, s.factoryMethod) } func (s *TimeoutCollectorsTestSuite) TearDownTest() { diff --git a/consensus/hotstuff/timeoutcollector/factory.go b/consensus/hotstuff/timeoutcollector/factory.go index e76c441d1ec..ba6c3fbc29f 100644 --- a/consensus/hotstuff/timeoutcollector/factory.go +++ b/consensus/hotstuff/timeoutcollector/factory.go @@ -11,10 +11,9 @@ import ( // TimeoutCollectorFactory implements hotstuff.TimeoutCollectorFactory, it is responsible for creating timeout collector // for given view. type TimeoutCollectorFactory struct { - log zerolog.Logger - notifier hotstuff.Consumer - collectorNotifier hotstuff.TimeoutCollectorConsumer - processorFactory hotstuff.TimeoutProcessorFactory + log zerolog.Logger + notifier hotstuff.TimeoutAggregationConsumer + processorFactory hotstuff.TimeoutProcessorFactory } var _ hotstuff.TimeoutCollectorFactory = (*TimeoutCollectorFactory)(nil) @@ -22,15 +21,13 @@ var _ hotstuff.TimeoutCollectorFactory = (*TimeoutCollectorFactory)(nil) // NewTimeoutCollectorFactory creates new instance of TimeoutCollectorFactory. // No error returns are expected during normal operations. func NewTimeoutCollectorFactory(log zerolog.Logger, - notifier hotstuff.Consumer, - collectorNotifier hotstuff.TimeoutCollectorConsumer, + notifier hotstuff.TimeoutAggregationConsumer, createProcessor hotstuff.TimeoutProcessorFactory, ) *TimeoutCollectorFactory { return &TimeoutCollectorFactory{ - log: log, - notifier: notifier, - collectorNotifier: collectorNotifier, - processorFactory: createProcessor, + log: log, + notifier: notifier, + processorFactory: createProcessor, } } @@ -44,7 +41,7 @@ func (f *TimeoutCollectorFactory) Create(view uint64) (hotstuff.TimeoutCollector if err != nil { return nil, fmt.Errorf("could not create TimeoutProcessor at view %d: %w", view, err) } - return NewTimeoutCollector(f.log, view, f.notifier, f.collectorNotifier, processor), nil + return NewTimeoutCollector(f.log, view, f.notifier, processor), nil } // TimeoutProcessorFactory implements hotstuff.TimeoutProcessorFactory, it is responsible for creating timeout processor diff --git a/consensus/hotstuff/timeoutcollector/timeout_collector.go b/consensus/hotstuff/timeoutcollector/timeout_collector.go index 28a9dc6f2d6..8b68aadb5cd 100644 --- a/consensus/hotstuff/timeoutcollector/timeout_collector.go +++ b/consensus/hotstuff/timeoutcollector/timeout_collector.go @@ -8,7 +8,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" + "github.com/onflow/flow-go/module/counters" ) // TimeoutCollector implements logic for collecting timeout objects. Performs deduplication, caching and processing @@ -16,13 +16,12 @@ import ( // their view is newer than any QC or TC previously known to the TimeoutCollector. // This module is safe to use in concurrent environment. type TimeoutCollector struct { - log zerolog.Logger - notifier hotstuff.Consumer - timeoutsCache *TimeoutObjectsCache // cache for tracking double timeout and timeout equivocation - collectorNotifier hotstuff.TimeoutCollectorConsumer - processor hotstuff.TimeoutProcessor - newestReportedQC counters.StrictMonotonousCounter // view of newest QC that was reported - newestReportedTC counters.StrictMonotonousCounter // view of newest TC that was reported + log zerolog.Logger + timeoutsCache *TimeoutObjectsCache // cache for tracking double timeout and timeout equivocation + notifier hotstuff.TimeoutAggregationConsumer + processor hotstuff.TimeoutProcessor + newestReportedQC counters.StrictMonotonousCounter // view of newest QC that was reported + newestReportedTC counters.StrictMonotonousCounter // view of newest TC that was reported } var _ hotstuff.TimeoutCollector = (*TimeoutCollector)(nil) @@ -30,8 +29,7 @@ var _ hotstuff.TimeoutCollector = (*TimeoutCollector)(nil) // NewTimeoutCollector creates new instance of TimeoutCollector func NewTimeoutCollector(log zerolog.Logger, view uint64, - notifier hotstuff.Consumer, - collectorNotifier hotstuff.TimeoutCollectorConsumer, + notifier hotstuff.TimeoutAggregationConsumer, processor hotstuff.TimeoutProcessor, ) *TimeoutCollector { return &TimeoutCollector{ @@ -39,12 +37,11 @@ func NewTimeoutCollector(log zerolog.Logger, Str("component", "hotstuff.timeout_collector"). Uint64("view", view). Logger(), - notifier: notifier, - timeoutsCache: NewTimeoutObjectsCache(view), - processor: processor, - collectorNotifier: collectorNotifier, - newestReportedQC: counters.NewMonotonousCounter(0), - newestReportedTC: counters.NewMonotonousCounter(0), + notifier: notifier, + timeoutsCache: NewTimeoutObjectsCache(view), + processor: processor, + newestReportedQC: counters.NewMonotonousCounter(0), + newestReportedTC: counters.NewMonotonousCounter(0), } } @@ -91,6 +88,7 @@ func (c *TimeoutCollector) processTimeout(timeout *model.TimeoutObject) error { return fmt.Errorf("internal error while processing timeout: %w", err) } + // TODO: consider moving OnTimeoutProcessed to TimeoutAggregationConsumer, need to fix telemetry for this. c.notifier.OnTimeoutProcessed(timeout) // In the following, we emit notifications about new QCs, if their view is newer than any QC previously @@ -112,12 +110,12 @@ func (c *TimeoutCollector) processTimeout(timeout *model.TimeoutObject) error { // system can only arrive earlier in our weakly ordered implementation. Hence, if anything, the recipient // receives the desired information _earlier_ but not later. if c.newestReportedQC.Set(timeout.NewestQC.View) { - c.collectorNotifier.OnNewQcDiscovered(timeout.NewestQC) + c.notifier.OnNewQcDiscovered(timeout.NewestQC) } // Same explanation for weak ordering of QCs also applies to TCs. if timeout.LastViewTC != nil { if c.newestReportedTC.Set(timeout.LastViewTC.View) { - c.collectorNotifier.OnNewTcDiscovered(timeout.LastViewTC) + c.notifier.OnNewTcDiscovered(timeout.LastViewTC) } } diff --git a/consensus/hotstuff/timeoutcollector/timeout_collector_test.go b/consensus/hotstuff/timeoutcollector/timeout_collector_test.go index 691209cb179..f30b953c1cf 100644 --- a/consensus/hotstuff/timeoutcollector/timeout_collector_test.go +++ b/consensus/hotstuff/timeoutcollector/timeout_collector_test.go @@ -27,23 +27,21 @@ func TestTimeoutCollector(t *testing.T) { type TimeoutCollectorTestSuite struct { suite.Suite - view uint64 - notifier *mocks.Consumer - collectorNotifier *mocks.TimeoutCollectorConsumer - processor *mocks.TimeoutProcessor - collector *TimeoutCollector + view uint64 + notifier *mocks.TimeoutAggregationConsumer + processor *mocks.TimeoutProcessor + collector *TimeoutCollector } func (s *TimeoutCollectorTestSuite) SetupTest() { s.view = 1000 - s.notifier = mocks.NewConsumer(s.T()) - s.collectorNotifier = mocks.NewTimeoutCollectorConsumer(s.T()) + s.notifier = mocks.NewTimeoutAggregationConsumer(s.T()) s.processor = mocks.NewTimeoutProcessor(s.T()) - s.collectorNotifier.On("OnNewQcDiscovered", mock.Anything).Maybe() - s.collectorNotifier.On("OnNewTcDiscovered", mock.Anything).Maybe() + s.notifier.On("OnNewQcDiscovered", mock.Anything).Maybe() + s.notifier.On("OnNewTcDiscovered", mock.Anything).Maybe() - s.collector = NewTimeoutCollector(unittest.Logger(), s.view, s.notifier, s.collectorNotifier, s.processor) + s.collector = NewTimeoutCollector(unittest.Logger(), s.view, s.notifier, s.processor) } // TestView tests that `View` returns the same value that was passed in constructor @@ -145,10 +143,10 @@ func (s *TimeoutCollectorTestSuite) TestAddTimeout_TONotifications() { s.T().Fatal("invalid test configuration") } - *s.collectorNotifier = *mocks.NewTimeoutCollectorConsumer(s.T()) + *s.notifier = *mocks.NewTimeoutAggregationConsumer(s.T()) var highestReportedQC *flow.QuorumCertificate - s.collectorNotifier.On("OnNewQcDiscovered", mock.Anything).Run(func(args mock.Arguments) { + s.notifier.On("OnNewQcDiscovered", mock.Anything).Run(func(args mock.Arguments) { qc := args.Get(0).(*flow.QuorumCertificate) if highestReportedQC == nil || highestReportedQC.View < qc.View { highestReportedQC = qc @@ -156,7 +154,7 @@ func (s *TimeoutCollectorTestSuite) TestAddTimeout_TONotifications() { }) lastViewTC := helper.MakeTC(helper.WithTCView(s.view - 1)) - s.collectorNotifier.On("OnNewTcDiscovered", lastViewTC).Once() + s.notifier.On("OnNewTcDiscovered", lastViewTC).Once() timeouts := make([]*model.TimeoutObject, 0, qcCount) for i := 0; i < qcCount; i++ { @@ -174,7 +172,6 @@ func (s *TimeoutCollectorTestSuite) TestAddTimeout_TONotifications() { expectedHighestQC := timeouts[len(timeouts)-1].NewestQC // shuffle timeouts in random order - rand.Seed(time.Now().UnixNano()) rand.Shuffle(len(timeouts), func(i, j int) { timeouts[i], timeouts[j] = timeouts[j], timeouts[i] }) diff --git a/consensus/hotstuff/validator.go b/consensus/hotstuff/validator.go index 17a14ea0603..5bcc77f1810 100644 --- a/consensus/hotstuff/validator.go +++ b/consensus/hotstuff/validator.go @@ -22,7 +22,7 @@ type Validator interface { // ValidateProposal checks the validity of a proposal. // During normal operations, the following error returns are expected: - // * model.InvalidBlockError if the block is invalid + // * model.InvalidProposalError if the block is invalid // * model.ErrViewForUnknownEpoch if the proposal refers unknown epoch ValidateProposal(proposal *model.Proposal) error diff --git a/consensus/hotstuff/validator/validator.go b/consensus/hotstuff/validator/validator.go index f52366ad540..b9cafdc5d89 100644 --- a/consensus/hotstuff/validator/validator.go +++ b/consensus/hotstuff/validator/validator.go @@ -197,7 +197,7 @@ func (v *Validator) ValidateQC(qc *flow.QuorumCertificate) error { // A block is considered as valid if it's a valid extension of existing forks. // Note it doesn't check if it's conflicting with finalized block // During normal operations, the following error returns are expected: -// - model.InvalidBlockError if the block is invalid +// - model.InvalidProposalError if the block is invalid // - model.ErrViewForUnknownEpoch if the proposal refers unknown epoch // // Any other error should be treated as exception @@ -208,7 +208,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { // validate the proposer's vote and get his identity _, err := v.ValidateVote(proposal.ProposerVote()) if model.IsInvalidVoteError(err) { - return newInvalidBlockError(block, fmt.Errorf("invalid proposer signature: %w", err)) + return model.NewInvalidProposalErrorf(proposal, "invalid proposer signature: %w", err) } if err != nil { return fmt.Errorf("error verifying leader signature for block %x: %w", block.BlockID, err) @@ -220,7 +220,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { return fmt.Errorf("error determining leader for block %x: %w", block.BlockID, err) } if leader != block.ProposerID { - return newInvalidBlockError(block, fmt.Errorf("proposer %s is not leader (%s) for view %d", block.ProposerID, leader, block.View)) + return model.NewInvalidProposalErrorf(proposal, "proposer %s is not leader (%s) for view %d", block.ProposerID, leader, block.View) } // The Block must contain a proof that the primary legitimately entered the respective view. @@ -231,23 +231,23 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { if !lastViewSuccessful { // check if proposal is correctly structured if proposal.LastViewTC == nil { - return newInvalidBlockError(block, fmt.Errorf("QC in block is not for previous view, so expecting a TC but none is included in block")) + return model.NewInvalidProposalErrorf(proposal, "QC in block is not for previous view, so expecting a TC but none is included in block") } // check if included TC is for previous view if proposal.Block.View != proposal.LastViewTC.View+1 { - return newInvalidBlockError(block, fmt.Errorf("QC in block is not for previous view, so expecting a TC for view %d but got TC for view %d", proposal.Block.View-1, proposal.LastViewTC.View)) + return model.NewInvalidProposalErrorf(proposal, "QC in block is not for previous view, so expecting a TC for view %d but got TC for view %d", proposal.Block.View-1, proposal.LastViewTC.View) } // Check if proposal extends either the newest QC specified in the TC, or a newer QC // in edge cases a leader may construct a TC and QC concurrently such that TC contains // an older QC - in these case we still want to build on the newest QC, so this case is allowed. if proposal.Block.QC.View < proposal.LastViewTC.NewestQC.View { - return newInvalidBlockError(block, fmt.Errorf("TC in block contains a newer QC than the block itself, which is a protocol violation")) + return model.NewInvalidProposalErrorf(proposal, "TC in block contains a newer QC than the block itself, which is a protocol violation") } } else if proposal.LastViewTC != nil { // last view ended with QC, including TC is a protocol violation - return newInvalidBlockError(block, fmt.Errorf("last view has ended with QC but proposal includes LastViewTC")) + return model.NewInvalidProposalErrorf(proposal, "last view has ended with QC but proposal includes LastViewTC") } // Check signatures, keep the most expensive the last to check @@ -256,7 +256,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { err = v.ValidateQC(qc) if err != nil { if model.IsInvalidQCError(err) { - return newInvalidBlockError(block, fmt.Errorf("invalid qc included: %w", err)) + return model.NewInvalidProposalErrorf(proposal, "invalid qc included: %w", err) } if errors.Is(err, model.ErrViewForUnknownEpoch) { // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the @@ -272,7 +272,7 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { err = v.ValidateTC(proposal.LastViewTC) if err != nil { if model.IsInvalidTCError(err) { - return newInvalidBlockError(block, fmt.Errorf("proposals TC's is not valid: %w", err)) + return model.NewInvalidProposalErrorf(proposal, "proposals TC's is not valid: %w", err) } if errors.Is(err, model.ErrViewForUnknownEpoch) { // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the @@ -323,14 +323,6 @@ func (v *Validator) ValidateVote(vote *model.Vote) (*flow.Identity, error) { return voter, nil } -func newInvalidBlockError(block *model.Block, err error) error { - return model.InvalidBlockError{ - BlockID: block.BlockID, - View: block.View, - Err: err, - } -} - func newInvalidQCError(qc *flow.QuorumCertificate, err error) error { return model.InvalidQCError{ BlockID: qc.BlockID, diff --git a/consensus/hotstuff/validator/validator_test.go b/consensus/hotstuff/validator/validator_test.go index 8dbf03736d1..6eb3da069ce 100644 --- a/consensus/hotstuff/validator/validator_test.go +++ b/consensus/hotstuff/validator/validator_test.go @@ -5,7 +5,6 @@ import ( "fmt" "math/rand" "testing" - "time" "github.com/onflow/flow-go/module/signature" @@ -46,7 +45,6 @@ type ProposalSuite struct { func (ps *ProposalSuite) SetupTest() { // the leader is a random node for now - rand.Seed(time.Now().UnixNano()) ps.finalized = uint64(rand.Uint32() + 1) ps.participants = unittest.IdentityListFixture(8, unittest.WithRole(flow.RoleConsensus)) ps.leader = ps.participants[0] @@ -116,7 +114,7 @@ func (ps *ProposalSuite) TestProposalSignatureError() { assert.Error(ps.T(), err, "a proposal should be rejected if signature check fails") // check that the error is not one that leads to invalid - assert.False(ps.T(), model.IsInvalidBlockError(err), "if signature check fails, we should not receive an ErrorInvalidBlock") + assert.False(ps.T(), model.IsInvalidProposalError(err), "if signature check fails, we should not receive an ErrorInvalidBlock") } func (ps *ProposalSuite) TestProposalSignatureInvalidFormat() { @@ -131,7 +129,7 @@ func (ps *ProposalSuite) TestProposalSignatureInvalidFormat() { assert.Error(ps.T(), err, "a proposal with an invalid signature should be rejected") // check that the error is an invalid proposal error to allow creating slashing challenge - assert.True(ps.T(), model.IsInvalidBlockError(err), "if signature is invalid, we should generate an invalid error") + assert.True(ps.T(), model.IsInvalidProposalError(err), "if signature is invalid, we should generate an invalid error") } func (ps *ProposalSuite) TestProposalSignatureInvalid() { @@ -146,7 +144,7 @@ func (ps *ProposalSuite) TestProposalSignatureInvalid() { assert.Error(ps.T(), err, "a proposal with an invalid signature should be rejected") // check that the error is an invalid proposal error to allow creating slashing challenge - assert.True(ps.T(), model.IsInvalidBlockError(err), "if signature is invalid, we should generate an invalid error") + assert.True(ps.T(), model.IsInvalidProposalError(err), "if signature is invalid, we should generate an invalid error") } func (ps *ProposalSuite) TestProposalWrongLeader() { @@ -163,12 +161,12 @@ func (ps *ProposalSuite) TestProposalWrongLeader() { assert.Error(ps.T(), err, "a proposal from the wrong proposer should be rejected") // check that the error is an invalid proposal error to allow creating slashing challenge - assert.True(ps.T(), model.IsInvalidBlockError(err), "if the proposal has wrong proposer, we should generate a invalid error") + assert.True(ps.T(), model.IsInvalidProposalError(err), "if the proposal has wrong proposer, we should generate a invalid error") } // TestProposalQCInvalid checks that Validator handles the verifier's error returns correctly. // In case of `model.InvalidFormatError` and model.ErrInvalidSignature`, we expect the Validator -// to recognize those as an invalid QC, i.e. returns an `model.InvalidBlockError`. +// to recognize those as an invalid QC, i.e. returns an `model.InvalidProposalError`. // In contrast, unexpected exceptions and `model.InvalidSignerError` should _not_ be // interpreted as a sign of an invalid QC. func (ps *ProposalSuite) TestProposalQCInvalid() { @@ -180,7 +178,7 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) - assert.True(ps.T(), model.IsInvalidBlockError(err), "if the block's QC signature is invalid, an ErrorInvalidBlock error should be raised") + assert.True(ps.T(), model.IsInvalidProposalError(err), "if the block's QC signature is invalid, an ErrorInvalidBlock error should be raised") }) ps.Run("invalid-format", func() { @@ -190,7 +188,7 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) - assert.True(ps.T(), model.IsInvalidBlockError(err), "if the block's QC has an invalid format, an ErrorInvalidBlock error should be raised") + assert.True(ps.T(), model.IsInvalidProposalError(err), "if the block's QC has an invalid format, an ErrorInvalidBlock error should be raised") }) // Theoretically, `VerifyQC` could also return a `model.InvalidSignerError`. However, @@ -207,7 +205,7 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) assert.Error(ps.T(), err) - assert.False(ps.T(), model.IsInvalidBlockError(err)) + assert.False(ps.T(), model.IsInvalidProposalError(err)) }) ps.Run("unknown-exception", func() { @@ -219,7 +217,7 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) assert.ErrorIs(ps.T(), err, exception) - assert.False(ps.T(), model.IsInvalidBlockError(err)) + assert.False(ps.T(), model.IsInvalidProposalError(err)) }) ps.Run("verify-qc-err-view-for-unknown-epoch", func() { @@ -227,11 +225,11 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return(model.ErrViewForUnknownEpoch) ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(nil) - // check that validation fails and the failure is considered internal exception and NOT an InvalidBlock error + // check that validation fails and the failure is considered internal exception and NOT an InvalidProposal error err := ps.validator.ValidateProposal(ps.proposal) assert.Error(ps.T(), err) assert.NotErrorIs(ps.T(), err, model.ErrViewForUnknownEpoch) - assert.False(ps.T(), model.IsInvalidBlockError(err)) + assert.False(ps.T(), model.IsInvalidProposalError(err)) }) } @@ -247,7 +245,7 @@ func (ps *ProposalSuite) TestProposalQCError() { assert.Error(ps.T(), err, "a proposal with an invalid QC should be rejected") // check that the error is an invalid proposal error to allow creating slashing challenge - assert.False(ps.T(), model.IsInvalidBlockError(err), "if we can't verify the QC, we should not generate a invalid error") + assert.False(ps.T(), model.IsInvalidProposalError(err), "if we can't verify the QC, we should not generate a invalid error") } // TestProposalWithLastViewTC tests different scenarios where last view has ended with TC @@ -286,7 +284,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { // in this case proposal without LastViewTC is considered invalid ) err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyQC") ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) @@ -304,7 +302,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { helper.WithTCNewestQC(ps.block.QC))), ) err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyQC") ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) @@ -323,7 +321,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { helper.WithTCNewestQC(helper.MakeQC(helper.WithQCView(ps.block.View+1))))), ) err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyQC") ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) @@ -347,7 +345,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { // this is considered an invalid TC, because highest QC's view is not equal to max{NewestQCViews} proposal.LastViewTC.NewestQCViews[0] = proposal.LastViewTC.NewestQC.View + 1 err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err) && model.IsInvalidTCError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) ps.Run("included-tc-threshold-not-reached", func() { @@ -368,7 +366,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { )), ) err = ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err) && model.IsInvalidTCError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) ps.Run("included-tc-highest-qc-invalid", func() { @@ -394,7 +392,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { ps.verifier.On("VerifyQC", ps.voters, qc.SigData, qc.View, qc.BlockID).Return(model.ErrInvalidSignature).Once() err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err) && model.IsInvalidTCError(err)) }) ps.Run("verify-qc-err-view-for-unknown-epoch", func() { newestQC := helper.MakeQC( @@ -420,7 +418,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { newestQC.View, newestQC.BlockID).Return(model.ErrViewForUnknownEpoch).Once() err := ps.validator.ValidateProposal(proposal) require.Error(ps.T(), err) - require.False(ps.T(), model.IsInvalidBlockError(err)) + require.False(ps.T(), model.IsInvalidProposalError(err)) require.False(ps.T(), model.IsInvalidTCError(err)) require.NotErrorIs(ps.T(), err, model.ErrViewForUnknownEpoch) }) @@ -440,7 +438,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { ps.verifier.On("VerifyTC", ps.voters, []byte(proposal.LastViewTC.SigData), proposal.LastViewTC.View, proposal.LastViewTC.NewestQCViews).Return(model.ErrInvalidSignature).Once() err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err) && model.IsInvalidTCError(err)) ps.verifier.AssertCalled(ps.T(), "VerifyTC", ps.voters, []byte(proposal.LastViewTC.SigData), proposal.LastViewTC.View, proposal.LastViewTC.NewestQCViews) }) @@ -455,7 +453,7 @@ func (ps *ProposalSuite) TestProposalWithLastViewTC() { helper.WithLastViewTC(helper.MakeTC()), ) err := ps.validator.ValidateProposal(proposal) - require.True(ps.T(), model.IsInvalidBlockError(err)) + require.True(ps.T(), model.IsInvalidProposalError(err)) ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") }) ps.verifier.AssertExpectations(ps.T()) @@ -671,7 +669,7 @@ func (qs *QCSuite) TestQCSignatureError() { // TestQCSignatureInvalid verifies that the Validator correctly handles the model.ErrInvalidSignature. // This error return from `Verifier.VerifyQC` is an expected failure case in case of a byzantine input, where -// one of the signatures in the QC is broken. Hence, the Validator should wrap it as InvalidBlockError. +// one of the signatures in the QC is broken. Hence, the Validator should wrap it as InvalidProposalError. func (qs *QCSuite) TestQCSignatureInvalid() { // change the verifier to fail the QC signature *qs.verifier = mocks.Verifier{} @@ -695,7 +693,7 @@ func (qs *QCSuite) TestQCVerifyQC_ErrViewForUnknownEpoch() { // TestQCSignatureInvalidFormat verifies that the Validator correctly handles the model.InvalidFormatError. // This error return from `Verifier.VerifyQC` is an expected failure case in case of a byzantine input, where -// some binary vector (e.g. `sigData`) is broken. Hence, the Validator should wrap it as InvalidBlockError. +// some binary vector (e.g. `sigData`) is broken. Hence, the Validator should wrap it as InvalidProposalError. func (qs *QCSuite) TestQCSignatureInvalidFormat() { // change the verifier to fail the QC signature *qs.verifier = mocks.Verifier{} @@ -710,7 +708,7 @@ func (qs *QCSuite) TestQCSignatureInvalidFormat() { // In the validator, we previously checked the total weight of all signers meets the supermajority threshold, // which is a _positive_ number. Hence, there must be at least one signer. Hence, `Verifier.VerifyQC` // returning this error would be a symptom of a fatal internal bug. The Validator should _not_ interpret -// this error as an invalid QC / invalid block, i.e. it should _not_ return an `InvalidBlockError`. +// this error as an invalid QC / invalid block, i.e. it should _not_ return an `InvalidProposalError`. func (qs *QCSuite) TestQCEmptySigners() { *qs.verifier = mocks.Verifier{} qs.verifier.On("VerifyQC", mock.Anything, qs.qc.SigData, qs.block.View, qs.block.BlockID).Return( @@ -719,7 +717,7 @@ func (qs *QCSuite) TestQCEmptySigners() { // the Validator should _not_ interpret this as a invalid QC, but as an internal error err := qs.validator.ValidateQC(qs.qc) assert.True(qs.T(), model.IsInsufficientSignaturesError(err)) // unexpected error should be wrapped and propagated upwards - assert.False(qs.T(), model.IsInvalidBlockError(err), err, "should _not_ interpret this as a invalid QC, but as an internal error") + assert.False(qs.T(), model.IsInvalidProposalError(err), err, "should _not_ interpret this as a invalid QC, but as an internal error") } func TestValidateTC(t *testing.T) { @@ -753,7 +751,6 @@ func (s *TCSuite) SetupTest() { s.indices, err = signature.EncodeSignersToIndices(s.participants.NodeIDs(), s.signers.NodeIDs()) require.NoError(s.T(), err) - rand.Seed(time.Now().UnixNano()) view := uint64(int(rand.Uint32()) + len(s.participants)) highQCViews := make([]uint64, 0, len(s.signers)) diff --git a/consensus/hotstuff/vote_collector.go b/consensus/hotstuff/vote_collector.go index be5c6460723..157ef5338a7 100644 --- a/consensus/hotstuff/vote_collector.go +++ b/consensus/hotstuff/vote_collector.go @@ -59,7 +59,7 @@ type VoteCollector interface { // ProcessBlock performs validation of block signature and processes block with respected collector. // Calling this function will mark conflicting collector as stale and change state of valid collectors // It returns nil if the block is valid. - // It returns model.InvalidBlockError if block is invalid. + // It returns model.InvalidProposalError if block is invalid. // It returns other error if there is exception processing the block. ProcessBlock(block *model.Proposal) error @@ -115,6 +115,6 @@ type VoteProcessorFactory interface { // Create instantiates a VerifyingVoteProcessor for processing votes for a specific proposal. // Caller can be sure that proposal vote was successfully verified and processed. // Expected error returns during normal operations: - // * model.InvalidBlockError - proposal has invalid proposer vote + // * model.InvalidProposalError - proposal has invalid proposer vote Create(log zerolog.Logger, proposal *model.Proposal) (VerifyingVoteProcessor, error) } diff --git a/consensus/hotstuff/voteaggregator/vote_aggregator.go b/consensus/hotstuff/voteaggregator/vote_aggregator.go index 6f0063f0037..6471cc6ada6 100644 --- a/consensus/hotstuff/voteaggregator/vote_aggregator.go +++ b/consensus/hotstuff/voteaggregator/vote_aggregator.go @@ -12,9 +12,9 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/common/fifoqueue" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/counters" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/metrics" @@ -37,7 +37,7 @@ type VoteAggregator struct { log zerolog.Logger hotstuffMetrics module.HotstuffMetrics engineMetrics module.EngineMetrics - notifier hotstuff.Consumer + notifier hotstuff.VoteAggregationViolationConsumer lowestRetainedView counters.StrictMonotonousCounter // lowest view, for which we still process votes collectors hotstuff.VoteCollectors queuedMessagesNotifier engine.Notifier @@ -58,7 +58,7 @@ func NewVoteAggregator( hotstuffMetrics module.HotstuffMetrics, engineMetrics module.EngineMetrics, mempoolMetrics module.MempoolMetrics, - notifier hotstuff.Consumer, + notifier hotstuff.VoteAggregationViolationConsumer, lowestRetainedView uint64, collectors hotstuff.VoteCollectors, ) (*VoteAggregator, error) { @@ -246,7 +246,7 @@ func (va *VoteAggregator) processQueuedBlock(block *model.Proposal) error { err = collector.ProcessBlock(block) if err != nil { - if model.IsInvalidBlockError(err) { + if model.IsInvalidProposalError(err) { // We are attempting process a block which is invalid // This should never happen, because any component that feeds blocks into VoteAggregator // needs to make sure that it's submitting for processing ONLY valid blocks. diff --git a/consensus/hotstuff/voteaggregator/vote_aggregator_test.go b/consensus/hotstuff/voteaggregator/vote_aggregator_test.go index 792c42cbca5..006ab52b744 100644 --- a/consensus/hotstuff/voteaggregator/vote_aggregator_test.go +++ b/consensus/hotstuff/voteaggregator/vote_aggregator_test.go @@ -29,7 +29,7 @@ type VoteAggregatorTestSuite struct { aggregator *VoteAggregator collectors *mocks.VoteCollectors - consumer *mocks.Consumer + consumer *mocks.VoteAggregationConsumer stopAggregator context.CancelFunc errs <-chan error } @@ -37,7 +37,7 @@ type VoteAggregatorTestSuite struct { func (s *VoteAggregatorTestSuite) SetupTest() { var err error s.collectors = mocks.NewVoteCollectors(s.T()) - s.consumer = mocks.NewConsumer(s.T()) + s.consumer = mocks.NewVoteAggregationConsumer(s.T()) s.collectors.On("Start", mock.Anything).Once() unittest.ReadyDoneify(s.collectors) @@ -95,7 +95,7 @@ func (s *VoteAggregatorTestSuite) TestProcessInvalidBlock() { collector := mocks.NewVoteCollector(s.T()) collector.On("ProcessBlock", block).Run(func(_ mock.Arguments) { close(processed) - }).Return(model.InvalidBlockError{}) + }).Return(model.InvalidProposalError{}) s.collectors.On("GetOrCreateCollector", block.Block.View).Return(collector, true, nil).Once() // submit block for processing @@ -106,7 +106,7 @@ func (s *VoteAggregatorTestSuite) TestProcessInvalidBlock() { select { case err := <-s.errs: require.Error(s.T(), err) - require.False(s.T(), model.IsInvalidBlockError(err)) + require.False(s.T(), model.IsInvalidProposalError(err)) case <-time.After(100 * time.Millisecond): s.T().Fatalf("expected error but haven't received anything") } diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go b/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go index ef1fa25df85..1c005388d40 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "sync" "testing" - "time" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -30,7 +29,6 @@ import ( modulemock "github.com/onflow/flow-go/module/mock" msig "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/state/protocol/inmem" - "github.com/onflow/flow-go/state/protocol/seed" storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -597,7 +595,6 @@ func TestCombinedVoteProcessorV2_PropertyCreatingQCCorrectness(testifyT *testing } // shuffle votes in random order - rand.Seed(time.Now().UnixNano()) rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) @@ -745,7 +742,6 @@ func TestCombinedVoteProcessorV2_PropertyCreatingQCLiveness(testifyT *testing.T) } // shuffle votes in random order - rand.Seed(time.Now().UnixNano()) rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) @@ -789,7 +785,7 @@ func TestCombinedVoteProcessorV2_BuildVerifyQC(t *testing.T) { epochLookup.On("EpochForViewWithFallback", view).Return(epochCounter, nil) // all committee members run DKG - dkgData, err := bootstrapDKG.RunFastKG(11, unittest.RandomBytes(32)) + dkgData, err := bootstrapDKG.RandomBeaconKG(11, unittest.RandomBytes(32)) require.NoError(t, err) // signers hold objects that are created with private key and can sign votes and proposals @@ -948,10 +944,10 @@ func TestReadRandomSourceFromPackedQCV2(t *testing.T) { qc, err := buildQCWithPackerAndSigData(packer, block, blockSigData) require.NoError(t, err) - randomSource, err := seed.FromParentQCSignature(qc.SigData) + randomSource, err := model.BeaconSignature(qc) require.NoError(t, err) - randomSourceAgain, err := seed.FromParentQCSignature(qc.SigData) + randomSourceAgain, err := model.BeaconSignature(qc) require.NoError(t, err) // verify the random source is deterministic diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go b/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go index 01497d59ff5..831a68e1650 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "sync" "testing" - "time" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -647,7 +646,6 @@ func TestCombinedVoteProcessorV3_PropertyCreatingQCCorrectness(testifyT *testing } // shuffle votes in random order - rand.Seed(time.Now().UnixNano()) rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) @@ -880,7 +878,6 @@ func TestCombinedVoteProcessorV3_PropertyCreatingQCLiveness(testifyT *testing.T) } // shuffle votes in random order - rand.Seed(time.Now().UnixNano()) rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) @@ -924,7 +921,7 @@ func TestCombinedVoteProcessorV3_BuildVerifyQC(t *testing.T) { view := uint64(20) epochLookup.On("EpochForViewWithFallback", view).Return(epochCounter, nil) - dkgData, err := bootstrapDKG.RunFastKG(11, unittest.RandomBytes(32)) + dkgData, err := bootstrapDKG.RandomBeaconKG(11, unittest.RandomBytes(32)) require.NoError(t, err) // signers hold objects that are created with private key and can sign votes and proposals diff --git a/consensus/hotstuff/votecollector/factory.go b/consensus/hotstuff/votecollector/factory.go index 31d36119978..2c515fc052c 100644 --- a/consensus/hotstuff/votecollector/factory.go +++ b/consensus/hotstuff/votecollector/factory.go @@ -28,7 +28,7 @@ type baseFactory func(log zerolog.Logger, block *model.Block) (hotstuff.Verifyin // * delegates the creation of the actual instances to baseFactory // * adds the logic to verify the proposer's vote for its own block // Thereby, VoteProcessorFactory guarantees that only proposals with valid proposer -// vote are accepted (as per API specification). Otherwise, an `model.InvalidBlockError` +// vote are accepted (as per API specification). Otherwise, an `model.InvalidProposalError` // is returned. type VoteProcessorFactory struct { baseFactory baseFactory @@ -39,7 +39,7 @@ var _ hotstuff.VoteProcessorFactory = (*VoteProcessorFactory)(nil) // Create instantiates a VerifyingVoteProcessor for the given block proposal. // A VerifyingVoteProcessor are only created for proposals with valid proposer votes. // Expected error returns during normal operations: -// * model.InvalidBlockError - proposal has invalid proposer vote +// * model.InvalidProposalError - proposal has invalid proposer vote func (f *VoteProcessorFactory) Create(log zerolog.Logger, proposal *model.Proposal) (hotstuff.VerifyingVoteProcessor, error) { processor, err := f.baseFactory(log, proposal.Block) if err != nil { @@ -49,11 +49,7 @@ func (f *VoteProcessorFactory) Create(log zerolog.Logger, proposal *model.Propos err = processor.Process(proposal.ProposerVote()) if err != nil { if model.IsInvalidVoteError(err) { - return nil, model.InvalidBlockError{ - BlockID: proposal.Block.BlockID, - View: proposal.Block.View, - Err: fmt.Errorf("invalid proposer vote: %w", err), - } + return nil, model.NewInvalidProposalErrorf(proposal, "invalid proposer vote: %w", err) } return nil, fmt.Errorf("processing proposer's vote for block %v failed: %w", proposal.Block.BlockID, err) } diff --git a/consensus/hotstuff/votecollector/factory_test.go b/consensus/hotstuff/votecollector/factory_test.go index 52cbafe9955..9adeaef98f8 100644 --- a/consensus/hotstuff/votecollector/factory_test.go +++ b/consensus/hotstuff/votecollector/factory_test.go @@ -58,7 +58,7 @@ func TestVoteProcessorFactory_CreateWithInvalidVote(t *testing.T) { processor, err := voteProcessorFactory.Create(unittest.Logger(), proposal) require.Error(t, err) require.Nil(t, processor) - require.True(t, model.IsInvalidBlockError(err)) + require.True(t, model.IsInvalidProposalError(err)) mockedProcessor.AssertExpectations(t) }) @@ -80,7 +80,7 @@ func TestVoteProcessorFactory_CreateWithInvalidVote(t *testing.T) { require.ErrorIs(t, err, exception) require.Nil(t, processor) // an unexpected exception should _not_ be interpreted as the block being invalid - require.False(t, model.IsInvalidBlockError(err)) + require.False(t, model.IsInvalidProposalError(err)) mockedProcessor.AssertExpectations(t) }) @@ -107,7 +107,7 @@ func TestVoteProcessorFactory_CreateProcessException(t *testing.T) { require.ErrorIs(t, err, exception) require.Nil(t, processor) // an unexpected exception should _not_ be interpreted as the block being invalid - require.False(t, model.IsInvalidBlockError(err)) + require.False(t, model.IsInvalidProposalError(err)) mockedFactory.AssertExpectations(t) } diff --git a/consensus/hotstuff/votecollector/statemachine.go b/consensus/hotstuff/votecollector/statemachine.go index 6b7173196ab..d62159ea9ef 100644 --- a/consensus/hotstuff/votecollector/statemachine.go +++ b/consensus/hotstuff/votecollector/statemachine.go @@ -25,7 +25,7 @@ type VoteCollector struct { sync.Mutex log zerolog.Logger workers hotstuff.Workers - notifier hotstuff.Consumer + notifier hotstuff.VoteAggregationConsumer createVerifyingProcessor VerifyingVoteProcessorFactory votesCache VotesCache @@ -47,7 +47,7 @@ type atomicValueWrapper struct { func NewStateMachineFactory( log zerolog.Logger, - notifier hotstuff.Consumer, + notifier hotstuff.VoteAggregationConsumer, verifyingVoteProcessorFactory VerifyingVoteProcessorFactory, ) voteaggregator.NewCollectorFactoryMethod { return func(view uint64, workers hotstuff.Workers) (hotstuff.VoteCollector, error) { @@ -59,7 +59,7 @@ func NewStateMachine( view uint64, log zerolog.Logger, workers hotstuff.Workers, - notifier hotstuff.Consumer, + notifier hotstuff.VoteAggregationConsumer, verifyingVoteProcessorFactory VerifyingVoteProcessorFactory, ) *VoteCollector { log = log.With(). diff --git a/consensus/hotstuff/votecollector/statemachine_test.go b/consensus/hotstuff/votecollector/statemachine_test.go index 8ad19e98903..007dcce1fe2 100644 --- a/consensus/hotstuff/votecollector/statemachine_test.go +++ b/consensus/hotstuff/votecollector/statemachine_test.go @@ -32,7 +32,7 @@ type StateMachineTestSuite struct { suite.Suite view uint64 - notifier *mocks.Consumer + notifier *mocks.VoteAggregationConsumer workerPool *workerpool.WorkerPool factoryMethod VerifyingVoteProcessorFactory mockedProcessors map[flow.Identifier]*mocks.VerifyingVoteProcessor @@ -49,7 +49,7 @@ func (s *StateMachineTestSuite) TearDownTest() { func (s *StateMachineTestSuite) SetupTest() { s.view = 1000 s.mockedProcessors = make(map[flow.Identifier]*mocks.VerifyingVoteProcessor) - s.notifier = &mocks.Consumer{} + s.notifier = mocks.NewVoteAggregationConsumer(s.T()) s.factoryMethod = func(log zerolog.Logger, block *model.Proposal) (hotstuff.VerifyingVoteProcessor, error) { if processor, found := s.mockedProcessors[block.Block.BlockID]; found { @@ -152,7 +152,6 @@ func (s *StateMachineTestSuite) TestAddVote_VerifyingState() { s.T().Run("add-invalid-vote", func(t *testing.T) { vote := unittest.VoteForBlockFixture(block, unittest.WithVoteView(s.view)) processor.On("Process", vote).Return(model.NewInvalidVoteErrorf(vote, "")).Once() - s.notifier.On("OnVoteProcessed", vote).Once() s.notifier.On("OnInvalidVoteDetected", mock.Anything).Run(func(args mock.Arguments) { invalidVoteErr := args.Get(0).(model.InvalidVoteError) require.Equal(s.T(), vote, invalidVoteErr.Vote) diff --git a/consensus/integration/integration_test.go b/consensus/integration/integration_test.go index 6ba804d103d..13a337d3291 100644 --- a/consensus/integration/integration_test.go +++ b/consensus/integration/integration_test.go @@ -24,6 +24,7 @@ func runNodes(signalerCtx irrecoverable.SignalerContext, nodes []*Node) { n.timeoutAggregator.Start(signalerCtx) n.compliance.Start(signalerCtx) n.messageHub.Start(signalerCtx) + n.sync.Start(signalerCtx) <-util.AllReady(n.committee, n.hot, n.voteAggregator, n.timeoutAggregator, n.compliance, n.sync, n.messageHub) }(n) } diff --git a/consensus/integration/network_test.go b/consensus/integration/network_test.go index 181e3e79adc..1a3eb31d58b 100644 --- a/consensus/integration/network_test.go +++ b/consensus/integration/network_test.go @@ -67,6 +67,8 @@ type Network struct { mocknetwork.Network } +var _ network.Network = (*Network)(nil) + // Register registers an Engine of the attached node to the channel via a Conduit, and returns the // Conduit instance. func (n *Network) Register(channel channels.Channel, engine network.MessageProcessor) (network.Conduit, error) { @@ -158,7 +160,11 @@ func (n *Network) publish(event interface{}, channel channels.Channel, targetIDs // Engines attached to the same channel on other nodes. The targeted nodes are selected based on the selector. // In this test helper implementation, multicast uses submit method under the hood. func (n *Network) multicast(event interface{}, channel channels.Channel, num uint, targetIDs ...flow.Identifier) error { - targetIDs = flow.Sample(num, targetIDs...) + var err error + targetIDs, err = flow.Sample(num, targetIDs...) + if err != nil { + return fmt.Errorf("sampling failed: %w", err) + } return n.submit(event, channel, targetIDs...) } @@ -170,6 +176,15 @@ type Conduit struct { queue chan message } +// ReportMisbehavior reports the misbehavior of a node on sending a message to the current node that appears valid +// based on the networking layer but is considered invalid by the current node based on the Flow protocol. +// This method is a no-op in the test helper implementation. +func (c *Conduit) ReportMisbehavior(_ network.MisbehaviorReport) { + // no-op +} + +var _ network.Conduit = (*Conduit)(nil) + func (c *Conduit) Submit(event interface{}, targetIDs ...flow.Identifier) error { if c.ctx.Err() != nil { return fmt.Errorf("conduit closed") diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index b24b5b16ee4..79fc57a56f1 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -40,6 +40,7 @@ import ( "github.com/onflow/flow-go/module/buffer" builder "github.com/onflow/flow-go/module/builder/consensus" synccore "github.com/onflow/flow-go/module/chainsync" + modulecompliance "github.com/onflow/flow-go/module/compliance" finalizer "github.com/onflow/flow-go/module/finalizer/consensus" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/irrecoverable" @@ -314,7 +315,7 @@ func createConsensusIdentities(t *testing.T, n int) *run.ParticipantData { // completeConsensusIdentities runs KG process and fills nodeInfos with missing random beacon keys func completeConsensusIdentities(t *testing.T, nodeInfos []bootstrap.NodeInfo) *run.ParticipantData { - dkgData, err := bootstrapDKG.RunFastKG(len(nodeInfos), unittest.RandomBytes(48)) + dkgData, err := bootstrapDKG.RandomBeaconKG(len(nodeInfos), unittest.RandomBytes(48)) require.NoError(t, err) participantData := &run.ParticipantData{ @@ -375,7 +376,8 @@ func createNode( setupsDB := storage.NewEpochSetups(metricsCollector, db) commitsDB := storage.NewEpochCommits(metricsCollector, db) statusesDB := storage.NewEpochStatuses(metricsCollector, db) - consumer := events.NewDistributor() + versionBeaconDB := storage.NewVersionBeacons(db) + protocolStateEvents := events.NewDistributor() localID := identity.ID() @@ -395,6 +397,7 @@ func createNode( setupsDB, commitsDB, statusesDB, + versionBeaconDB, rootSnapshot, ) require.NoError(t, err) @@ -405,7 +408,7 @@ func createNode( fullState, err := bprotocol.NewFullConsensusState( log, tracer, - consumer, + protocolStateEvents, state, indexDB, payloadsDB, @@ -432,9 +435,9 @@ func createNode( // log with node index logConsumer := notifications.NewLogConsumer(log) - notifier := pubsub.NewDistributor() - notifier.AddConsumer(counterConsumer) - notifier.AddConsumer(logConsumer) + hotstuffDistributor := pubsub.NewDistributor() + hotstuffDistributor.AddConsumer(counterConsumer) + hotstuffDistributor.AddConsumer(logConsumer) require.Equal(t, participant.nodeInfo.NodeID, localID) privateKeys, err := participant.nodeInfo.PrivateKeys() @@ -472,7 +475,7 @@ func createNode( // selector := filter.HasRole(flow.RoleConsensus) committee, err := committees.NewConsensusCommittee(state, localID) require.NoError(t, err) - consumer.AddConsumer(committee) + protocolStateEvents.AddConsumer(committee) // initialize the block finalizer final := finalizer.NewFinalizer(db, headersDB, fullState, trace.NewNoopTracer()) @@ -480,9 +483,10 @@ func createNode( syncCore, err := synccore.New(log, synccore.DefaultConfig(), metricsCollector, rootHeader.ChainID) require.NoError(t, err) - qcDistributor := pubsub.NewQCCreatedDistributor() + voteAggregationDistributor := pubsub.NewVoteAggregationDistributor() + voteAggregationDistributor.AddVoteAggregationConsumer(logConsumer) - forks, err := consensus.NewForks(rootHeader, headersDB, final, notifier, rootHeader, rootQC) + forks, err := consensus.NewForks(rootHeader, headersDB, final, hotstuffDistributor, rootHeader, rootQC) require.NoError(t, err) validator := consensus.NewValidator(metricsCollector, committee) @@ -514,9 +518,9 @@ func createNode( livenessData, err := persist.GetLivenessData() require.NoError(t, err) - voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(committee, qcDistributor.OnQcConstructedFromVotes) + voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(committee, voteAggregationDistributor.OnQcConstructedFromVotes) - createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) + createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, voteAggregationDistributor, voteProcessorFactory.Create) voteCollectors := voteaggregator.NewVoteCollectors(log, livenessData.CurrentView, workerpool.New(2), createCollectorFactoryMethod) voteAggregator, err := voteaggregator.NewVoteAggregator( @@ -524,36 +528,39 @@ func createNode( metricsCollector, metricsCollector, metricsCollector, - notifier, + voteAggregationDistributor, livenessData.CurrentView, voteCollectors, ) require.NoError(t, err) - timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() - timeoutCollectorDistributor.AddConsumer(logConsumer) + timeoutAggregationDistributor := pubsub.NewTimeoutAggregationDistributor() + timeoutAggregationDistributor.AddTimeoutCollectorConsumer(logConsumer) timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory( log, - timeoutCollectorDistributor, + timeoutAggregationDistributor, committee, validator, msig.ConsensusTimeoutTag, ) timeoutCollectorsFactory := timeoutcollector.NewTimeoutCollectorFactory( log, - notifier, - timeoutCollectorDistributor, + timeoutAggregationDistributor, timeoutProcessorFactory, ) - timeoutCollectors := timeoutaggregator.NewTimeoutCollectors(log, livenessData.CurrentView, timeoutCollectorsFactory) + timeoutCollectors := timeoutaggregator.NewTimeoutCollectors( + log, + metricsCollector, + livenessData.CurrentView, + timeoutCollectorsFactory, + ) timeoutAggregator, err := timeoutaggregator.NewTimeoutAggregator( log, metricsCollector, metricsCollector, metricsCollector, - notifier, livenessData.CurrentView, timeoutCollectors, ) @@ -562,12 +569,12 @@ func createNode( hotstuffModules := &consensus.HotstuffModules{ Forks: forks, Validator: validator, - Notifier: notifier, + Notifier: hotstuffDistributor, Committee: committee, Signer: signer, Persist: persist, - QCCreatedDistributor: qcDistributor, - TimeoutCollectorDistributor: timeoutCollectorDistributor, + VoteCollectorDistributor: voteAggregationDistributor.VoteCollectorDistributor, + TimeoutCollectorDistributor: timeoutAggregationDistributor.TimeoutCollectorDistributor, VoteAggregator: voteAggregator, TimeoutAggregator: timeoutAggregator, } @@ -576,6 +583,7 @@ func createNode( hot, err := consensus.NewParticipant( log, metricsCollector, + metricsCollector, build, rootHeader, []*flow.Header{}, @@ -594,6 +602,7 @@ func createNode( metricsCollector, metricsCollector, metricsCollector, + hotstuffDistributor, tracer, headersDB, payloadsDB, @@ -604,15 +613,13 @@ func createNode( hot, voteAggregator, timeoutAggregator, + modulecompliance.DefaultConfig(), ) require.NoError(t, err) comp, err := compliance.NewEngine(log, me, compCore) require.NoError(t, err) - finalizedHeader, err := synceng.NewFinalizedHeaderCache(log, state, pubsub.NewFinalizationDistributor()) - require.NoError(t, err) - identities, err := state.Final().Identities(filter.And( filter.HasRole(flow.RoleConsensus), filter.Not(filter.HasNodeID(me.NodeID())), @@ -626,10 +633,10 @@ func createNode( metricsCollector, net, me, + state, blocksDB, comp, syncCore, - finalizedHeader, idProvider, func(cfg *synceng.Config) { // use a small pool and scan interval for sync engine @@ -653,7 +660,7 @@ func createNode( ) require.NoError(t, err) - notifier.AddConsumer(messageHub) + hotstuffDistributor.AddConsumer(messageHub) node.compliance = comp node.sync = sync diff --git a/consensus/participant.go b/consensus/participant.go index b783c55d472..b6b65cb06b5 100644 --- a/consensus/participant.go +++ b/consensus/participant.go @@ -27,6 +27,7 @@ import ( func NewParticipant( log zerolog.Logger, metrics module.HotstuffMetrics, + mempoolMetrics module.MempoolMetrics, builder module.Builder, finalized *flow.Header, pending []*flow.Header, @@ -34,10 +35,8 @@ func NewParticipant( options ...Option, ) (*eventloop.EventLoop, error) { - // initialize the default configuration + // initialize the default configuration and apply the configuration options cfg := DefaultParticipantConfig() - - // apply the configuration options for _, option := range options { option(&cfg) } @@ -46,28 +45,31 @@ func NewParticipant( modules.VoteAggregator.PruneUpToView(finalized.View) modules.TimeoutAggregator.PruneUpToView(finalized.View) - // recover hotstuff state (inserts all pending blocks into Forks and VoteAggregator) - err := recovery.Participant(log, modules.Forks, modules.VoteAggregator, modules.Validator, pending) + // recover HotStuff state from all pending blocks + qcCollector := recovery.NewCollector[*flow.QuorumCertificate]() + tcCollector := recovery.NewCollector[*flow.TimeoutCertificate]() + err := recovery.Recover(log, pending, + recovery.ForksState(modules.Forks), // add pending blocks to Forks + recovery.VoteAggregatorState(modules.VoteAggregator), // accept votes for all pending blocks + recovery.CollectParentQCs(qcCollector), // collect QCs from all pending block to initialize PaceMaker (below) + recovery.CollectTCs(tcCollector), // collect TCs from all pending block to initialize PaceMaker (below) + ) if err != nil { - return nil, fmt.Errorf("could not recover hotstuff state: %w", err) + return nil, fmt.Errorf("failed to scan tree of pending blocks: %w", err) } - // initialize the timeout config - timeoutConfig, err := timeout.NewConfig( - cfg.TimeoutMinimum, - cfg.TimeoutMaximum, - cfg.TimeoutAdjustmentFactor, - cfg.HappyPathMaxRoundFailures, - cfg.BlockRateDelay, - cfg.MaxTimeoutObjectRebroadcastInterval, - ) + // initialize dynamically updatable timeout config + timeoutConfig, err := timeout.NewConfig(cfg.TimeoutMinimum, cfg.TimeoutMaximum, cfg.TimeoutAdjustmentFactor, cfg.HappyPathMaxRoundFailures, cfg.MaxTimeoutObjectRebroadcastInterval) if err != nil { return nil, fmt.Errorf("could not initialize timeout config: %w", err) } // initialize the pacemaker controller := timeout.NewController(timeoutConfig) - pacemaker, err := pacemaker.New(controller, modules.Notifier, modules.Persist) + pacemaker, err := pacemaker.New(controller, cfg.ProposalDurationProvider, modules.Notifier, modules.Persist, + pacemaker.WithQCs(qcCollector.Retrieve()...), + pacemaker.WithTCs(tcCollector.Retrieve()...), + ) if err != nil { return nil, fmt.Errorf("could not initialize flow pacemaker: %w", err) } @@ -100,22 +102,14 @@ func NewParticipant( } // initialize and return the event loop - loop, err := eventloop.NewEventLoop(log, metrics, eventHandler, cfg.StartupTime) + loop, err := eventloop.NewEventLoop(log, metrics, mempoolMetrics, eventHandler, cfg.StartupTime) if err != nil { return nil, fmt.Errorf("could not initialize event loop: %w", err) } // add observer, event loop needs to receive events from distributor - modules.QCCreatedDistributor.AddConsumer(loop) - modules.TimeoutCollectorDistributor.AddConsumer(loop) - - // register dynamically updatable configs - if cfg.Registrar != nil { - err = cfg.Registrar.RegisterDurationConfig("hotstuff-block-rate-delay", timeoutConfig.GetBlockRateDelay, timeoutConfig.SetBlockRateDelay) - if err != nil { - return nil, fmt.Errorf("failed to register block rate delay config: %w", err) - } - } + modules.VoteCollectorDistributor.AddVoteCollectorConsumer(loop) + modules.TimeoutCollectorDistributor.AddTimeoutCollectorConsumer(loop) return loop, nil } @@ -131,7 +125,7 @@ func NewValidator(metrics module.HotstuffMetrics, committee hotstuff.DynamicComm } // NewForks recovers trusted root and creates new forks manager -func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.FinalizationConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { +func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.FollowerConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { // recover the trusted root trustedRoot, err := recoverTrustedRoot(final, headers, rootHeader, rootQC) if err != nil { diff --git a/consensus/recovery/cluster/state.go b/consensus/recovery/cluster/state.go index aeae9bd9d6c..7cc8446190d 100644 --- a/consensus/recovery/cluster/state.go +++ b/consensus/recovery/cluster/state.go @@ -8,18 +8,24 @@ import ( "github.com/onflow/flow-go/storage" ) -// FindLatest retrieves the latest finalized header and all of its pending -// children. These pending children have been verified by the compliance layer -// but are NOT guaranteed to have been verified by HotStuff. They MUST be -// validated by HotStuff during the recovery process. +// FindLatest returns: +// - [first value] latest finalized header +// - [second value] all known descendants (i.e. pending blocks) +// - No errors expected during normal operations. +// +// All returned blocks have been verified by the compliance layer, i.e. they are guaranteed to be valid. +// The descendants are listed in ancestor-first order, i.e. for any block B = descendants[i], B's parent +// must be included at an index _smaller_ than i, unless B's parent is the latest finalized block. +// +// Note: this is an expensive method, which is intended to help recover from a crash, e.g. help to +// re-built the in-memory consensus state. func FindLatest(state cluster.State, headers storage.Headers) (*flow.Header, []*flow.Header, error) { - - finalized, err := state.Final().Head() + finalizedSnapshot := state.Final() // state snapshot at latest finalized block + finalizedBlock, err := finalizedSnapshot.Head() // header of latest finalized block if err != nil { return nil, nil, fmt.Errorf("could not get finalized header: %w", err) } - - pendingIDs, err := state.Final().Pending() + pendingIDs, err := finalizedSnapshot.Pending() // find IDs of all blocks descending from the finalized block if err != nil { return nil, nil, fmt.Errorf("could not get pending children: %w", err) } @@ -33,5 +39,5 @@ func FindLatest(state cluster.State, headers storage.Headers) (*flow.Header, []* pending = append(pending, header) } - return finalized, pending, nil + return finalizedBlock, pending, nil } diff --git a/consensus/recovery/follower.go b/consensus/recovery/follower.go deleted file mode 100644 index 6ad8ae1945c..00000000000 --- a/consensus/recovery/follower.go +++ /dev/null @@ -1,34 +0,0 @@ -package recovery - -import ( - "fmt" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" -) - -// Follower recovers the HotStuff state for a follower instance. -// It reads the pending blocks from storage and pass them to the input Forks -// instance to recover its state from before the restart. -func Follower( - log zerolog.Logger, - forks hotstuff.Forks, - validator hotstuff.Validator, - pending []*flow.Header, -) error { - return Recover(log, pending, validator, func(proposal *model.Proposal) error { - // add it to forks - err := forks.AddProposal(proposal) - if err != nil { - return fmt.Errorf("could not add block to forks: %w", err) - } - log.Debug(). - Uint64("block_view", proposal.Block.View). - Hex("block_id", proposal.Block.BlockID[:]). - Msg("block recovered") - return nil - }) -} diff --git a/consensus/recovery/participant.go b/consensus/recovery/participant.go deleted file mode 100644 index c19c6c578f7..00000000000 --- a/consensus/recovery/participant.go +++ /dev/null @@ -1,35 +0,0 @@ -package recovery - -import ( - "fmt" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" -) - -// Participant recovers the HotStuff state for a consensus participant. -// It reads the pending blocks from storage and pass them to the input Forks -// instance to recover its state from before the restart. -func Participant( - log zerolog.Logger, - forks hotstuff.Forks, - voteAggregator hotstuff.VoteAggregator, - validator hotstuff.Validator, - pending []*flow.Header, -) error { - return Recover(log, pending, validator, func(proposal *model.Proposal) error { - // add it to forks - err := forks.AddProposal(proposal) - if err != nil { - return fmt.Errorf("could not add block to forks: %w", err) - } - - // recover the proposer's vote - voteAggregator.AddBlock(proposal) - - return nil - }) -} diff --git a/consensus/recovery/protocol/state.go b/consensus/recovery/protocol/state.go index 18df422dbf3..1bbc20b1bf1 100644 --- a/consensus/recovery/protocol/state.go +++ b/consensus/recovery/protocol/state.go @@ -8,25 +8,29 @@ import ( "github.com/onflow/flow-go/storage" ) -// FindLatest retrieves the latest finalized header and all of its pending -// children. These pending children have been verified by the compliance layer -// but are NOT guaranteed to have been verified by HotStuff. They MUST be -// validated by HotStuff during the recovery process. +// FindLatest returns: +// - [first value] latest finalized header +// - [second value] all known descendants (i.e. pending blocks) +// - No errors expected during normal operations. +// +// All returned blocks have been verified by the compliance layer, i.e. they are guaranteed to be valid. +// The descendants are listed in ancestor-first order, i.e. for any block B = descendants[i], B's parent +// must be included at an index _smaller_ than i, unless B's parent is the latest finalized block. +// +// Note: this is an expensive method, which is intended to help recover from a crash, e.g. help to +// re-built the in-memory consensus state. func FindLatest(state protocol.State, headers storage.Headers) (*flow.Header, []*flow.Header, error) { - - // find finalized block - finalized, err := state.Final().Head() + finalizedSnapshot := state.Final() // state snapshot at latest finalized block + finalizedBlock, err := finalizedSnapshot.Head() // header of latest finalized block if err != nil { return nil, nil, fmt.Errorf("could not find finalized block") } - - // find all pending blockIDs - pendingIDs, err := state.Final().Descendants() + pendingIDs, err := finalizedSnapshot.Descendants() // find IDs of all blocks descending from the finalized block if err != nil { return nil, nil, fmt.Errorf("could not find pending block") } - // find all pending header by ID + // retrieve the headers for each of the pending blocks pending := make([]*flow.Header, 0, len(pendingIDs)) for _, pendingID := range pendingIDs { pendingHeader, err := headers.ByBlockID(pendingID) @@ -36,5 +40,5 @@ func FindLatest(state protocol.State, headers storage.Headers) (*flow.Header, [] pending = append(pending, pendingHeader) } - return finalized, pending, nil + return finalizedBlock, pending, nil } diff --git a/consensus/recovery/recover.go b/consensus/recovery/recover.go index fa5895ffbff..a470aedc3ce 100644 --- a/consensus/recovery/recover.go +++ b/consensus/recovery/recover.go @@ -1,7 +1,6 @@ package recovery import ( - "errors" "fmt" "github.com/rs/zerolog" @@ -9,52 +8,113 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/logging" ) -// Recover implements the core logic for recovering HotStuff state after a restart. -// It receives the list `pending` that should contain _all_ blocks that have been -// received but not finalized, and that share the latest finalized block as a common -// ancestor. -func Recover(log zerolog.Logger, pending []*flow.Header, validator hotstuff.Validator, onProposal func(*model.Proposal) error) error { +// BlockScanner describes a function for ingesting pending blocks. +// Any returned errors are considered fatal. +type BlockScanner func(proposal *model.Proposal) error + +// Recover is a utility method for recovering the HotStuff state after a restart. +// It receives the list `pending` containing _all_ blocks that +// - have passed the compliance layer and stored in the protocol state +// - descend from the latest finalized block +// - are listed in ancestor-first order (i.e. for any block B ∈ pending, B's parent must +// be listed before B, unless B's parent is the latest finalized block) +// +// CAUTION: all pending blocks are required to be valid (guaranteed if the block passed the compliance layer) +func Recover(log zerolog.Logger, pending []*flow.Header, scanners ...BlockScanner) error { log.Info().Int("total", len(pending)).Msgf("recovery started") // add all pending blocks to forks for _, header := range pending { + proposal := model.ProposalFromFlow(header) // convert the header into a proposal + for _, s := range scanners { + err := s(proposal) + if err != nil { + return fmt.Errorf("scanner failed to ingest proposal: %w", err) + } + } + log.Debug(). + Uint64("view", proposal.Block.View). + Hex("block_id", proposal.Block.BlockID[:]). + Msg("block recovered") + } + + log.Info().Msgf("recovery completed") + return nil +} - // convert the header into a proposal - proposal := model.ProposalFromFlow(header) - - // verify the proposal - err := validator.ValidateProposal(proposal) - if model.IsInvalidBlockError(err) { - log.Warn(). - Hex("block_id", logging.ID(proposal.Block.BlockID)). - Err(err). - Msg("invalid proposal") - continue +// ForksState recovers Forks' internal state of blocks descending from the latest +// finalized block. Caution, input blocks must be valid and in parent-first order +// (unless parent is the latest finalized block). +func ForksState(forks hotstuff.Forks) BlockScanner { + return func(proposal *model.Proposal) error { + err := forks.AddValidatedBlock(proposal.Block) + if err != nil { + return fmt.Errorf("could not add block %v to forks: %w", proposal.Block.BlockID, err) } - if errors.Is(err, model.ErrUnverifiableBlock) { - log.Warn(). - Hex("block_id", logging.ID(proposal.Block.BlockID)). - Hex("qc_block_id", logging.ID(proposal.Block.QC.BlockID)). - Msg("unverifiable proposal") - - // even if the block is unverifiable because the QC has been - // pruned, it still needs to be added to the forks, otherwise, - // a new block with a QC to this block will fail to be added - // to forks and crash the event loop. - } else if err != nil { - return fmt.Errorf("cannot validate proposal (%x): %w", proposal.Block.BlockID, err) + return nil + } +} + +// VoteAggregatorState recovers the VoteAggregator's internal state as follows: +// - Add all blocks descending from the latest finalized block to accept votes. +// Those blocks should be rapidly pruned as the node catches up. +// +// Caution: input blocks must be valid. +func VoteAggregatorState(voteAggregator hotstuff.VoteAggregator) BlockScanner { + return func(proposal *model.Proposal) error { + voteAggregator.AddBlock(proposal) + return nil + } +} + +// CollectParentQCs collects all parent QCs included in the blocks descending from the +// latest finalized block. Caution, input blocks must be valid. +func CollectParentQCs(collector Collector[*flow.QuorumCertificate]) BlockScanner { + return func(proposal *model.Proposal) error { + qc := proposal.Block.QC + if qc != nil { + collector.Append(qc) } + return nil + } +} - err = onProposal(proposal) - if err != nil { - return fmt.Errorf("cannot recover proposal: %w", err) +// CollectTCs collect all TCs included in the blocks descending from the +// latest finalized block. Caution, input blocks must be valid. +func CollectTCs(collector Collector[*flow.TimeoutCertificate]) BlockScanner { + return func(proposal *model.Proposal) error { + tc := proposal.LastViewTC + if tc != nil { + collector.Append(tc) } + return nil } +} - log.Info().Msgf("recovery completed") +// Collector for objects of generic type. Essentially, it is a stateful list. +// Safe to be passed by value. Retrieve() returns the current state of the list +// and is unaffected by subsequent appends. +type Collector[T any] struct { + list *[]T +} - return nil +func NewCollector[T any]() Collector[T] { + list := make([]T, 0, 5) // heuristic: pre-allocate with some basic capacity + return Collector[T]{list: &list} +} + +// Append adds new elements to the end of the list. +func (c Collector[T]) Append(t ...T) { + *c.list = append(*c.list, t...) +} + +// Retrieve returns the current state of the list (unaffected by subsequent append) +func (c Collector[T]) Retrieve() []T { + // Under the hood, the slice is a struct containing a pointer to an underlying array and a + // `len` variable indicating how many of the array elements are occupied. Here, we are + // returning the slice struct by value, i.e. we _copy_ the array pointer and the `len` value + // and return the copy. Therefore, the returned slice is unaffected by subsequent append. + return *c.list } diff --git a/consensus/recovery/recover_test.go b/consensus/recovery/recover_test.go index 3f337fb6da0..ac0fb0c3d4f 100644 --- a/consensus/recovery/recover_test.go +++ b/consensus/recovery/recover_test.go @@ -3,10 +3,8 @@ package recovery import ( "testing" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -15,41 +13,89 @@ import ( func TestRecover(t *testing.T) { finalized := unittest.BlockHeaderFixture() blocks := unittest.ChainFixtureFrom(100, finalized) - pending := make([]*flow.Header, 0) for _, b := range blocks { pending = append(pending, b.Header) } + + // Recover with `pending` blocks and record what blocks are forwarded to `onProposal` recovered := make([]*model.Proposal, 0) - onProposal := func(block *model.Proposal) error { + scanner := func(block *model.Proposal) error { recovered = append(recovered, block) return nil } + err := Recover(unittest.Logger(), pending, scanner) + require.NoError(t, err) - // make 3 invalid blocks extend from the last valid block - invalidblocks := unittest.ChainFixtureFrom(3, pending[len(pending)-1]) - invalid := make(map[flow.Identifier]struct{}) - for _, b := range invalidblocks { - invalid[b.ID()] = struct{}{} - pending = append(pending, b.Header) + // should forward blocks in exact order, just converting flow.Header to pending block + require.Len(t, recovered, len(pending)) + for i, r := range recovered { + require.Equal(t, model.ProposalFromFlow(pending[i]), r) } +} - validator := &mocks.Validator{} - validator.On("ValidateProposal", mock.Anything).Return(func(proposal *model.Proposal) error { - header := model.ProposalToFlow(proposal) - _, isInvalid := invalid[header.ID()] - if isInvalid { - return &model.InvalidBlockError{ - BlockID: header.ID(), - View: header.View, - } - } +func TestRecoverEmptyInput(t *testing.T) { + scanner := func(block *model.Proposal) error { + require.Fail(t, "no proposal expected") return nil + } + err := Recover(unittest.Logger(), []*flow.Header{}, scanner) + require.NoError(t, err) +} + +func TestCollector(t *testing.T) { + t.Run("empty retrieve", func(t *testing.T) { + c := NewCollector[string]() + require.Empty(t, c.Retrieve()) }) - err := Recover(unittest.Logger(), pending, validator, onProposal) - require.NoError(t, err) + t.Run("append", func(t *testing.T) { + c := NewCollector[string]() + strings := []string{"a", "b", "c"} + appended := 0 + for _, s := range strings { + c.Append(s) + appended++ + require.Equal(t, strings[:appended], c.Retrieve()) + } + }) - // only pending blocks are valid - require.Len(t, recovered, len(pending)) + t.Run("append multiple", func(t *testing.T) { + c := NewCollector[string]() + strings := []string{"a", "b", "c", "d", "e"} + + c.Append(strings[0], strings[1]) + require.Equal(t, strings[:2], c.Retrieve()) + + c.Append(strings[2], strings[3], strings[4]) + require.Equal(t, strings, c.Retrieve()) + }) + + t.Run("safely passed by value", func(t *testing.T) { + strings := []string{"a", "b"} + c := NewCollector[string]() + c.Append(strings[0]) + + // pass by value + c2 := c + require.Equal(t, strings[:1], c2.Retrieve()) + + // add to original; change could be reflected by c2: + c.Append(strings[1]) + require.Equal(t, strings, c2.Retrieve()) + }) + + t.Run("append after retrieve", func(t *testing.T) { + c := NewCollector[string]() + strings := []string{"a", "b", "c", "d", "e"} + + c.Append(strings[0], strings[1]) + retrieved := c.Retrieve() + require.Equal(t, strings[:2], retrieved) + + // appending further elements shouldn't affect previously retrieved list + c.Append(strings[2], strings[3], strings[4]) + require.Equal(t, strings[:2], retrieved) + require.Equal(t, strings, c.Retrieve()) + }) } diff --git a/crypto/Dockerfile b/crypto/Dockerfile index 37a0b373171..d75e9543de4 100644 --- a/crypto/Dockerfile +++ b/crypto/Dockerfile @@ -1,6 +1,6 @@ # gcr.io/dl-flow/golang-cmake -FROM golang:1.19-buster +FROM golang:1.20-buster RUN apt-get update RUN apt-get -y install cmake zip RUN go install github.com/axw/gocov/gocov@latest diff --git a/crypto/bls12381_utils.go b/crypto/bls12381_utils.go index 08a71e8cf5a..50676fc2c04 100644 --- a/crypto/bls12381_utils.go +++ b/crypto/bls12381_utils.go @@ -135,7 +135,7 @@ func mapToZr(x *scalar, src []byte) bool { // writeScalar writes a G2 point in a slice of bytes func writeScalar(dest []byte, x *scalar) { C.bn_write_bin((*C.uchar)(&dest[0]), - (C.int)(prKeyLengthBLSBLS12381), + (C.ulong)(prKeyLengthBLSBLS12381), (*C.bn_st)(x), ) } @@ -144,7 +144,7 @@ func writeScalar(dest []byte, x *scalar) { func readScalar(x *scalar, src []byte) { C.bn_read_bin((*C.bn_st)(x), (*C.uchar)(&src[0]), - (C.int)(len(src)), + (C.ulong)(len(src)), ) } diff --git a/crypto/bls_core.c b/crypto/bls_core.c index 4c87aa11496..32b56a5d03d 100644 --- a/crypto/bls_core.c +++ b/crypto/bls_core.c @@ -117,26 +117,6 @@ static int bls_verify_ep(const ep2_t pk, const ep_t s, const byte* data, const i // elemsG2[0] = -g2 ep2_neg(elemsG2[0], core_get()->ep2_g); // could be hardcoded - // TODO: temporary fix to delete once a bug in Relic is fixed - // The DOUBLE_PAIRING is still preferred over non-buggy SINGLE_PAIRING as - // the verification is 1.5x faster - // if sig=h then ret <- pk == g2 - if (ep_cmp(elemsG1[0], elemsG1[1])==RLC_EQ && ep2_cmp(elemsG2[1], core_get()->ep2_g)==RLC_EQ) { - ret = VALID; - goto out; - } - // if pk = -g2 then ret <- s == -h - if (ep2_cmp(elemsG2[0], elemsG2[1])==RLC_EQ) { - ep_st sum; ep_new(&sum); - ep_add(&sum, elemsG1[0], elemsG1[1]); - if (ep_is_infty(&sum)) { - ep_free(&sum); - ret = VALID; - goto out; - } - ep_free(&sum); - } - fp12_t pair; fp12_new(&pair); // double pairing with Optimal Ate @@ -525,9 +505,10 @@ void bls_batchVerify(const int sigs_len, byte* results, const ep2_st* pks_input, if ( read_ret != RLC_OK || check_membership_G1(&sigs[i]) != VALID) { if (read_ret == UNDEFINED) // unexpected error case goto out; - // set signature as infinity and set result as invald + // set signature and key to infinity (no effect on the aggregation tree) + // and set result to invalid ep_set_infty(&sigs[i]); - ep2_copy(&pks[i], (ep2_st*) &pks_input[i]); + ep2_set_infty(&pks[i]); results[i] = INVALID; // multiply signatures and public keys at the same index by random coefficients } else { diff --git a/crypto/bls_multisig.go b/crypto/bls_multisig.go index 1dfe29abc05..af2c6ce2f3c 100644 --- a/crypto/bls_multisig.go +++ b/crypto/bls_multisig.go @@ -455,8 +455,11 @@ func VerifyBLSSignatureManyMessages( // Each signature at index (i) of the input signature slice is verified against // the public key of the same index (i) in the input key slice. // The input hasher is the same used to generate all signatures. -// The returned boolean slice is a slice so that the value at index (i) is true -// if signature (i) verifies against public key (i), and false otherwise. +// The returned boolean slice is of the same length of the signatures slice, +// where the boolean at index (i) is true if signature (i) verifies against +// public key (i), and false otherwise. +// In the case where an error occurs during the execution of the function, +// all the returned boolean values are `false`. // // The caller must make sure the input public keys's proofs of possession have been // verified prior to calling this function (or each input key is sum of public @@ -482,50 +485,58 @@ func BatchVerifyBLSSignaturesOneMessage( // set BLS context blsInstance.reInit() + // boolean array returned when errors occur + falseSlice := make([]bool, len(sigs)) + // empty list check if len(pks) == 0 { - return []bool{}, fmt.Errorf("invalid list of public keys: %w", blsAggregateEmptyListError) + return falseSlice, fmt.Errorf("invalid list of public keys: %w", blsAggregateEmptyListError) } if len(pks) != len(sigs) { - return []bool{}, invalidInputsErrorf( + return falseSlice, invalidInputsErrorf( "keys length %d and signatures length %d are mismatching", len(pks), len(sigs)) } - verifBool := make([]bool, len(sigs)) if err := checkBLSHasher(kmac); err != nil { - return verifBool, err + return falseSlice, err } - // an invalid signature with an incorrect header but correct length - invalidSig := make([]byte, signatureLengthBLSBLS12381) - invalidSig[0] = invalidBLSSignatureHeader // incorrect header - // flatten the shares (required by the C layer) flatSigs := make([]byte, 0, signatureLengthBLSBLS12381*len(sigs)) pkPoints := make([]pointG2, 0, len(pks)) + getIdentityPoint := func() pointG2 { + pk, _ := IdentityBLSPublicKey().(*pubKeyBLSBLS12381) // second value is guaranteed to be true + return pk.point + } + + returnBool := make([]bool, len(sigs)) for i, pk := range pks { pkBLS, ok := pk.(*pubKeyBLSBLS12381) if !ok { - return verifBool, fmt.Errorf("key at index %d is invalid: %w", i, notBLSKeyError) + return falseSlice, fmt.Errorf("key at index %d is invalid: %w", i, notBLSKeyError) } - pkPoints = append(pkPoints, pkBLS.point) if len(sigs[i]) != signatureLengthBLSBLS12381 || pkBLS.isIdentity { - // force the signature to be invalid by replacing it with an invalid array - // that fails the deserialization in C.ep_read_bin_compact - flatSigs = append(flatSigs, invalidSig...) + // case of invalid signature: set the signature and public key at index `i` + // to identities so that there is no effect on the aggregation tree computation. + // However, the boolean return for index `i` is set to `false` and won't be overwritten. + returnBool[i] = false + pkPoints = append(pkPoints, getIdentityPoint()) + flatSigs = append(flatSigs, identityBLSSignature...) } else { + returnBool[i] = true // default to true + pkPoints = append(pkPoints, pkBLS.point) flatSigs = append(flatSigs, sigs[i]...) } } // hash the input to 128 bytes h := kmac.ComputeHash(message) - verifInt := make([]byte, len(verifBool)) + verifInt := make([]byte, len(sigs)) C.bls_batchVerify( (C.int)(len(verifInt)), @@ -538,12 +549,13 @@ func BatchVerifyBLSSignaturesOneMessage( for i, v := range verifInt { if (C.int)(v) != valid && (C.int)(v) != invalid { - return verifBool, fmt.Errorf("batch verification failed") + return falseSlice, fmt.Errorf("batch verification failed") + } + if returnBool[i] { // only overwrite if not previously set to false + returnBool[i] = ((C.int)(v) == valid) } - verifBool[i] = ((C.int)(v) == valid) } - - return verifBool, nil + return returnBool, nil } // blsAggregateEmptyListError is returned when a list of BLS objects (e.g. signatures or keys) diff --git a/crypto/bls_test.go b/crypto/bls_test.go index adb02d02a29..c967546f640 100644 --- a/crypto/bls_test.go +++ b/crypto/bls_test.go @@ -114,6 +114,13 @@ func invalidSK(t *testing.T) PrivateKey { return sk } +// Utility function that flips a point sign bit to negate the point +// this is shortcut which works only for zcash BLS12-381 compressed serialization +// Applicable to both signatures and public keys +func negatePoint(pointbytes []byte) { + pointbytes[0] ^= 0x20 +} + // BLS tests func TestBLSBLS12381Hasher(t *testing.T) { rand := getPRG(t) @@ -657,6 +664,27 @@ func TestBLSBatchVerify(t *testing.T) { sigs, sks, input, valid) }) + // valid signatures but indices aren't correct: sig[i] is correct under pks[j] + // and sig[j] is correct under pks[j]. + // implementations simply aggregating all signatures and keys would fail this test. + t.Run("valid signatures with incorrect indices", func(t *testing.T) { + i := mrand.Intn(sigsNum-1) + 1 + j := mrand.Intn(i) + // swap correct keys + pks[i], pks[j] = pks[j], pks[i] + + valid, err := BatchVerifyBLSSignaturesOneMessage(pks, sigs, input, kmac) + require.NoError(t, err) + expectedValid[i], expectedValid[j] = false, false + assert.Equal(t, valid, expectedValid, + "Verification of %s failed, private keys are %s, input is %x, results is %v", + sigs, sks, input, valid) + + // restore keys + pks[i], pks[j] = pks[j], pks[i] + expectedValid[i], expectedValid[j] = true, true + }) + // one valid signature t.Run("one valid signature", func(t *testing.T) { valid, err := BatchVerifyBLSSignaturesOneMessage(pks[:1], sigs[:1], input, kmac) @@ -680,7 +708,6 @@ func TestBLSBatchVerify(t *testing.T) { // some signatures are invalid t.Run("some signatures are invalid", func(t *testing.T) { - for i := 0; i < invalidSigsNum; i++ { // alter invalidSigsNum random signatures alterSignature(sigs[indices[i]]) expectedValid[indices[i]] = false @@ -715,16 +742,19 @@ func TestBLSBatchVerify(t *testing.T) { valid, err := BatchVerifyBLSSignaturesOneMessage(pks[:0], sigs[:0], input, kmac) require.Error(t, err) assert.True(t, IsBLSAggregateEmptyListError(err)) - assert.Equal(t, valid, []bool{}, + assert.Equal(t, valid, expectedValid[:0], "verification should fail with empty list key, got %v", valid) }) // test incorrect inputs t.Run("inconsistent inputs", func(t *testing.T) { + for i := 0; i < sigsNum; i++ { + expectedValid[i] = false + } valid, err := BatchVerifyBLSSignaturesOneMessage(pks[:len(pks)-1], sigs, input, kmac) require.Error(t, err) assert.True(t, IsInvalidInputsError(err)) - assert.Equal(t, valid, []bool{}, + assert.Equal(t, valid, expectedValid, "verification should fail with incorrect input lenghts, got %v", valid) }) @@ -975,7 +1005,7 @@ func TestBLSErrorTypes(t *testing.T) { // VerifyBLSSignatureManyMessages bench // Bench the slowest case where all messages and public keys are distinct. -// (2*n) pairings without aggrgetion Vs (n+1) pairings with aggregation. +// (2*n) pairings without aggregation Vs (n+1) pairings with aggregation. // The function is faster whenever there are redundant messages or public keys. func BenchmarkVerifySignatureManyMessages(b *testing.B) { // inputs @@ -1083,7 +1113,6 @@ func TestBLSIdentity(t *testing.T) { t.Run("identity signature comparison", func(t *testing.T) { // verify that constructed identity signatures are recognized as such by IsBLSSignatureIdentity. // construct identity signature by summing (aggregating) a random signature and its inverse. - assert.True(t, IsBLSSignatureIdentity(identityBLSSignature)) // sum up a random signature and its inverse to get identity @@ -1092,7 +1121,7 @@ func TestBLSIdentity(t *testing.T) { require.NoError(t, err) oppositeSig := make([]byte, signatureLengthBLSBLS12381) copy(oppositeSig, sig) - oppositeSig[0] ^= 0x20 // flip the last 3rd bit to flip the point sign + negatePoint(oppositeSig) aggSig, err := AggregateBLSSignatures([]Signature{sig, oppositeSig}) require.NoError(t, err) assert.True(t, IsBLSSignatureIdentity(aggSig)) diff --git a/crypto/build_dependency.sh b/crypto/build_dependency.sh index bd5d612e9cb..4bfe99dbad2 100644 --- a/crypto/build_dependency.sh +++ b/crypto/build_dependency.sh @@ -14,7 +14,7 @@ fi rm -rf "${RELIC_DIR}" # relic version or tag -relic_version="05feb20da8507260c9b3736dc1fd2efe7876d812" +relic_version="7d885d1ba34be61bf22190943a73549a910c1714" # clone a specific version of Relic without history if it's tagged. # git -c http.sslVerify=true clone --branch $(relic_version) --single-branch --depth 1 https://github.com/relic-toolkit/relic.git ${RELIC_DIR_NAME} || { echo "git clone failed"; exit 1; } diff --git a/crypto/ecdsa_test.go b/crypto/ecdsa_test.go index 342162668cf..cf9a137e1e7 100644 --- a/crypto/ecdsa_test.go +++ b/crypto/ecdsa_test.go @@ -159,7 +159,7 @@ func TestECDSAUtils(t *testing.T) { // TestScalarMult is a unit test of the scalar multiplication // This is only a sanity check meant to make sure the curve implemented -// is checked against an independant test vector +// is checked against an independent test vector func TestScalarMult(t *testing.T) { secp256k1 := secp256k1Instance.curve p256 := p256Instance.curve diff --git a/crypto/go.mod b/crypto/go.mod index c7fe54f9ff5..9895e1c35db 100644 --- a/crypto/go.mod +++ b/crypto/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/crypto -go 1.19 +go 1.20 require ( github.com/btcsuite/btcd/btcec/v2 v2.2.1 diff --git a/crypto/relic_build.sh b/crypto/relic_build.sh index 3045e22f59e..6cff3a6b478 100755 --- a/crypto/relic_build.sh +++ b/crypto/relic_build.sh @@ -63,9 +63,9 @@ PRIME=(-DFP_PRIME=381) # BN_METH=(-DBN_KARAT=0 -DBN_METHD="COMBA;COMBA;MONTY;SLIDE;BINAR;BASIC") FP_METH=(-DFP_KARAT=0 -DFP_METHD="INTEG;INTEG;INTEG;MONTY;MONTY;JMPDS;SLIDE") -PRIMES=(-DFP_PMERS=OFF -DFP_QNRES=ON -DFP_WIDTH=2) +PRIMES=(-DFP_PMERS=OFF -DFP_QNRES=ON) FPX_METH=(-DFPX_METHD="INTEG;INTEG;LAZYR") -EP_METH=(-DEP_MIXED=ON -DEP_PLAIN=OFF -DEP_ENDOM=ON -DEP_SUPER=OFF -DEP_DEPTH=4 -DEP_WIDTH=2 \ +EP_METH=(-DEP_MIXED=ON -DEP_PLAIN=OFF -DEP_ENDOM=ON -DEP_SUPER=OFF\ -DEP_CTMAP=ON -DEP_METHD="JACOB;LWNAF;COMBS;INTER") PP_METH=(-DPP_METHD="LAZYR;OATEP") diff --git a/docs/CruiseControl_BlockTimeController/EpochSimulation_000.png b/docs/CruiseControl_BlockTimeController/EpochSimulation_000.png new file mode 100644 index 00000000000..d9d852d7228 Binary files /dev/null and b/docs/CruiseControl_BlockTimeController/EpochSimulation_000.png differ diff --git a/docs/CruiseControl_BlockTimeController/EpochSimulation_005-0.png b/docs/CruiseControl_BlockTimeController/EpochSimulation_005-0.png new file mode 100644 index 00000000000..550b82fc3ae Binary files /dev/null and b/docs/CruiseControl_BlockTimeController/EpochSimulation_005-0.png differ diff --git a/docs/CruiseControl_BlockTimeController/EpochSimulation_005-1.png b/docs/CruiseControl_BlockTimeController/EpochSimulation_005-1.png new file mode 100644 index 00000000000..e058fbbe775 Binary files /dev/null and b/docs/CruiseControl_BlockTimeController/EpochSimulation_005-1.png differ diff --git a/docs/CruiseControl_BlockTimeController/EpochSimulation_028.png b/docs/CruiseControl_BlockTimeController/EpochSimulation_028.png new file mode 100644 index 00000000000..50dd514b5a2 Binary files /dev/null and b/docs/CruiseControl_BlockTimeController/EpochSimulation_028.png differ diff --git a/docs/CruiseControl_BlockTimeController/EpochSimulation_029.png b/docs/CruiseControl_BlockTimeController/EpochSimulation_029.png new file mode 100644 index 00000000000..bcee262c740 Binary files /dev/null and b/docs/CruiseControl_BlockTimeController/EpochSimulation_029.png differ diff --git a/docs/CruiseControl_BlockTimeController/EpochSimulation_030.png b/docs/CruiseControl_BlockTimeController/EpochSimulation_030.png new file mode 100644 index 00000000000..83f8f5c3833 Binary files /dev/null and b/docs/CruiseControl_BlockTimeController/EpochSimulation_030.png differ diff --git a/docs/CruiseControl_BlockTimeController/PID_controller_for_block-rate-delay.png b/docs/CruiseControl_BlockTimeController/PID_controller_for_block-rate-delay.png new file mode 100644 index 00000000000..78b6cb680f8 Binary files /dev/null and b/docs/CruiseControl_BlockTimeController/PID_controller_for_block-rate-delay.png differ diff --git a/docs/CruiseControl_BlockTimeController/ViewDurationConvention.png b/docs/CruiseControl_BlockTimeController/ViewDurationConvention.png new file mode 100644 index 00000000000..549a8241d9f Binary files /dev/null and b/docs/CruiseControl_BlockTimeController/ViewDurationConvention.png differ diff --git a/docs/CruiseControl_BlockTimeController/ViewRate.png b/docs/CruiseControl_BlockTimeController/ViewRate.png new file mode 100644 index 00000000000..003de946b15 Binary files /dev/null and b/docs/CruiseControl_BlockTimeController/ViewRate.png differ diff --git a/docs/ErrorHandling.png b/docs/ErrorHandling.png new file mode 100644 index 00000000000..2d75c31aca3 Binary files /dev/null and b/docs/ErrorHandling.png differ diff --git a/engine/Readme.md b/engine/Readme.md index 8faebe0b332..cd082cdf557 100644 --- a/engine/Readme.md +++ b/engine/Readme.md @@ -1,5 +1,4 @@ # Notifier - The Notifier implements the following state machine ![Notifier State Machine](/docs/NotifierStateMachine.png) diff --git a/engine/access/access_test.go b/engine/access/access_test.go index 6c16f01fc00..b3979979fb9 100644 --- a/engine/access/access_test.go +++ b/engine/access/access_test.go @@ -19,22 +19,23 @@ import ( "google.golang.org/protobuf/testing/protocmp" "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/cmd/build" hsmock "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/access/ingestion" accessmock "github.com/onflow/flow-go/engine/access/mock" - "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" factorymock "github.com/onflow/flow-go/engine/access/rpc/backend/mock" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/mempool/stdmap" "github.com/onflow/flow-go/module/metrics" - module "github.com/onflow/flow-go/module/mock" + mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/mocknetwork" @@ -44,25 +45,30 @@ import ( "github.com/onflow/flow-go/storage/badger/operation" "github.com/onflow/flow-go/storage/util" "github.com/onflow/flow-go/utils/unittest" + "github.com/onflow/flow-go/utils/unittest/mocks" ) type Suite struct { suite.Suite state *protocol.State - snapshot *protocol.Snapshot + sealedSnapshot *protocol.Snapshot + finalSnapshot *protocol.Snapshot epochQuery *protocol.EpochQuery params *protocol.Params signerIndicesDecoder *hsmock.BlockSignerDecoder signerIds flow.IdentifierList log zerolog.Logger net *mocknetwork.Network - request *module.Requester + request *mockmodule.Requester collClient *accessmock.AccessAPIClient execClient *accessmock.ExecutionAPIClient - me *module.Local + me *mockmodule.Local rootBlock *flow.Header + sealedBlock *flow.Header + finalizedBlock *flow.Header chainID flow.ChainID metrics *metrics.NoopCollector + finalizedHeaderCache module.FinalizedHeaderCache backend *backend.Backend } @@ -76,25 +82,41 @@ func (suite *Suite) SetupTest() { suite.log = zerolog.New(os.Stderr) suite.net = new(mocknetwork.Network) suite.state = new(protocol.State) - suite.snapshot = new(protocol.Snapshot) + suite.finalSnapshot = new(protocol.Snapshot) + suite.sealedSnapshot = new(protocol.Snapshot) + + suite.rootBlock = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(0)) + suite.sealedBlock = suite.rootBlock + suite.finalizedBlock = unittest.BlockHeaderWithParentFixture(suite.sealedBlock) suite.epochQuery = new(protocol.EpochQuery) - suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() - suite.state.On("Final").Return(suite.snapshot, nil).Maybe() - suite.snapshot.On("Epochs").Return(suite.epochQuery).Maybe() + suite.state.On("Sealed").Return(suite.sealedSnapshot, nil).Maybe() + suite.state.On("Final").Return(suite.finalSnapshot, nil).Maybe() + suite.finalSnapshot.On("Epochs").Return(suite.epochQuery).Maybe() + suite.sealedSnapshot.On("Head").Return( + func() *flow.Header { + return suite.sealedBlock + }, + nil, + ).Maybe() + suite.finalSnapshot.On("Head").Return( + func() *flow.Header { + return suite.finalizedBlock + }, + nil, + ).Maybe() - suite.rootBlock = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(0)) suite.params = new(protocol.Params) - suite.params.On("Root").Return(suite.rootBlock, nil) + suite.params.On("FinalizedRoot").Return(suite.rootBlock, nil) suite.params.On("SporkRootBlockHeight").Return(suite.rootBlock.Height, nil) suite.state.On("Params").Return(suite.params).Maybe() suite.collClient = new(accessmock.AccessAPIClient) suite.execClient = new(accessmock.ExecutionAPIClient) - suite.request = new(module.Requester) + suite.request = new(mockmodule.Requester) suite.request.On("EntityByID", mock.Anything, mock.Anything) - suite.me = new(module.Local) + suite.me = new(mockmodule.Local) suite.signerIds = unittest.IdentifierListFixture(4) suite.signerIndicesDecoder = new(hsmock.BlockSignerDecoder) @@ -107,6 +129,7 @@ func (suite *Suite) SetupTest() { suite.chainID = flow.Testnet suite.metrics = metrics.NewNoopCollector() + suite.finalizedHeaderCache = mocks.NewFinalizedHeaderCache(suite.T(), suite.state) } func (suite *Suite) RunTest( @@ -133,9 +156,10 @@ func (suite *Suite) RunTest( nil, suite.log, backend.DefaultSnapshotHistoryLimit, + nil, + false, ) - - handler := access.NewHandler(suite.backend, suite.chainID.Chain(), access.WithBlockSignerDecoder(suite.signerIndicesDecoder)) + handler := access.NewHandler(suite.backend, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me, access.WithBlockSignerDecoder(suite.signerIndicesDecoder)) f(handler, db, all) }) } @@ -157,7 +181,7 @@ func (suite *Suite) TestSendAndGetTransaction() { Return(referenceBlock, nil). Twice() - suite.snapshot. + suite.finalSnapshot. On("Head"). Return(referenceBlock, nil). Once() @@ -195,15 +219,14 @@ func (suite *Suite) TestSendAndGetTransaction() { func (suite *Suite) TestSendExpiredTransaction() { suite.RunTest(func(handler *access.Handler, _ *badger.DB, _ *storage.All) { - referenceBlock := unittest.BlockHeaderFixture() + referenceBlock := suite.finalizedBlock + transaction := unittest.TransactionFixture() + transaction.SetReferenceBlockID(referenceBlock.ID()) // create latest block that is past the expiry window latestBlock := unittest.BlockHeaderFixture() latestBlock.Height = referenceBlock.Height + flow.DefaultTransactionExpiry*2 - transaction := unittest.TransactionFixture() - transaction.SetReferenceBlockID(referenceBlock.ID()) - refSnapshot := new(protocol.Snapshot) suite.state. @@ -215,10 +238,8 @@ func (suite *Suite) TestSendExpiredTransaction() { Return(referenceBlock, nil). Twice() - suite.snapshot. - On("Head"). - Return(latestBlock, nil). - Once() + //Advancing final state to expire ref block + suite.finalizedBlock = latestBlock req := &accessproto.SendTransactionRequest{ Transaction: convert.TransactionToMessage(transaction.TransactionBody), @@ -243,9 +264,9 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() { transaction := unittest.TransactionFixture() transaction.SetReferenceBlockID(referenceBlock.ID()) - // setup the state and snapshot mock expectations - suite.state.On("AtBlockID", referenceBlock.ID()).Return(suite.snapshot, nil) - suite.snapshot.On("Head").Return(referenceBlock, nil) + // setup the state and finalSnapshot mock expectations + suite.state.On("AtBlockID", referenceBlock.ID()).Return(suite.finalSnapshot, nil) + suite.finalSnapshot.On("Head").Return(referenceBlock, nil) // create storage metrics := metrics.NewNoopCollector() @@ -308,9 +329,11 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() { nil, suite.log, backend.DefaultSnapshotHistoryLimit, + nil, + false, ) - handler := access.NewHandler(backend, suite.chainID.Chain()) + handler := access.NewHandler(backend, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me) // Send transaction 1 resp, err := handler.SendTransaction(context.Background(), sendReq1) @@ -362,7 +385,11 @@ func (suite *Suite) TestGetBlockByIDAndHeight() { err := db.Update(operation.IndexBlockHeight(block2.Header.Height, block2.ID())) require.NoError(suite.T(), err) - assertHeaderResp := func(resp *accessproto.BlockHeaderResponse, err error, header *flow.Header) { + assertHeaderResp := func( + resp *accessproto.BlockHeaderResponse, + err error, + header *flow.Header, + ) { require.NoError(suite.T(), err) require.NotNil(suite.T(), resp) actual := resp.Block @@ -374,7 +401,11 @@ func (suite *Suite) TestGetBlockByIDAndHeight() { require.Equal(suite.T(), expectedBlockHeader, header) } - assertBlockResp := func(resp *accessproto.BlockResponse, err error, block *flow.Block) { + assertBlockResp := func( + resp *accessproto.BlockResponse, + err error, + block *flow.Block, + ) { require.NoError(suite.T(), err) require.NotNil(suite.T(), resp) actual := resp.Block @@ -386,7 +417,11 @@ func (suite *Suite) TestGetBlockByIDAndHeight() { require.Equal(suite.T(), expectedBlock.ID(), block.ID()) } - assertLightBlockResp := func(resp *accessproto.BlockResponse, err error, block *flow.Block) { + assertLightBlockResp := func( + resp *accessproto.BlockResponse, + err error, + block *flow.Block, + ) { require.NoError(suite.T(), err) require.NotNil(suite.T(), resp) actual := resp.Block @@ -394,7 +429,7 @@ func (suite *Suite) TestGetBlockByIDAndHeight() { require.Equal(suite.T(), expectedMessage, actual) } - suite.snapshot.On("Head").Return(block1.Header, nil) + suite.finalSnapshot.On("Head").Return(block1.Header, nil) suite.Run("get header 1 by ID", func() { // get header by ID id := block1.ID() @@ -479,12 +514,16 @@ func (suite *Suite) TestGetExecutionResultByBlockID() { er := unittest.ExecutionResultFixture( unittest.WithExecutionResultBlockID(blockID), - unittest.WithServiceEvents(2)) + unittest.WithServiceEvents(3)) require.NoError(suite.T(), all.Results.Store(er)) require.NoError(suite.T(), all.Results.Index(blockID, er.ID())) - assertResp := func(resp *accessproto.ExecutionResultForBlockIDResponse, err error, executionResult *flow.ExecutionResult) { + assertResp := func( + resp *accessproto.ExecutionResultForBlockIDResponse, + err error, + executionResult *flow.ExecutionResult, + ) { require.NoError(suite.T(), err) require.NotNil(suite.T(), resp) er := resp.ExecutionResult @@ -508,7 +547,7 @@ func (suite *Suite) TestGetExecutionResultByBlockID() { } for i, serviceEvent := range executionResult.ServiceEvents { - assert.Equal(suite.T(), serviceEvent.Type, er.ServiceEvents[i].Type) + assert.Equal(suite.T(), serviceEvent.Type.String(), er.ServiceEvents[i].Type) event := serviceEvent.Event marshalledEvent, err := json.Marshal(event) @@ -557,7 +596,7 @@ func (suite *Suite) TestGetSealedTransaction() { results := bstorage.NewExecutionResults(suite.metrics, db) receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize) enIdentities := unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution)) - enNodeIDs := flow.IdentifierList(enIdentities.NodeIDs()) + enNodeIDs := enIdentities.NodeIDs() // create block -> collection -> transactions block, collection := suite.createChain() @@ -569,19 +608,17 @@ func (suite *Suite) TestGetSealedTransaction() { Once() suite.request.On("Request", mock.Anything, mock.Anything).Return() - suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() - colIdentities := unittest.IdentityListFixture(1, unittest.WithRole(flow.RoleCollection)) allIdentities := append(colIdentities, enIdentities...) - suite.snapshot.On("Identities", mock.Anything).Return(allIdentities, nil).Once() + suite.finalSnapshot.On("Identities", mock.Anything).Return(allIdentities, nil).Once() exeEventResp := execproto.GetTransactionResultResponse{ Events: nil, } // generate receipts - executionReceipts := unittest.ReceiptsForBlockFixture(&block, enNodeIDs) + executionReceipts := unittest.ReceiptsForBlockFixture(block, enNodeIDs) // assume execution node returns an empty list of events suite.execClient.On("GetTransactionResult", mock.Anything, mock.Anything).Return(&exeEventResp, nil) @@ -619,25 +656,21 @@ func (suite *Suite) TestGetSealedTransaction() { enNodeIDs.Strings(), suite.log, backend.DefaultSnapshotHistoryLimit, + nil, + false, ) - handler := access.NewHandler(backend, suite.chainID.Chain()) - - rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, rpc.Config{}, nil, nil, all.Blocks, all.Headers, collections, transactions, receipts, - results, suite.chainID, metrics, metrics, 0, 0, false, false, nil, nil) - require.NoError(suite.T(), err) - rpcEng, err := rpcEngBuilder.WithLegacy().Build() - require.NoError(suite.T(), err) + handler := access.NewHandler(backend, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me) // create the ingest engine ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections, - transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted, rpcEng) + transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted) require.NoError(suite.T(), err) // 1. Assume that follower engine updated the block storage and the protocol state. The block is reported as sealed - err = all.Blocks.Store(&block) + err = all.Blocks.Store(block) require.NoError(suite.T(), err) - suite.snapshot.On("Head").Return(block.Header, nil).Twice() + suite.sealedBlock = block.Header background, cancel := context.WithCancel(context.Background()) defer cancel() @@ -655,9 +688,8 @@ func (suite *Suite) TestGetSealedTransaction() { // 3. Request engine is used to request missing collection suite.request.On("EntityByID", collection.ID(), mock.Anything).Return() - // 4. Ingest engine receives the requested collection and all the execution receipts - ingestEng.OnCollection(originID, &collection) + ingestEng.OnCollection(originID, collection) for _, r := range executionReceipts { err = ingestEng.Process(channels.ReceiveReceipts, enNodeIDs[0], r) @@ -677,6 +709,249 @@ func (suite *Suite) TestGetSealedTransaction() { }) } +// TestGetTransactionResult tests different approaches to using the GetTransactionResult query, including using +// transaction ID, block ID, and collection ID. +func (suite *Suite) TestGetTransactionResult() { + unittest.RunWithBadgerDB(suite.T(), func(db *badger.DB) { + all := util.StorageLayer(suite.T(), db) + results := bstorage.NewExecutionResults(suite.metrics, db) + receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize) + + originID := unittest.IdentifierFixture() + + *suite.state = protocol.State{} + + // create block -> collection -> transactions + block, collection := suite.createChain() + blockNegative, collectionNegative := suite.createChain() + blockId := block.ID() + blockNegativeId := blockNegative.ID() + + finalSnapshot := new(protocol.Snapshot) + finalSnapshot.On("Head").Return(block.Header, nil) + + suite.state.On("Params").Return(suite.params) + suite.state.On("Final").Return(finalSnapshot) + suite.state.On("Sealed").Return(suite.sealedSnapshot) + sealedBlock := unittest.GenesisFixture().Header + // specifically for this test we will consider that sealed block is far behind finalized, so we get EXECUTED status + suite.sealedSnapshot.On("Head").Return(sealedBlock, nil) + + err := all.Blocks.Store(block) + require.NoError(suite.T(), err) + err = all.Blocks.Store(blockNegative) + require.NoError(suite.T(), err) + + suite.state.On("AtBlockID", blockId).Return(suite.sealedSnapshot) + + colIdentities := unittest.IdentityListFixture(1, unittest.WithRole(flow.RoleCollection)) + enIdentities := unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution)) + + enNodeIDs := enIdentities.NodeIDs() + allIdentities := append(colIdentities, enIdentities...) + finalSnapshot.On("Identities", mock.Anything).Return(allIdentities, nil) + + // assume execution node returns an empty list of events + suite.execClient.On("GetTransactionResult", mock.Anything, mock.Anything).Return(&execproto.GetTransactionResultResponse{ + Events: nil, + }, nil) + + // setup mocks + conduit := new(mocknetwork.Conduit) + suite.net.On("Register", channels.ReceiveReceipts, mock.Anything).Return(conduit, nil).Once() + suite.request.On("Request", mock.Anything, mock.Anything).Return() + + // create a mock connection factory + connFactory := new(factorymock.ConnectionFactory) + connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) + + // initialize storage + metrics := metrics.NewNoopCollector() + transactions := bstorage.NewTransactions(metrics, db) + collections := bstorage.NewCollections(db, transactions) + err = collections.Store(collectionNegative) + require.NoError(suite.T(), err) + collectionsToMarkFinalized, err := stdmap.NewTimes(100) + require.NoError(suite.T(), err) + collectionsToMarkExecuted, err := stdmap.NewTimes(100) + require.NoError(suite.T(), err) + blocksToMarkExecuted, err := stdmap.NewTimes(100) + require.NoError(suite.T(), err) + + backend := backend.New(suite.state, + suite.collClient, + nil, + all.Blocks, + all.Headers, + collections, + transactions, + receipts, + results, + suite.chainID, + suite.metrics, + connFactory, + false, + backend.DefaultMaxHeightRange, + nil, + enNodeIDs.Strings(), + suite.log, + backend.DefaultSnapshotHistoryLimit, + nil, + false, + ) + + handler := access.NewHandler(backend, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me) + + // create the ingest engine + ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections, + transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted) + require.NoError(suite.T(), err) + + background, cancel := context.WithCancel(context.Background()) + defer cancel() + + ctx := irrecoverable.NewMockSignalerContext(suite.T(), background) + ingestEng.Start(ctx) + <-ingestEng.Ready() + + processExecutionReceipts := func( + block *flow.Block, + collection *flow.Collection, + enNodeIDs flow.IdentifierList, + originID flow.Identifier, + ingestEng *ingestion.Engine, + ) { + executionReceipts := unittest.ReceiptsForBlockFixture(block, enNodeIDs) + // Ingest engine was notified by the follower engine about a new block. + // Follower engine --> Ingest engine + mb := &model.Block{ + BlockID: block.ID(), + } + ingestEng.OnFinalizedBlock(mb) + + // Ingest engine receives the requested collection and all the execution receipts + ingestEng.OnCollection(originID, collection) + + for _, r := range executionReceipts { + err = ingestEng.Process(channels.ReceiveReceipts, enNodeIDs[0], r) + require.NoError(suite.T(), err) + } + } + processExecutionReceipts(block, collection, enNodeIDs, originID, ingestEng) + processExecutionReceipts(blockNegative, collectionNegative, enNodeIDs, originID, ingestEng) + + txId := collection.Transactions[0].ID() + collectionId := collection.ID() + txIdNegative := collectionNegative.Transactions[0].ID() + collectionIdNegative := collectionNegative.ID() + + assertTransactionResult := func( + resp *accessproto.TransactionResultResponse, + err error, + ) { + require.NoError(suite.T(), err) + actualTxId := flow.HashToID(resp.TransactionId) + require.Equal(suite.T(), txId, actualTxId) + actualBlockId := flow.HashToID(resp.BlockId) + require.Equal(suite.T(), blockId, actualBlockId) + actualCollectionId := flow.HashToID(resp.CollectionId) + require.Equal(suite.T(), collectionId, actualCollectionId) + } + + // Test behaviour with transactionId provided + // POSITIVE + suite.Run("Get transaction result by transaction ID", func() { + getReq := &accessproto.GetTransactionRequest{ + Id: txId[:], + } + resp, err := handler.GetTransactionResult(context.Background(), getReq) + assertTransactionResult(resp, err) + }) + + // Test behaviour with blockId provided + suite.Run("Get transaction result by block ID", func() { + getReq := &accessproto.GetTransactionRequest{ + Id: txId[:], + BlockId: blockId[:], + } + resp, err := handler.GetTransactionResult(context.Background(), getReq) + assertTransactionResult(resp, err) + }) + + suite.Run("Get transaction result with wrong transaction ID and correct block ID", func() { + getReq := &accessproto.GetTransactionRequest{ + Id: txIdNegative[:], + BlockId: blockId[:], + } + resp, err := handler.GetTransactionResult(context.Background(), getReq) + require.Error(suite.T(), err) + require.Nil(suite.T(), resp) + }) + + suite.Run("Get transaction result with wrong block ID and correct transaction ID", func() { + getReq := &accessproto.GetTransactionRequest{ + Id: txId[:], + BlockId: blockNegativeId[:], + } + resp, err := handler.GetTransactionResult(context.Background(), getReq) + require.Error(suite.T(), err) + require.Nil(suite.T(), resp) + }) + + // Test behaviour with collectionId provided + suite.Run("Get transaction result by collection ID", func() { + getReq := &accessproto.GetTransactionRequest{ + Id: txId[:], + CollectionId: collectionId[:], + } + resp, err := handler.GetTransactionResult(context.Background(), getReq) + assertTransactionResult(resp, err) + }) + + suite.Run("Get transaction result with wrong collection ID but correct transaction ID", func() { + getReq := &accessproto.GetTransactionRequest{ + Id: txId[:], + CollectionId: collectionIdNegative[:], + } + resp, err := handler.GetTransactionResult(context.Background(), getReq) + require.Error(suite.T(), err) + require.Nil(suite.T(), resp) + }) + + suite.Run("Get transaction result with wrong transaction ID and correct collection ID", func() { + getReq := &accessproto.GetTransactionRequest{ + Id: txIdNegative[:], + CollectionId: collectionId[:], + } + resp, err := handler.GetTransactionResult(context.Background(), getReq) + require.Error(suite.T(), err) + require.Nil(suite.T(), resp) + }) + + // Test behaviour with blockId and collectionId provided + suite.Run("Get transaction result by block ID and collection ID", func() { + getReq := &accessproto.GetTransactionRequest{ + Id: txId[:], + BlockId: blockId[:], + CollectionId: collectionId[:], + } + resp, err := handler.GetTransactionResult(context.Background(), getReq) + assertTransactionResult(resp, err) + }) + + suite.Run("Get transaction result by block ID with wrong collection ID", func() { + getReq := &accessproto.GetTransactionRequest{ + Id: txId[:], + BlockId: blockId[:], + CollectionId: collectionIdNegative[:], + } + resp, err := handler.GetTransactionResult(context.Background(), getReq) + require.Error(suite.T(), err) + require.Nil(suite.T(), resp) + }) + }) +} + // TestExecuteScript tests the three execute Script related calls to make sure that the execution api is called with // the correct block id func (suite *Suite) TestExecuteScript() { @@ -688,7 +963,8 @@ func (suite *Suite) TestExecuteScript() { receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize) identities := unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution)) - suite.snapshot.On("Identities", mock.Anything).Return(identities, nil) + suite.sealedSnapshot.On("Identities", mock.Anything).Return(identities, nil) + suite.finalSnapshot.On("Identities", mock.Anything).Return(identities, nil) // create a mock connection factory connFactory := new(factorymock.ConnectionFactory) @@ -712,9 +988,11 @@ func (suite *Suite) TestExecuteScript() { flow.IdentifierList(identities.NodeIDs()).Strings(), suite.log, backend.DefaultSnapshotHistoryLimit, + nil, + false, ) - handler := access.NewHandler(suite.backend, suite.chainID.Chain()) + handler := access.NewHandler(suite.backend, suite.chainID.Chain(), suite.finalizedHeaderCache, suite.me) // initialize metrics related storage metrics := metrics.NewNoopCollector() @@ -730,36 +1008,35 @@ func (suite *Suite) TestExecuteScript() { Once() // create the ingest engine ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections, - transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted, nil) + transactions, results, receipts, metrics, collectionsToMarkFinalized, collectionsToMarkExecuted, blocksToMarkExecuted) require.NoError(suite.T(), err) + // create another block as a predecessor of the block created earlier + prevBlock := unittest.BlockWithParentFixture(suite.finalizedBlock) + // create a block and a seal pointing to that block - lastBlock := unittest.BlockFixture() - lastBlock.Header.Height = 2 - err = all.Blocks.Store(&lastBlock) + lastBlock := unittest.BlockWithParentFixture(prevBlock.Header) + err = all.Blocks.Store(lastBlock) require.NoError(suite.T(), err) err = db.Update(operation.IndexBlockHeight(lastBlock.Header.Height, lastBlock.ID())) require.NoError(suite.T(), err) - suite.snapshot.On("Head").Return(lastBlock.Header, nil).Once() - + //update latest sealed block + suite.sealedBlock = lastBlock.Header // create execution receipts for each of the execution node and the last block - executionReceipts := unittest.ReceiptsForBlockFixture(&lastBlock, identities.NodeIDs()) + executionReceipts := unittest.ReceiptsForBlockFixture(lastBlock, identities.NodeIDs()) // notify the ingest engine about the receipts for _, r := range executionReceipts { err = ingestEng.ProcessLocal(r) require.NoError(suite.T(), err) } - // create another block as a predecessor of the block created earlier - prevBlock := unittest.BlockFixture() - prevBlock.Header.Height = lastBlock.Header.Height - 1 - err = all.Blocks.Store(&prevBlock) + err = all.Blocks.Store(prevBlock) require.NoError(suite.T(), err) err = db.Update(operation.IndexBlockHeight(prevBlock.Header.Height, prevBlock.ID())) require.NoError(suite.T(), err) // create execution receipts for each of the execution node and the previous block - executionReceipts = unittest.ReceiptsForBlockFixture(&prevBlock, identities.NodeIDs()) + executionReceipts = unittest.ReceiptsForBlockFixture(prevBlock, identities.NodeIDs()) // notify the ingest engine about the receipts for _, r := range executionReceipts { err = ingestEng.ProcessLocal(r) @@ -783,8 +1060,17 @@ func (suite *Suite) TestExecuteScript() { suite.execClient.On("ExecuteScriptAtBlockID", ctx, &executionReq).Return(&executionResp, nil).Once() + finalizedHeader := suite.finalizedHeaderCache.Get() + finalizedHeaderId := finalizedHeader.ID() + nodeId := suite.me.NodeID() + expectedResp := accessproto.ExecuteScriptResponse{ Value: executionResp.GetValue(), + Metadata: &entitiesproto.Metadata{ + LatestFinalizedBlockId: finalizedHeaderId[:], + LatestFinalizedHeight: finalizedHeader.Height, + NodeId: nodeId[:], + }, } return &expectedResp } @@ -796,10 +1082,9 @@ func (suite *Suite) TestExecuteScript() { } suite.Run("execute script at latest block", func() { - suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() suite.state. On("AtBlockID", lastBlock.ID()). - Return(suite.snapshot, nil) + Return(suite.sealedSnapshot, nil) expectedResp := setupExecClientMock(lastBlock.ID()) req := accessproto.ExecuteScriptAtLatestBlockRequest{ @@ -812,7 +1097,7 @@ func (suite *Suite) TestExecuteScript() { suite.Run("execute script at block id", func() { suite.state. On("AtBlockID", prevBlock.ID()). - Return(suite.snapshot, nil) + Return(suite.sealedSnapshot, nil) expectedResp := setupExecClientMock(prevBlock.ID()) id := prevBlock.ID() @@ -827,7 +1112,7 @@ func (suite *Suite) TestExecuteScript() { suite.Run("execute script at block height", func() { suite.state. On("AtBlockID", prevBlock.ID()). - Return(suite.snapshot, nil) + Return(suite.sealedSnapshot, nil) expectedResp := setupExecClientMock(prevBlock.ID()) req := accessproto.ExecuteScriptAtBlockHeightRequest{ @@ -840,7 +1125,72 @@ func (suite *Suite) TestExecuteScript() { }) } -func (suite *Suite) createChain() (flow.Block, flow.Collection) { +// TestAPICallNodeVersionInfo tests the GetNodeVersionInfo query and check response returns correct node version +// information +func (suite *Suite) TestAPICallNodeVersionInfo() { + suite.RunTest(func(handler *access.Handler, db *badger.DB, all *storage.All) { + sporkId := unittest.IdentifierFixture() + protocolVersion := uint(unittest.Uint64InRange(10, 30)) + + suite.params.On("SporkID").Return(sporkId, nil) + suite.params.On("ProtocolVersion").Return(protocolVersion, nil) + + req := &accessproto.GetNodeVersionInfoRequest{} + resp, err := handler.GetNodeVersionInfo(context.Background(), req) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), resp) + + respNodeVersionInfo := resp.Info + suite.Require().Equal(respNodeVersionInfo, &entitiesproto.NodeVersionInfo{ + Semver: build.Version(), + Commit: build.Commit(), + SporkId: sporkId[:], + ProtocolVersion: uint64(protocolVersion), + }) + }) +} + +// TestLastFinalizedBlockHeightResult tests on example of the GetBlockHeaderByID function that the LastFinalizedBlock +// field in the response matches the finalized header from cache. It also tests that the LastFinalizedBlock field is +// updated correctly when a block with a greater height is finalized. +func (suite *Suite) TestLastFinalizedBlockHeightResult() { + suite.RunTest(func(handler *access.Handler, db *badger.DB, all *storage.All) { + block := unittest.BlockWithParentFixture(suite.finalizedBlock) + newFinalizedBlock := unittest.BlockWithParentFixture(block.Header) + + // store new block + require.NoError(suite.T(), all.Blocks.Store(block)) + + assertFinalizedBlockHeader := func(resp *accessproto.BlockHeaderResponse, err error) { + require.NoError(suite.T(), err) + require.NotNil(suite.T(), resp) + + finalizedHeaderId := suite.finalizedBlock.ID() + nodeId := suite.me.NodeID() + + require.Equal(suite.T(), &entitiesproto.Metadata{ + LatestFinalizedBlockId: finalizedHeaderId[:], + LatestFinalizedHeight: suite.finalizedBlock.Height, + NodeId: nodeId[:], + }, resp.Metadata) + } + + id := block.ID() + req := &accessproto.GetBlockHeaderByIDRequest{ + Id: id[:], + } + + resp, err := handler.GetBlockHeaderByID(context.Background(), req) + assertFinalizedBlockHeader(resp, err) + + suite.finalizedBlock = newFinalizedBlock.Header + + resp, err = handler.GetBlockHeaderByID(context.Background(), req) + assertFinalizedBlockHeader(resp, err) + }) +} + +func (suite *Suite) createChain() (*flow.Block, *flow.Collection) { collection := unittest.CollectionFixture(10) refBlockID := unittest.IdentifierFixture() // prepare cluster committee members @@ -855,9 +1205,8 @@ func (suite *Suite) createChain() (flow.Block, flow.Collection) { ReferenceBlockID: refBlockID, SignerIndices: indices, } - block := unittest.BlockFixture() - block.Payload.Guarantees = []*flow.CollectionGuarantee{guarantee} - block.Header.PayloadHash = block.Payload.Hash() + block := unittest.BlockWithParentFixture(suite.finalizedBlock) + block.SetPayload(unittest.PayloadFixture(unittest.WithGuarantees(guarantee))) cluster := new(protocol.Cluster) cluster.On("Members").Return(clusterCommittee, nil) @@ -865,13 +1214,12 @@ func (suite *Suite) createChain() (flow.Block, flow.Collection) { epoch.On("ClusterByChainID", mock.Anything).Return(cluster, nil) epochs := new(protocol.EpochQuery) epochs.On("Current").Return(epoch) - snap := protocol.NewSnapshot(suite.T()) + snap := new(protocol.Snapshot) snap.On("Epochs").Return(epochs).Maybe() snap.On("Params").Return(suite.params).Maybe() snap.On("Head").Return(block.Header, nil).Maybe() - suite.state.On("AtBlockID", mock.Anything).Return(snap).Once() // initial height lookup in ingestion engine suite.state.On("AtBlockID", refBlockID).Return(snap) - return block, collection + return block, &collection } diff --git a/engine/access/apiproxy/access_api_proxy.go b/engine/access/apiproxy/access_api_proxy.go index b4588397660..f5898686fc6 100644 --- a/engine/access/apiproxy/access_api_proxy.go +++ b/engine/access/apiproxy/access_api_proxy.go @@ -2,26 +2,17 @@ package apiproxy import ( "context" - "fmt" - "sync" "time" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials/insecure" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" "google.golang.org/grpc/status" "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" - "github.com/onflow/flow-go/engine/access/rpc/backend" + "github.com/onflow/flow-go/engine/common/grpc/forwarder" "github.com/onflow/flow-go/engine/protocol" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/utils/grpcutils" ) // FlowAccessAPIRouter is a structure that represents the routing proxy algorithm. @@ -51,88 +42,6 @@ func (h *FlowAccessAPIRouter) log(handler, rpc string, err error) { logger.Info().Msg("request succeeded") } -// reconnectingClient returns an active client, or -// creates one, if the last one is not ready anymore. -func (h *FlowAccessAPIForwarder) reconnectingClient(i int) error { - timeout := h.timeout - - if h.connections[i] == nil || h.connections[i].GetState() != connectivity.Ready { - identity := h.ids[i] - var connection *grpc.ClientConn - var err error - if identity.NetworkPubKey == nil { - connection, err = grpc.Dial( - identity.Address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(h.maxMsgSize))), - grpc.WithTransportCredentials(insecure.NewCredentials()), - backend.WithClientUnaryInterceptor(timeout)) - if err != nil { - return err - } - } else { - tlsConfig, err := grpcutils.DefaultClientTLSConfig(identity.NetworkPubKey) - if err != nil { - return fmt.Errorf("failed to get default TLS client config using public flow networking key %s %w", identity.NetworkPubKey.String(), err) - } - - connection, err = grpc.Dial( - identity.Address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(h.maxMsgSize))), - grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), - backend.WithClientUnaryInterceptor(timeout)) - if err != nil { - return fmt.Errorf("cannot connect to %s %w", identity.Address, err) - } - } - connection.Connect() - time.Sleep(1 * time.Second) - state := connection.GetState() - if state != connectivity.Ready && state != connectivity.Connecting { - return fmt.Errorf("%v", state) - } - h.connections[i] = connection - h.upstream[i] = access.NewAccessAPIClient(connection) - } - - return nil -} - -// faultTolerantClient implements an upstream connection that reconnects on errors -// a reasonable amount of time. -func (h *FlowAccessAPIForwarder) faultTolerantClient() (access.AccessAPIClient, error) { - if h.upstream == nil || len(h.upstream) == 0 { - return nil, status.Errorf(codes.Unimplemented, "method not implemented") - } - - // Reasoning: A retry count of three gives an acceptable 5% failure ratio from a 37% failure ratio. - // A bigger number is problematic due to the DNS resolve and connection times, - // plus the need to log and debug each individual connection failure. - // - // This reasoning eliminates the need of making this parameter configurable. - // The logic works rolling over a single connection as well making clean code. - const retryMax = 3 - - h.lock.Lock() - defer h.lock.Unlock() - - var err error - for i := 0; i < retryMax; i++ { - h.roundRobin++ - h.roundRobin = h.roundRobin % len(h.upstream) - err = h.reconnectingClient(h.roundRobin) - if err != nil { - continue - } - state := h.connections[h.roundRobin].GetState() - if state != connectivity.Ready && state != connectivity.Connecting { - continue - } - return h.upstream[h.roundRobin], nil - } - - return nil, status.Errorf(codes.Unavailable, err.Error()) -} - // Ping pings the service. It is special in the sense that it responds successful, // only if all underlying services are ready. func (h *FlowAccessAPIRouter) Ping(context context.Context, req *access.PingRequest) (*access.PingResponse, error) { @@ -140,6 +49,12 @@ func (h *FlowAccessAPIRouter) Ping(context context.Context, req *access.PingRequ return &access.PingResponse{}, nil } +func (h *FlowAccessAPIRouter) GetNodeVersionInfo(ctx context.Context, request *access.GetNodeVersionInfoRequest) (*access.GetNodeVersionInfoResponse, error) { + res, err := h.Observer.GetNodeVersionInfo(ctx, request) + h.log("observer", "GetNodeVersionInfo", err) + return res, err +} + func (h *FlowAccessAPIRouter) GetLatestBlockHeader(context context.Context, req *access.GetLatestBlockHeaderRequest) (*access.BlockHeaderResponse, error) { res, err := h.Observer.GetLatestBlockHeader(context, req) h.log("observer", "GetLatestBlockHeader", err) @@ -284,63 +199,51 @@ func (h *FlowAccessAPIRouter) GetExecutionResultForBlockID(context context.Conte return res, err } +func (h *FlowAccessAPIRouter) GetExecutionResultByID(context context.Context, req *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) { + res, err := h.Upstream.GetExecutionResultByID(context, req) + h.log("upstream", "GetExecutionResultByID", err) + return res, err +} + // FlowAccessAPIForwarder forwards all requests to a set of upstream access nodes or observers type FlowAccessAPIForwarder struct { - lock sync.Mutex - roundRobin int - ids flow.IdentityList - upstream []access.AccessAPIClient - connections []*grpc.ClientConn - timeout time.Duration - maxMsgSize uint + *forwarder.Forwarder } func NewFlowAccessAPIForwarder(identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*FlowAccessAPIForwarder, error) { - forwarder := &FlowAccessAPIForwarder{maxMsgSize: maxMsgSize} - err := forwarder.setFlowAccessAPI(identities, timeout) - return forwarder, err -} - -// setFlowAccessAPI sets a backend access API that forwards some requests to an upstream node. -// It is used by Observer services, Blockchain Data Service, etc. -// Make sure that this is just for observation and not a staked participant in the flow network. -// This means that observers see a copy of the data but there is no interaction to ensure integrity from the root block. -func (ret *FlowAccessAPIForwarder) setFlowAccessAPI(accessNodeAddressAndPort flow.IdentityList, timeout time.Duration) error { - ret.timeout = timeout - ret.ids = accessNodeAddressAndPort - ret.upstream = make([]access.AccessAPIClient, accessNodeAddressAndPort.Count()) - ret.connections = make([]*grpc.ClientConn, accessNodeAddressAndPort.Count()) - for i, identity := range accessNodeAddressAndPort { - // Store the faultTolerantClient setup parameters such as address, public, key and timeout, so that - // we can refresh the API on connection loss - ret.ids[i] = identity - - // We fail on any single error on startup, so that - // we identify bootstrapping errors early - err := ret.reconnectingClient(i) - if err != nil { - return err - } + forwarder, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) + if err != nil { + return nil, err } - ret.roundRobin = 0 - return nil + return &FlowAccessAPIForwarder{ + Forwarder: forwarder, + }, nil } // Ping pings the service. It is special in the sense that it responds successful, // only if all underlying services are ready. func (h *FlowAccessAPIForwarder) Ping(context context.Context, req *access.PingRequest) (*access.PingResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } return upstream.Ping(context, req) } +func (h *FlowAccessAPIForwarder) GetNodeVersionInfo(context context.Context, req *access.GetNodeVersionInfoRequest) (*access.GetNodeVersionInfoResponse, error) { + // This is a passthrough request + upstream, err := h.FaultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetNodeVersionInfo(context, req) +} + func (h *FlowAccessAPIForwarder) GetLatestBlockHeader(context context.Context, req *access.GetLatestBlockHeaderRequest) (*access.BlockHeaderResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -349,7 +252,7 @@ func (h *FlowAccessAPIForwarder) GetLatestBlockHeader(context context.Context, r func (h *FlowAccessAPIForwarder) GetBlockHeaderByID(context context.Context, req *access.GetBlockHeaderByIDRequest) (*access.BlockHeaderResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -358,7 +261,7 @@ func (h *FlowAccessAPIForwarder) GetBlockHeaderByID(context context.Context, req func (h *FlowAccessAPIForwarder) GetBlockHeaderByHeight(context context.Context, req *access.GetBlockHeaderByHeightRequest) (*access.BlockHeaderResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -367,7 +270,7 @@ func (h *FlowAccessAPIForwarder) GetBlockHeaderByHeight(context context.Context, func (h *FlowAccessAPIForwarder) GetLatestBlock(context context.Context, req *access.GetLatestBlockRequest) (*access.BlockResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -376,7 +279,7 @@ func (h *FlowAccessAPIForwarder) GetLatestBlock(context context.Context, req *ac func (h *FlowAccessAPIForwarder) GetBlockByID(context context.Context, req *access.GetBlockByIDRequest) (*access.BlockResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -385,7 +288,7 @@ func (h *FlowAccessAPIForwarder) GetBlockByID(context context.Context, req *acce func (h *FlowAccessAPIForwarder) GetBlockByHeight(context context.Context, req *access.GetBlockByHeightRequest) (*access.BlockResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -394,7 +297,7 @@ func (h *FlowAccessAPIForwarder) GetBlockByHeight(context context.Context, req * func (h *FlowAccessAPIForwarder) GetCollectionByID(context context.Context, req *access.GetCollectionByIDRequest) (*access.CollectionResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -403,7 +306,7 @@ func (h *FlowAccessAPIForwarder) GetCollectionByID(context context.Context, req func (h *FlowAccessAPIForwarder) SendTransaction(context context.Context, req *access.SendTransactionRequest) (*access.SendTransactionResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -412,7 +315,7 @@ func (h *FlowAccessAPIForwarder) SendTransaction(context context.Context, req *a func (h *FlowAccessAPIForwarder) GetTransaction(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -421,7 +324,7 @@ func (h *FlowAccessAPIForwarder) GetTransaction(context context.Context, req *ac func (h *FlowAccessAPIForwarder) GetTransactionResult(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResultResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -430,7 +333,7 @@ func (h *FlowAccessAPIForwarder) GetTransactionResult(context context.Context, r func (h *FlowAccessAPIForwarder) GetTransactionResultByIndex(context context.Context, req *access.GetTransactionByIndexRequest) (*access.TransactionResultResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -439,7 +342,7 @@ func (h *FlowAccessAPIForwarder) GetTransactionResultByIndex(context context.Con func (h *FlowAccessAPIForwarder) GetTransactionResultsByBlockID(context context.Context, req *access.GetTransactionsByBlockIDRequest) (*access.TransactionResultsResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -447,7 +350,7 @@ func (h *FlowAccessAPIForwarder) GetTransactionResultsByBlockID(context context. } func (h *FlowAccessAPIForwarder) GetTransactionsByBlockID(context context.Context, req *access.GetTransactionsByBlockIDRequest) (*access.TransactionsResponse, error) { - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -456,7 +359,7 @@ func (h *FlowAccessAPIForwarder) GetTransactionsByBlockID(context context.Contex func (h *FlowAccessAPIForwarder) GetAccount(context context.Context, req *access.GetAccountRequest) (*access.GetAccountResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -465,7 +368,7 @@ func (h *FlowAccessAPIForwarder) GetAccount(context context.Context, req *access func (h *FlowAccessAPIForwarder) GetAccountAtLatestBlock(context context.Context, req *access.GetAccountAtLatestBlockRequest) (*access.AccountResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -474,7 +377,7 @@ func (h *FlowAccessAPIForwarder) GetAccountAtLatestBlock(context context.Context func (h *FlowAccessAPIForwarder) GetAccountAtBlockHeight(context context.Context, req *access.GetAccountAtBlockHeightRequest) (*access.AccountResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -483,7 +386,7 @@ func (h *FlowAccessAPIForwarder) GetAccountAtBlockHeight(context context.Context func (h *FlowAccessAPIForwarder) ExecuteScriptAtLatestBlock(context context.Context, req *access.ExecuteScriptAtLatestBlockRequest) (*access.ExecuteScriptResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -492,7 +395,7 @@ func (h *FlowAccessAPIForwarder) ExecuteScriptAtLatestBlock(context context.Cont func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockID(context context.Context, req *access.ExecuteScriptAtBlockIDRequest) (*access.ExecuteScriptResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -501,7 +404,7 @@ func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockID(context context.Context, func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockHeight(context context.Context, req *access.ExecuteScriptAtBlockHeightRequest) (*access.ExecuteScriptResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -510,7 +413,7 @@ func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockHeight(context context.Cont func (h *FlowAccessAPIForwarder) GetEventsForHeightRange(context context.Context, req *access.GetEventsForHeightRangeRequest) (*access.EventsResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -519,7 +422,7 @@ func (h *FlowAccessAPIForwarder) GetEventsForHeightRange(context context.Context func (h *FlowAccessAPIForwarder) GetEventsForBlockIDs(context context.Context, req *access.GetEventsForBlockIDsRequest) (*access.EventsResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -528,7 +431,7 @@ func (h *FlowAccessAPIForwarder) GetEventsForBlockIDs(context context.Context, r func (h *FlowAccessAPIForwarder) GetNetworkParameters(context context.Context, req *access.GetNetworkParametersRequest) (*access.GetNetworkParametersResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -537,7 +440,7 @@ func (h *FlowAccessAPIForwarder) GetNetworkParameters(context context.Context, r func (h *FlowAccessAPIForwarder) GetLatestProtocolStateSnapshot(context context.Context, req *access.GetLatestProtocolStateSnapshotRequest) (*access.ProtocolStateSnapshotResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -546,9 +449,18 @@ func (h *FlowAccessAPIForwarder) GetLatestProtocolStateSnapshot(context context. func (h *FlowAccessAPIForwarder) GetExecutionResultForBlockID(context context.Context, req *access.GetExecutionResultForBlockIDRequest) (*access.ExecutionResultForBlockIDResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } return upstream.GetExecutionResultForBlockID(context, req) } + +func (h *FlowAccessAPIForwarder) GetExecutionResultByID(context context.Context, req *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) { + // This is a passthrough request + upstream, err := h.FaultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetExecutionResultByID(context, req) +} diff --git a/engine/access/apiproxy/access_api_proxy_test.go b/engine/access/apiproxy/access_api_proxy_test.go index 9f5a5aa74b8..d20c5ee705d 100644 --- a/engine/access/apiproxy/access_api_proxy_test.go +++ b/engine/access/apiproxy/access_api_proxy_test.go @@ -11,6 +11,7 @@ import ( "google.golang.org/grpc" grpcinsecure "google.golang.org/grpc/credentials/insecure" + "github.com/onflow/flow-go/engine/common/grpc/forwarder" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/grpcutils" "github.com/onflow/flow-go/utils/unittest" @@ -137,7 +138,8 @@ func TestNewFlowCachedAccessAPIProxy(t *testing.T) { // Prepare a proxy that fails due to the second connection being idle l := flow.IdentityList{{Address: unittest.IPPort("11634")}, {Address: unittest.IPPort("11635")}} c := FlowAccessAPIForwarder{} - err = c.setFlowAccessAPI(l, time.Second) + c.Forwarder, err = forwarder.NewForwarder(l, time.Second, grpcutils.DefaultMaxMsgSize) + if err == nil { t.Fatal(fmt.Errorf("should not start with one connection ready")) } @@ -153,7 +155,7 @@ func TestNewFlowCachedAccessAPIProxy(t *testing.T) { // Prepare a proxy l = flow.IdentityList{{Address: unittest.IPPort("11634")}, {Address: unittest.IPPort("11635")}} c = FlowAccessAPIForwarder{} - err = c.setFlowAccessAPI(l, time.Second) + c.Forwarder, err = forwarder.NewForwarder(l, time.Second, grpcutils.DefaultMaxMsgSize) if err != nil { t.Fatal(err) } diff --git a/engine/access/ingestion/engine.go b/engine/access/ingestion/engine.go index 58b0617a2bd..74f3721823f 100644 --- a/engine/access/ingestion/engine.go +++ b/engine/access/ingestion/engine.go @@ -12,7 +12,6 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/common/fifoqueue" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" @@ -79,12 +78,10 @@ type Engine struct { executionResults storage.ExecutionResults // metrics - transactionMetrics module.TransactionMetrics + metrics module.AccessMetrics collectionsToMarkFinalized *stdmap.Times collectionsToMarkExecuted *stdmap.Times blocksToMarkExecuted *stdmap.Times - - rpcEngine *rpc.Engine } // New creates a new access ingestion engine @@ -100,11 +97,10 @@ func New( transactions storage.Transactions, executionResults storage.ExecutionResults, executionReceipts storage.ExecutionReceipts, - transactionMetrics module.TransactionMetrics, + accessMetrics module.AccessMetrics, collectionsToMarkFinalized *stdmap.Times, collectionsToMarkExecuted *stdmap.Times, blocksToMarkExecuted *stdmap.Times, - rpcEngine *rpc.Engine, ) (*Engine, error) { executionReceiptsRawQueue, err := fifoqueue.NewFifoQueue(defaultQueueCapacity) if err != nil { @@ -152,11 +148,10 @@ func New( executionResults: executionResults, executionReceipts: executionReceipts, maxReceiptHeight: 0, - transactionMetrics: transactionMetrics, + metrics: accessMetrics, collectionsToMarkFinalized: collectionsToMarkFinalized, collectionsToMarkExecuted: collectionsToMarkExecuted, blocksToMarkExecuted: blocksToMarkExecuted, - rpcEngine: rpcEngine, // queue / notifier for execution receipts executionReceiptsNotifier: engine.NewNotifier(), @@ -201,7 +196,7 @@ func (e *Engine) Start(parent irrecoverable.SignalerContext) { // If the index has already been initialized, this is a no-op. // No errors are expected during normal operation. func (e *Engine) initLastFullBlockHeightIndex() error { - rootBlock, err := e.state.Params().Root() + rootBlock, err := e.state.Params().FinalizedRoot() if err != nil { return fmt.Errorf("failed to get root block: %w", err) } @@ -210,6 +205,13 @@ func (e *Engine) initLastFullBlockHeightIndex() error { return fmt.Errorf("failed to update last full block height during ingestion engine startup: %w", err) } + lastFullHeight, err := e.blocks.GetLastFullBlockHeight() + if err != nil { + return fmt.Errorf("failed to get last full block height during ingestion engine startup: %w", err) + } + + e.metrics.UpdateLastFullBlockHeight(lastFullHeight) + return nil } @@ -382,9 +384,6 @@ func (e *Engine) processFinalizedBlock(blockID flow.Identifier) error { return fmt.Errorf("failed to lookup block: %w", err) } - // Notify rpc handler of new finalized block height - e.rpcEngine.SubmitLocal(block) - // FIX: we can't index guarantees here, as we might have more than one block // with the same collection as long as it is not finalized @@ -449,13 +448,13 @@ func (e *Engine) trackFinalizedMetricForBlock(hb *model.Block) { } for _, t := range l.Transactions { - e.transactionMetrics.TransactionFinalized(t, now) + e.metrics.TransactionFinalized(t, now) } } if ti, found := e.blocksToMarkExecuted.ByID(hb.BlockID); found { e.trackExecutedMetricForBlock(block, ti) - e.transactionMetrics.UpdateExecutionReceiptMaxHeight(block.Header.Height) + e.metrics.UpdateExecutionReceiptMaxHeight(block.Header.Height) e.blocksToMarkExecuted.Remove(hb.BlockID) } } @@ -489,7 +488,7 @@ func (e *Engine) trackExecutionReceiptMetrics(r *flow.ExecutionReceipt) { return } - e.transactionMetrics.UpdateExecutionReceiptMaxHeight(b.Header.Height) + e.metrics.UpdateExecutionReceiptMaxHeight(b.Header.Height) e.trackExecutedMetricForBlock(b, now) } @@ -509,7 +508,7 @@ func (e *Engine) trackExecutedMetricForBlock(block *flow.Block, ti time.Time) { } for _, t := range l.Transactions { - e.transactionMetrics.TransactionExecuted(t, ti) + e.metrics.TransactionExecuted(t, ti) } } } @@ -527,14 +526,14 @@ func (e *Engine) handleCollection(originID flow.Identifier, entity flow.Entity) if ti, found := e.collectionsToMarkFinalized.ByID(light.ID()); found { for _, t := range light.Transactions { - e.transactionMetrics.TransactionFinalized(t, ti) + e.metrics.TransactionFinalized(t, ti) } e.collectionsToMarkFinalized.Remove(light.ID()) } if ti, found := e.collectionsToMarkExecuted.ByID(light.ID()); found { for _, t := range light.Transactions { - e.transactionMetrics.TransactionExecuted(t, ti) + e.metrics.TransactionExecuted(t, ti) } e.collectionsToMarkExecuted.Remove(light.ID()) } @@ -575,14 +574,6 @@ func (e *Engine) OnCollection(originID flow.Identifier, entity flow.Entity) { } } -// OnBlockIncorporated is a noop for this engine since access node is only dealing with finalized blocks -func (e *Engine) OnBlockIncorporated(*model.Block) { -} - -// OnDoubleProposeDetected is a noop for this engine since access node is only dealing with finalized blocks -func (e *Engine) OnDoubleProposeDetected(*model.Block, *model.Block) { -} - // requestMissingCollections requests missing collections for all blocks in the local db storage once at startup func (e *Engine) requestMissingCollections(ctx context.Context) error { @@ -685,7 +676,7 @@ func (e *Engine) requestMissingCollections(ctx context.Context) error { return nil } -// updateLastFullBlockReceivedIndex keeps the FullBlockHeight index upto date and requests missing collections if +// updateLastFullBlockReceivedIndex keeps the FullBlockHeight index up to date and requests missing collections if // the number of blocks missing collection have reached the defaultMissingCollsForBlkThreshold value. // (The FullBlockHeight index indicates that block for which all collections have been received) func (e *Engine) updateLastFullBlockReceivedIndex() { @@ -701,7 +692,7 @@ func (e *Engine) updateLastFullBlockReceivedIndex() { return } // use the root height as the last full height - header, err := e.state.Params().Root() + header, err := e.state.Params().FinalizedRoot() if err != nil { logError(err) return @@ -762,6 +753,8 @@ func (e *Engine) updateLastFullBlockReceivedIndex() { logError(err) return } + + e.metrics.UpdateLastFullBlockHeight(lastFullHeight) } // additionally, if more than threshold blocks have missing collection OR collections are missing since defaultMissingCollsForAgeThreshold, re-request those collections diff --git a/engine/access/ingestion/engine_test.go b/engine/access/ingestion/engine_test.go index 2f3afe79fd2..c4d0fb72141 100644 --- a/engine/access/ingestion/engine_test.go +++ b/engine/access/ingestion/engine_test.go @@ -15,7 +15,6 @@ import ( "github.com/stretchr/testify/suite" hotmodel "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/module/component" @@ -43,17 +42,19 @@ type Suite struct { params *protocol.Params } - me *module.Local - request *module.Requester - provider *mocknetwork.Engine - blocks *storage.Blocks - headers *storage.Headers - collections *storage.Collections - transactions *storage.Transactions - receipts *storage.ExecutionReceipts - results *storage.ExecutionResults - seals *storage.Seals - downloader *downloadermock.Downloader + me *module.Local + request *module.Requester + provider *mocknetwork.Engine + blocks *storage.Blocks + headers *storage.Headers + collections *storage.Collections + transactions *storage.Transactions + receipts *storage.ExecutionReceipts + results *storage.ExecutionResults + seals *storage.Seals + downloader *downloadermock.Downloader + sealedBlock *flow.Header + finalizedBlock *flow.Header eng *Engine cancel context.CancelFunc @@ -76,9 +77,16 @@ func (suite *Suite) SetupTest() { suite.proto.state = new(protocol.FollowerState) suite.proto.snapshot = new(protocol.Snapshot) suite.proto.params = new(protocol.Params) + suite.finalizedBlock = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(0)) suite.proto.state.On("Identity").Return(obsIdentity, nil) suite.proto.state.On("Final").Return(suite.proto.snapshot, nil) suite.proto.state.On("Params").Return(suite.proto.params) + suite.proto.snapshot.On("Head").Return( + func() *flow.Header { + return suite.finalizedBlock + }, + nil, + ).Maybe() suite.me = new(module.Local) suite.me.On("NodeID").Return(obsIdentity.NodeID) @@ -104,16 +112,9 @@ func (suite *Suite) SetupTest() { blocksToMarkExecuted, err := stdmap.NewTimes(100) require.NoError(suite.T(), err) - rpcEngBuilder, err := rpc.NewBuilder(log, suite.proto.state, rpc.Config{}, nil, nil, suite.blocks, suite.headers, suite.collections, - suite.transactions, suite.receipts, suite.results, flow.Testnet, metrics.NewNoopCollector(), metrics.NewNoopCollector(), 0, - 0, false, false, nil, nil) - require.NoError(suite.T(), err) - rpcEng, err := rpcEngBuilder.WithLegacy().Build() - require.NoError(suite.T(), err) - eng, err := New(log, net, suite.proto.state, suite.me, suite.request, suite.blocks, suite.headers, suite.collections, suite.transactions, suite.results, suite.receipts, metrics.NewNoopCollector(), collectionsToMarkFinalized, collectionsToMarkExecuted, - blocksToMarkExecuted, rpcEng) + blocksToMarkExecuted) require.NoError(suite.T(), err) suite.blocks.On("GetLastFullBlockHeight").Once().Return(uint64(0), errors.New("do nothing")) @@ -369,7 +370,7 @@ func (suite *Suite) TestRequestMissingCollections() { // consider collections are missing for all blocks suite.blocks.On("GetLastFullBlockHeight").Return(startHeight-1, nil) // consider the last test block as the head - suite.proto.snapshot.On("Head").Return(blocks[blkCnt-1].Header, nil) + suite.finalizedBlock = blocks[blkCnt-1].Header // p is the probability of not receiving the collection before the next poll and it // helps simulate the slow trickle of the requested collections being received @@ -556,13 +557,13 @@ func (suite *Suite) TestUpdateLastFullBlockReceivedIndex() { }) // consider the last test block as the head - suite.proto.snapshot.On("Head").Return(finalizedBlk.Header, nil) + suite.finalizedBlock = finalizedBlk.Header suite.Run("full block height index is created and advanced if not present", func() { // simulate the absence of the full block height index lastFullBlockHeight = 0 rtnErr = storerr.ErrNotFound - suite.proto.params.On("Root").Return(rootBlk.Header, nil) + suite.proto.params.On("FinalizedRoot").Return(rootBlk.Header, nil) suite.blocks.On("UpdateLastFullBlockHeight", finalizedHeight).Return(nil).Once() suite.eng.updateLastFullBlockReceivedIndex() diff --git a/engine/access/integration_unsecure_grpc_server_test.go b/engine/access/integration_unsecure_grpc_server_test.go new file mode 100644 index 00000000000..e2cf78ade5a --- /dev/null +++ b/engine/access/integration_unsecure_grpc_server_test.go @@ -0,0 +1,315 @@ +package access + +import ( + "context" + + "io" + "os" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + "github.com/onflow/flow-go/engine" + accessmock "github.com/onflow/flow-go/engine/access/mock" + "github.com/onflow/flow-go/engine/access/rpc" + "github.com/onflow/flow-go/engine/access/rpc/backend" + "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/blobs" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" + "github.com/onflow/flow-go/module/grpcserver" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mempool/herocache" + "github.com/onflow/flow-go/module/metrics" + module "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" + protocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/storage" + storagemock "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/grpcutils" + "github.com/onflow/flow-go/utils/unittest" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" + executiondataproto "github.com/onflow/flow/protobuf/go/flow/executiondata" +) + +// SameGRPCPortTestSuite verifies both AccessAPI and ExecutionDataAPI client continue to work when configured +// on the same port +type SameGRPCPortTestSuite struct { + suite.Suite + state *protocol.State + snapshot *protocol.Snapshot + epochQuery *protocol.EpochQuery + log zerolog.Logger + net *network.Network + request *module.Requester + collClient *accessmock.AccessAPIClient + execClient *accessmock.ExecutionAPIClient + me *module.Local + chainID flow.ChainID + metrics *metrics.NoopCollector + rpcEng *rpc.Engine + stateStreamEng *state_stream.Engine + + // storage + blocks *storagemock.Blocks + headers *storagemock.Headers + collections *storagemock.Collections + transactions *storagemock.Transactions + receipts *storagemock.ExecutionReceipts + seals *storagemock.Seals + results *storagemock.ExecutionResults + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer + + bs blobs.Blobstore + eds execution_data.ExecutionDataStore + broadcaster *engine.Broadcaster + execDataCache *cache.ExecutionDataCache + execDataHeroCache *herocache.BlockExecutionData + + blockMap map[uint64]*flow.Block +} + +func (suite *SameGRPCPortTestSuite) SetupTest() { + suite.log = zerolog.New(os.Stdout) + suite.net = new(network.Network) + suite.state = new(protocol.State) + suite.snapshot = new(protocol.Snapshot) + + suite.epochQuery = new(protocol.EpochQuery) + suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() + suite.state.On("Final").Return(suite.snapshot, nil).Maybe() + suite.snapshot.On("Epochs").Return(suite.epochQuery).Maybe() + suite.blocks = new(storagemock.Blocks) + suite.headers = new(storagemock.Headers) + suite.transactions = new(storagemock.Transactions) + suite.collections = new(storagemock.Collections) + suite.receipts = new(storagemock.ExecutionReceipts) + suite.results = new(storagemock.ExecutionResults) + suite.seals = new(storagemock.Seals) + + suite.collClient = new(accessmock.AccessAPIClient) + suite.execClient = new(accessmock.ExecutionAPIClient) + + suite.request = new(module.Requester) + suite.request.On("EntityByID", mock.Anything, mock.Anything) + + suite.me = new(module.Local) + suite.eds = execution_data.NewExecutionDataStore(suite.bs, execution_data.DefaultSerializer) + + suite.broadcaster = engine.NewBroadcaster() + + suite.execDataHeroCache = herocache.NewBlockExecutionData(state_stream.DefaultCacheSize, suite.log, metrics.NewNoopCollector()) + suite.execDataCache = cache.NewExecutionDataCache(suite.eds, suite.headers, suite.seals, suite.results, suite.execDataHeroCache) + + accessIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleAccess)) + suite.me. + On("NodeID"). + Return(accessIdentity.NodeID) + + suite.chainID = flow.Testnet + suite.metrics = metrics.NewNoopCollector() + + config := rpc.Config{ + UnsecureGRPCListenAddr: unittest.DefaultAddress, + SecureGRPCListenAddr: unittest.DefaultAddress, + HTTPListenAddr: unittest.DefaultAddress, + } + + blockCount := 5 + suite.blockMap = make(map[uint64]*flow.Block, blockCount) + // generate blockCount consecutive blocks with associated seal, result and execution data + rootBlock := unittest.BlockFixture() + parent := rootBlock.Header + suite.blockMap[rootBlock.Header.Height] = &rootBlock + + for i := 0; i < blockCount; i++ { + block := unittest.BlockWithParentFixture(parent) + suite.blockMap[block.Header.Height] = block + } + + // generate a server certificate that will be served by the GRPC server + networkingKey := unittest.NetworkingPrivKeyFixture() + x509Certificate, err := grpcutils.X509Certificate(networkingKey) + assert.NoError(suite.T(), err) + tlsConfig := grpcutils.DefaultServerTLSConfig(x509Certificate) + // set the transport credentials for the server to use + config.TransportCredentials = credentials.NewTLS(tlsConfig) + + suite.secureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.SecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + nil, + nil, + grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() + + suite.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.UnsecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + nil, + nil).Build() + + block := unittest.BlockHeaderFixture() + suite.snapshot.On("Head").Return(block, nil) + + backend := backend.New( + suite.state, + suite.collClient, + nil, + suite.blocks, + suite.headers, + suite.collections, + suite.transactions, + nil, + nil, + suite.chainID, + suite.metrics, + nil, + false, + 0, + nil, + nil, + suite.log, + 0, + nil, + false) + + // create rpc engine builder + rpcEngBuilder, err := rpc.NewBuilder( + suite.log, + suite.state, + config, + suite.chainID, + suite.metrics, + false, + suite.me, + backend, + backend, + suite.secureGrpcServer, + suite.unsecureGrpcServer, + ) + assert.NoError(suite.T(), err) + suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() + assert.NoError(suite.T(), err) + suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + + suite.headers.On("BlockIDByHeight", mock.AnythingOfType("uint64")).Return( + func(height uint64) flow.Identifier { + if block, ok := suite.blockMap[height]; ok { + return block.Header.ID() + } + return flow.ZeroID + }, + func(height uint64) error { + if _, ok := suite.blockMap[height]; ok { + return nil + } + return storage.ErrNotFound + }, + ).Maybe() + + conf := state_stream.Config{ + ClientSendTimeout: state_stream.DefaultSendTimeout, + ClientSendBufferSize: state_stream.DefaultSendBufferSize, + } + + // create state stream engine + suite.stateStreamEng, err = state_stream.NewEng( + suite.log, + conf, + nil, + suite.execDataCache, + suite.state, + suite.headers, + suite.seals, + suite.results, + suite.chainID, + rootBlock.Header.Height, + rootBlock.Header.Height, + suite.unsecureGrpcServer, + ) + assert.NoError(suite.T(), err) + + suite.rpcEng.Start(suite.ctx) + suite.stateStreamEng.Start(suite.ctx) + + suite.secureGrpcServer.Start(suite.ctx) + suite.unsecureGrpcServer.Start(suite.ctx) + + // wait for the servers to startup + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) + + // wait for the rpc engine to startup + unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second) + // wait for the state stream engine to startup + unittest.AssertClosesBefore(suite.T(), suite.stateStreamEng.Ready(), 2*time.Second) +} + +// TestEnginesOnTheSameGrpcPort verifies if both AccessAPI and ExecutionDataAPI client successfully connect and continue +// to work when configured on the same port +func (suite *SameGRPCPortTestSuite) TestEnginesOnTheSameGrpcPort() { + ctx := context.Background() + + conn, err := grpc.Dial( + suite.unsecureGrpcServer.GRPCAddress().String(), + grpc.WithTransportCredentials(insecure.NewCredentials())) + assert.NoError(suite.T(), err) + closer := io.Closer(conn) + + suite.Run("happy path - grpc access api client can connect successfully", func() { + req := &accessproto.GetNetworkParametersRequest{} + + // expect 2 upstream calls + suite.execClient.On("GetNetworkParameters", mock.Anything, mock.Anything).Return(nil, nil).Twice() + suite.collClient.On("GetNetworkParameters", mock.Anything, mock.Anything).Return(nil, nil).Twice() + + client := suite.unsecureAccessAPIClient(conn) + + _, err := client.GetNetworkParameters(ctx, req) + assert.NoError(suite.T(), err, "failed to get network") + }) + + suite.Run("happy path - grpc execution data api client can connect successfully", func() { + req := &executiondataproto.SubscribeEventsRequest{} + + client := suite.unsecureExecutionDataAPIClient(conn) + + _, err := client.SubscribeEvents(ctx, req) + assert.NoError(suite.T(), err, "failed to subscribe events") + }) + defer closer.Close() +} + +func TestSameGRPCTestSuite(t *testing.T) { + suite.Run(t, new(SameGRPCPortTestSuite)) +} + +// unsecureAccessAPIClient creates an unsecure grpc AccessAPI client +func (suite *SameGRPCPortTestSuite) unsecureAccessAPIClient(conn *grpc.ClientConn) accessproto.AccessAPIClient { + client := accessproto.NewAccessAPIClient(conn) + return client +} + +// unsecureExecutionDataAPIClient creates an unsecure ExecutionDataAPI client +func (suite *SameGRPCPortTestSuite) unsecureExecutionDataAPIClient(conn *grpc.ClientConn) executiondataproto.ExecutionDataAPIClient { + client := executiondataproto.NewExecutionDataAPIClient(conn) + return client +} diff --git a/engine/access/mock/access_api_client.go b/engine/access/mock/access_api_client.go index 91c7af50026..4e2b1d065c7 100644 --- a/engine/access/mock/access_api_client.go +++ b/engine/access/mock/access_api_client.go @@ -446,6 +446,39 @@ func (_m *AccessAPIClient) GetEventsForHeightRange(ctx context.Context, in *acce return r0, r1 } +// GetExecutionResultByID provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) GetExecutionResultByID(ctx context.Context, in *access.GetExecutionResultByIDRequest, opts ...grpc.CallOption) (*access.ExecutionResultByIDResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 *access.ExecutionResultByIDResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.GetExecutionResultByIDRequest, ...grpc.CallOption) (*access.ExecutionResultByIDResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.GetExecutionResultByIDRequest, ...grpc.CallOption) *access.ExecutionResultByIDResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*access.ExecutionResultByIDResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.GetExecutionResultByIDRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetExecutionResultForBlockID provides a mock function with given fields: ctx, in, opts func (_m *AccessAPIClient) GetExecutionResultForBlockID(ctx context.Context, in *access.GetExecutionResultForBlockIDRequest, opts ...grpc.CallOption) (*access.ExecutionResultForBlockIDResponse, error) { _va := make([]interface{}, len(opts)) @@ -611,6 +644,39 @@ func (_m *AccessAPIClient) GetNetworkParameters(ctx context.Context, in *access. return r0, r1 } +// GetNodeVersionInfo provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) GetNodeVersionInfo(ctx context.Context, in *access.GetNodeVersionInfoRequest, opts ...grpc.CallOption) (*access.GetNodeVersionInfoResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 *access.GetNodeVersionInfoResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.GetNodeVersionInfoRequest, ...grpc.CallOption) (*access.GetNodeVersionInfoResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.GetNodeVersionInfoRequest, ...grpc.CallOption) *access.GetNodeVersionInfoResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*access.GetNodeVersionInfoResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.GetNodeVersionInfoRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetTransaction provides a mock function with given fields: ctx, in, opts func (_m *AccessAPIClient) GetTransaction(ctx context.Context, in *access.GetTransactionRequest, opts ...grpc.CallOption) (*access.TransactionResponse, error) { _va := make([]interface{}, len(opts)) diff --git a/engine/access/mock/access_api_server.go b/engine/access/mock/access_api_server.go index b3aa12b4eff..1a2c3772e44 100644 --- a/engine/access/mock/access_api_server.go +++ b/engine/access/mock/access_api_server.go @@ -353,6 +353,32 @@ func (_m *AccessAPIServer) GetEventsForHeightRange(_a0 context.Context, _a1 *acc return r0, r1 } +// GetExecutionResultByID provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) GetExecutionResultByID(_a0 context.Context, _a1 *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) { + ret := _m.Called(_a0, _a1) + + var r0 *access.ExecutionResultByIDResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.GetExecutionResultByIDRequest) *access.ExecutionResultByIDResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*access.ExecutionResultByIDResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.GetExecutionResultByIDRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetExecutionResultForBlockID provides a mock function with given fields: _a0, _a1 func (_m *AccessAPIServer) GetExecutionResultForBlockID(_a0 context.Context, _a1 *access.GetExecutionResultForBlockIDRequest) (*access.ExecutionResultForBlockIDResponse, error) { ret := _m.Called(_a0, _a1) @@ -483,6 +509,32 @@ func (_m *AccessAPIServer) GetNetworkParameters(_a0 context.Context, _a1 *access return r0, r1 } +// GetNodeVersionInfo provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) GetNodeVersionInfo(_a0 context.Context, _a1 *access.GetNodeVersionInfoRequest) (*access.GetNodeVersionInfoResponse, error) { + ret := _m.Called(_a0, _a1) + + var r0 *access.GetNodeVersionInfoResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.GetNodeVersionInfoRequest) (*access.GetNodeVersionInfoResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.GetNodeVersionInfoRequest) *access.GetNodeVersionInfoResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*access.GetNodeVersionInfoResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.GetNodeVersionInfoRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetTransaction provides a mock function with given fields: _a0, _a1 func (_m *AccessAPIServer) GetTransaction(_a0 context.Context, _a1 *access.GetTransactionRequest) (*access.TransactionResponse, error) { ret := _m.Called(_a0, _a1) diff --git a/engine/access/relay/example_test.go b/engine/access/relay/example_test.go index 6574dce4567..9fe7086ce16 100644 --- a/engine/access/relay/example_test.go +++ b/engine/access/relay/example_test.go @@ -2,7 +2,6 @@ package relay_test import ( "fmt" - "math/rand" "github.com/rs/zerolog" @@ -21,10 +20,11 @@ func Example() { logger := zerolog.Nop() splitterNet := splitterNetwork.NewNetwork(net, logger) - // generate a random origin ID - var id flow.Identifier - rand.Seed(0) - rand.Read(id[:]) + // generate an origin ID + id, err := flow.HexStringToIdentifier("0194fdc2fa2ffcc041d3ff12045b73c86e4ff95ff662a5eee82abdf44a2d0b75") + if err != nil { + fmt.Println(err) + } // create engines engineProcessFunc := func(engineName string) testnet.EngineProcessFunc { @@ -39,7 +39,7 @@ func Example() { // register engines on the splitter network fooChannel := channels.Channel("foo-channel") barChannel := channels.Channel("bar-channel") - _, err := splitterNet.Register(fooChannel, fooEngine) + _, err = splitterNet.Register(fooChannel, fooEngine) if err != nil { fmt.Println(err) } diff --git a/engine/access/rest/README.md b/engine/access/rest/README.md index fd7b970493d..d94af68c238 100644 --- a/engine/access/rest/README.md +++ b/engine/access/rest/README.md @@ -6,10 +6,14 @@ available on our [docs site](https://docs.onflow.org/http-api/). ## Packages -- `rest`: The HTTP handlers for all the request, server generator and the select filter. +- `rest`: The HTTP handlers for the server generator and the select filter, implementation of handling local requests. - `middleware`: The common [middlewares](https://github.com/gorilla/mux#middleware) that all request pass through. - `models`: The generated models using openapi generators and implementation of model builders. - `request`: Implementation of API requests that provide validation for input data and build request models. +- `routes`: The common HTTP handlers for all the requests, tests for each request. +- `apiproxy`: Implementation of proxy backend handler which includes the local backend and forwards the methods which +can't be handled locally to an upstream using gRPC API. This is used by observers that don't have all data in their +local db. ## Request lifecycle @@ -37,7 +41,7 @@ make generate-openapi ### Adding New API Endpoints -A new endpoint can be added by first implementing a new request handler, a request handle is a function in the rest +A new endpoint can be added by first implementing a new request handler, a request handle is a function in the routes package that complies with function interfaced defined as: ```go @@ -48,6 +52,7 @@ generator models.LinkGenerator, ) (interface{}, error) ``` -That handler implementation needs to be added to the `router.go` with corresponding API endpoint and method. Adding a -new API endpoint also requires for a new request builder to be implemented and added in request package. Make sure to -not forget about adding tests for each of the API handler. +That handler implementation needs to be added to the `router.go` with corresponding API endpoint and method. If the data +is not available on observers, an override the method is needed in the backend handler `RestProxyHandler` for request +forwarding. Adding a new API endpoint also requires for a new request builder to be implemented and added in request +package. Make sure to not forget about adding tests for each of the API handler. diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go new file mode 100644 index 00000000000..01e7b56724d --- /dev/null +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -0,0 +1,344 @@ +package apiproxy + +import ( + "context" + "fmt" + "time" + + "google.golang.org/grpc/status" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/common/grpc/forwarder" + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" +) + +// RestProxyHandler is a structure that represents the proxy algorithm for observer node. +// It includes the local backend and forwards the methods which can't be handled locally to an upstream using gRPC API. +type RestProxyHandler struct { + access.API + *forwarder.Forwarder + Logger zerolog.Logger + Metrics metrics.ObserverMetrics + Chain flow.Chain +} + +// NewRestProxyHandler returns a new rest proxy handler for observer node. +func NewRestProxyHandler( + api access.API, + identities flow.IdentityList, + timeout time.Duration, + maxMsgSize uint, + log zerolog.Logger, + metrics metrics.ObserverMetrics, + chain flow.Chain, +) (*RestProxyHandler, error) { + + forwarder, err := forwarder.NewForwarder( + identities, + timeout, + maxMsgSize) + if err != nil { + return nil, fmt.Errorf("could not create REST forwarder: %w", err) + } + + restProxyHandler := &RestProxyHandler{ + Logger: log, + Metrics: metrics, + Chain: chain, + } + + restProxyHandler.API = api + restProxyHandler.Forwarder = forwarder + + return restProxyHandler, nil +} + +func (r *RestProxyHandler) log(handler, rpc string, err error) { + code := status.Code(err) + r.Metrics.RecordRPC(handler, rpc, code) + + logger := r.Logger.With(). + Str("handler", handler). + Str("rest_method", rpc). + Str("rest_code", code.String()). + Logger() + + if err != nil { + logger.Error().Err(err).Msg("request failed") + return + } + + logger.Info().Msg("request succeeded") +} + +// GetCollectionByID returns a collection by ID. +func (r *RestProxyHandler) GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow.LightCollection, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getCollectionByIDRequest := &accessproto.GetCollectionByIDRequest{ + Id: id[:], + } + + collectionResponse, err := upstream.GetCollectionByID(ctx, getCollectionByIDRequest) + r.log("upstream", "GetCollectionByID", err) + + if err != nil { + return nil, err + } + + transactions, err := convert.MessageToLightCollection(collectionResponse.Collection) + if err != nil { + return nil, err + } + + return transactions, nil +} + +// SendTransaction sends already created transaction. +func (r *RestProxyHandler) SendTransaction(ctx context.Context, tx *flow.TransactionBody) error { + upstream, err := r.FaultTolerantClient() + if err != nil { + return err + } + + transaction := convert.TransactionToMessage(*tx) + sendTransactionRequest := &accessproto.SendTransactionRequest{ + Transaction: transaction, + } + + _, err = upstream.SendTransaction(ctx, sendTransactionRequest) + r.log("upstream", "SendTransaction", err) + + return err +} + +// GetTransaction returns transaction by ID. +func (r *RestProxyHandler) GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getTransactionRequest := &accessproto.GetTransactionRequest{ + Id: id[:], + } + transactionResponse, err := upstream.GetTransaction(ctx, getTransactionRequest) + r.log("upstream", "GetTransaction", err) + + if err != nil { + return nil, err + } + + transactionBody, err := convert.MessageToTransaction(transactionResponse.Transaction, r.Chain) + if err != nil { + return nil, err + } + + return &transactionBody, nil +} + +// GetTransactionResult returns transaction result by the transaction ID. +func (r *RestProxyHandler) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*access.TransactionResult, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + + return nil, err + } + + getTransactionResultRequest := &accessproto.GetTransactionRequest{ + Id: id[:], + BlockId: blockID[:], + CollectionId: collectionID[:], + } + + transactionResultResponse, err := upstream.GetTransactionResult(ctx, getTransactionResultRequest) + r.log("upstream", "GetTransactionResult", err) + + if err != nil { + return nil, err + } + + return access.MessageToTransactionResult(transactionResultResponse), nil +} + +// GetAccountAtBlockHeight returns account by account address and block height. +func (r *RestProxyHandler) GetAccountAtBlockHeight(ctx context.Context, address flow.Address, height uint64) (*flow.Account, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getAccountAtBlockHeightRequest := &accessproto.GetAccountAtBlockHeightRequest{ + Address: address.Bytes(), + BlockHeight: height, + } + + accountResponse, err := upstream.GetAccountAtBlockHeight(ctx, getAccountAtBlockHeightRequest) + r.log("upstream", "GetAccountAtBlockHeight", err) + + if err != nil { + return nil, err + } + + return convert.MessageToAccount(accountResponse.Account) +} + +// ExecuteScriptAtLatestBlock executes script at latest block. +func (r *RestProxyHandler) ExecuteScriptAtLatestBlock(ctx context.Context, script []byte, arguments [][]byte) ([]byte, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + executeScriptAtLatestBlockRequest := &accessproto.ExecuteScriptAtLatestBlockRequest{ + Script: script, + Arguments: arguments, + } + executeScriptAtLatestBlockResponse, err := upstream.ExecuteScriptAtLatestBlock(ctx, executeScriptAtLatestBlockRequest) + r.log("upstream", "ExecuteScriptAtLatestBlock", err) + + if err != nil { + return nil, err + } + + return executeScriptAtLatestBlockResponse.Value, nil +} + +// ExecuteScriptAtBlockHeight executes script at the given block height . +func (r *RestProxyHandler) ExecuteScriptAtBlockHeight(ctx context.Context, blockHeight uint64, script []byte, arguments [][]byte) ([]byte, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + executeScriptAtBlockHeightRequest := &accessproto.ExecuteScriptAtBlockHeightRequest{ + BlockHeight: blockHeight, + Script: script, + Arguments: arguments, + } + executeScriptAtBlockHeightResponse, err := upstream.ExecuteScriptAtBlockHeight(ctx, executeScriptAtBlockHeightRequest) + r.log("upstream", "ExecuteScriptAtBlockHeight", err) + + if err != nil { + return nil, err + } + + return executeScriptAtBlockHeightResponse.Value, nil +} + +// ExecuteScriptAtBlockID executes script at the given block id . +func (r *RestProxyHandler) ExecuteScriptAtBlockID(ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte) ([]byte, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + executeScriptAtBlockIDRequest := &accessproto.ExecuteScriptAtBlockIDRequest{ + BlockId: blockID[:], + Script: script, + Arguments: arguments, + } + executeScriptAtBlockIDResponse, err := upstream.ExecuteScriptAtBlockID(ctx, executeScriptAtBlockIDRequest) + r.log("upstream", "ExecuteScriptAtBlockID", err) + + if err != nil { + return nil, err + } + + return executeScriptAtBlockIDResponse.Value, nil +} + +// GetEventsForHeightRange returns events by their name in the specified blocks heights. +func (r *RestProxyHandler) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64) ([]flow.BlockEvents, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getEventsForHeightRangeRequest := &accessproto.GetEventsForHeightRangeRequest{ + Type: eventType, + StartHeight: startHeight, + EndHeight: endHeight, + } + eventsResponse, err := upstream.GetEventsForHeightRange(ctx, getEventsForHeightRangeRequest) + r.log("upstream", "GetEventsForHeightRange", err) + + if err != nil { + return nil, err + } + + return convert.MessagesToBlockEvents(eventsResponse.Results), nil +} + +// GetEventsForBlockIDs returns events by their name in the specified block IDs. +func (r *RestProxyHandler) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + blockIds := convert.IdentifiersToMessages(blockIDs) + + getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ + Type: eventType, + BlockIds: blockIds, + } + eventsResponse, err := upstream.GetEventsForBlockIDs(ctx, getEventsForBlockIDsRequest) + r.log("upstream", "GetEventsForBlockIDs", err) + + if err != nil { + return nil, err + } + + return convert.MessagesToBlockEvents(eventsResponse.Results), nil +} + +// GetExecutionResultForBlockID gets execution result by provided block ID. +func (r *RestProxyHandler) GetExecutionResultForBlockID(ctx context.Context, blockID flow.Identifier) (*flow.ExecutionResult, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getExecutionResultForBlockID := &accessproto.GetExecutionResultForBlockIDRequest{ + BlockId: blockID[:], + } + executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(ctx, getExecutionResultForBlockID) + r.log("upstream", "GetExecutionResultForBlockID", err) + + if err != nil { + return nil, err + } + + return convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) +} + +// GetExecutionResultByID gets execution result by its ID. +func (r *RestProxyHandler) GetExecutionResultByID(ctx context.Context, id flow.Identifier) (*flow.ExecutionResult, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + executionResultByIDRequest := &accessproto.GetExecutionResultByIDRequest{ + Id: id[:], + } + + executionResultByIDResponse, err := upstream.GetExecutionResultByID(ctx, executionResultByIDRequest) + r.log("upstream", "GetExecutionResultByID", err) + + if err != nil { + return nil, err + } + + return convert.MessageToExecutionResult(executionResultByIDResponse.ExecutionResult) +} diff --git a/engine/access/rest/middleware/metrics.go b/engine/access/rest/middleware/metrics.go index c0d51d36eb6..54dd5dd2c6a 100644 --- a/engine/access/rest/middleware/metrics.go +++ b/engine/access/rest/middleware/metrics.go @@ -3,24 +3,20 @@ package middleware import ( "net/http" - "github.com/onflow/flow-go/module/metrics" - "github.com/slok/go-http-metrics/middleware" "github.com/slok/go-http-metrics/middleware/std" - metricsProm "github.com/slok/go-http-metrics/metrics/prometheus" - "github.com/gorilla/mux" -) -func MetricsMiddleware() mux.MiddlewareFunc { - r := metrics.NewRestCollector(metricsProm.Config{Prefix: "access_rest_api"}) - metricsMiddleware := middleware.New(middleware.Config{Recorder: r}) + "github.com/onflow/flow-go/module" +) +func MetricsMiddleware(restCollector module.RestMetrics) mux.MiddlewareFunc { + metricsMiddleware := middleware.New(middleware.Config{Recorder: restCollector}) return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { // This is a custom metric being called on every http request - r.AddTotalRequests(req.Context(), req.Method, req.URL.Path) + restCollector.AddTotalRequests(req.Context(), req.Method, req.URL.Path) // Modify the writer respWriter := &responseWriter{w, http.StatusOK} diff --git a/engine/access/rest/error.go b/engine/access/rest/models/error.go similarity index 98% rename from engine/access/rest/error.go rename to engine/access/rest/models/error.go index 7403510ba55..2247b38743b 100644 --- a/engine/access/rest/error.go +++ b/engine/access/rest/models/error.go @@ -1,4 +1,4 @@ -package rest +package models import "net/http" diff --git a/engine/access/rest/models/event.go b/engine/access/rest/models/event.go index b8af9e11d81..929dbb3f42c 100644 --- a/engine/access/rest/models/event.go +++ b/engine/access/rest/models/event.go @@ -41,11 +41,6 @@ type BlocksEvents []BlockEvents func (b *BlocksEvents) Build(blocksEvents []flow.BlockEvents) { evs := make([]BlockEvents, 0) for _, ev := range blocksEvents { - // don't include blocks without events - if len(ev.Events) == 0 { - continue - } - var blockEvent BlockEvents blockEvent.Build(ev) evs = append(evs, blockEvent) diff --git a/engine/access/rest/models/execution_result.go b/engine/access/rest/models/execution_result.go index 9a39b1a14b8..a8048b09883 100644 --- a/engine/access/rest/models/execution_result.go +++ b/engine/access/rest/models/execution_result.go @@ -5,7 +5,10 @@ import ( "github.com/onflow/flow-go/model/flow" ) -func (e *ExecutionResult) Build(exeResult *flow.ExecutionResult, link LinkGenerator) error { +func (e *ExecutionResult) Build( + exeResult *flow.ExecutionResult, + link LinkGenerator, +) error { self, err := SelfLink(exeResult.ID(), link.ExecutionResultLink) if err != nil { return err @@ -14,7 +17,7 @@ func (e *ExecutionResult) Build(exeResult *flow.ExecutionResult, link LinkGenera events := make([]Event, len(exeResult.ServiceEvents)) for i, e := range exeResult.ServiceEvents { events[i] = Event{ - Type_: e.Type, + Type_: e.Type.String(), } } diff --git a/engine/access/rest/models/model_node_version_info.go b/engine/access/rest/models/model_node_version_info.go new file mode 100644 index 00000000000..0e29f8d480a --- /dev/null +++ b/engine/access/rest/models/model_node_version_info.go @@ -0,0 +1,16 @@ +/* + * Access API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 1.0.0 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ +package models + +type NodeVersionInfo struct { + Semver string `json:"semver"` + Commit string `json:"commit"` + SporkId string `json:"spork_id"` + ProtocolVersion string `json:"protocol_version"` +} diff --git a/engine/access/rest/models/model_transaction_result.go b/engine/access/rest/models/model_transaction_result.go index 80a59bb91b0..59bcef536b6 100644 --- a/engine/access/rest/models/model_transaction_result.go +++ b/engine/access/rest/models/model_transaction_result.go @@ -9,10 +9,11 @@ package models type TransactionResult struct { - BlockId string `json:"block_id"` - Execution *TransactionExecution `json:"execution,omitempty"` - Status *TransactionStatus `json:"status"` - StatusCode int32 `json:"status_code"` + BlockId string `json:"block_id"` + CollectionId string `json:"collection_id"` + Execution *TransactionExecution `json:"execution,omitempty"` + Status *TransactionStatus `json:"status"` + StatusCode int32 `json:"status_code"` // Provided transaction error in case the transaction wasn't successful. ErrorMessage string `json:"error_message"` ComputationUsed string `json:"computation_used"` diff --git a/engine/access/rest/models/node_version_info.go b/engine/access/rest/models/node_version_info.go new file mode 100644 index 00000000000..6a85e9f8d42 --- /dev/null +++ b/engine/access/rest/models/node_version_info.go @@ -0,0 +1,13 @@ +package models + +import ( + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/util" +) + +func (t *NodeVersionInfo) Build(params *access.NodeVersionInfo) { + t.Semver = params.Semver + t.Commit = params.Commit + t.SporkId = params.SporkId.String() + t.ProtocolVersion = util.FromUint64(params.ProtocolVersion) +} diff --git a/engine/access/rest/models/transaction.go b/engine/access/rest/models/transaction.go index a20ebf30513..5553ec5bec6 100644 --- a/engine/access/rest/models/transaction.go +++ b/engine/access/rest/models/transaction.go @@ -98,6 +98,10 @@ func (t *TransactionResult) Build(txr *access.TransactionResult, txID flow.Ident t.BlockId = txr.BlockID.String() } + if txr.CollectionID != flow.ZeroID { // don't send back 0 ID + t.CollectionId = txr.CollectionID.String() + } + t.Status = &status t.Execution = &execution t.StatusCode = int32(txr.StatusCode) diff --git a/engine/access/rest/request/get_transaction.go b/engine/access/rest/request/get_transaction.go index 06c7a2492cd..e2748f2ef14 100644 --- a/engine/access/rest/request/get_transaction.go +++ b/engine/access/rest/request/get_transaction.go @@ -1,14 +1,47 @@ package request +import "github.com/onflow/flow-go/model/flow" + const resultExpandable = "result" +const blockIDQueryParam = "block_id" +const collectionIDQueryParam = "collection_id" + +type TransactionOptionals struct { + BlockID flow.Identifier + CollectionID flow.Identifier +} + +func (t *TransactionOptionals) Parse(r *Request) error { + var blockId ID + err := blockId.Parse(r.GetQueryParam(blockIDQueryParam)) + if err != nil { + return err + } + t.BlockID = blockId.Flow() + + var collectionId ID + err = collectionId.Parse(r.GetQueryParam(collectionIDQueryParam)) + if err != nil { + return err + } + t.CollectionID = collectionId.Flow() + + return nil +} type GetTransaction struct { GetByIDRequest + TransactionOptionals ExpandsResult bool } func (g *GetTransaction) Build(r *Request) error { - err := g.GetByIDRequest.Build(r) + err := g.TransactionOptionals.Parse(r) + if err != nil { + return err + } + + err = g.GetByIDRequest.Build(r) g.ExpandsResult = r.Expands(resultExpandable) return err @@ -16,4 +49,16 @@ func (g *GetTransaction) Build(r *Request) error { type GetTransactionResult struct { GetByIDRequest + TransactionOptionals +} + +func (g *GetTransactionResult) Build(r *Request) error { + err := g.TransactionOptionals.Parse(r) + if err != nil { + return err + } + + err = g.GetByIDRequest.Build(r) + + return err } diff --git a/engine/access/rest/accounts.go b/engine/access/rest/routes/accounts.go similarity index 94% rename from engine/access/rest/accounts.go rename to engine/access/rest/routes/accounts.go index 36371bf6c57..972c2ba68ac 100644 --- a/engine/access/rest/accounts.go +++ b/engine/access/rest/routes/accounts.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "github.com/onflow/flow-go/access" @@ -10,7 +10,7 @@ import ( func GetAccount(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetAccountRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it diff --git a/engine/access/rest/accounts_test.go b/engine/access/rest/routes/accounts_test.go similarity index 93% rename from engine/access/rest/accounts_test.go rename to engine/access/rest/routes/accounts_test.go index 61982ff5f9c..b8bebea8e85 100644 --- a/engine/access/rest/accounts_test.go +++ b/engine/access/rest/routes/accounts_test.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "fmt" @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest/middleware" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" ) @@ -33,7 +34,15 @@ func accountURL(t *testing.T, address string, height string) string { return u.String() } -func TestGetAccount(t *testing.T) { +// TestAccessGetAccount tests local getAccount request. +// +// Runs the following tests: +// 1. Get account by address at latest sealed block. +// 2. Get account by address at latest finalized block. +// 3. Get account by address at height. +// 4. Get account by address at height condensed. +// 5. Get invalid account. +func TestAccessGetAccount(t *testing.T) { backend := &mock.API{} t.Run("get by address at latest sealed block", func(t *testing.T) { @@ -165,6 +174,7 @@ func getAccountRequest(t *testing.T, account *flow.Account, height string, expan q.Add(middleware.ExpandQueryParam, fieldParam) req.URL.RawQuery = q.Encode() } + require.NoError(t, err) return req } diff --git a/engine/access/rest/blocks.go b/engine/access/rest/routes/blocks.go similarity index 91% rename from engine/access/rest/blocks.go rename to engine/access/rest/routes/blocks.go index e729f67a9bd..c26f14dd8bf 100644 --- a/engine/access/rest/blocks.go +++ b/engine/access/rest/routes/blocks.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "context" @@ -18,10 +18,11 @@ import ( func GetBlocksByIDs(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockByIDsRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } blocks := make([]*models.Block, len(req.IDs)) + for i, id := range req.IDs { block, err := getBlock(forID(&id), r, backend, link) if err != nil { @@ -33,10 +34,11 @@ func GetBlocksByIDs(r *request.Request, backend access.API, link models.LinkGene return blocks, nil } +// GetBlocksByHeight gets blocks by height. func GetBlocksByHeight(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } if req.FinalHeight || req.SealedHeight { @@ -72,7 +74,7 @@ func GetBlocksByHeight(r *request.Request, backend access.API, link models.LinkG req.EndHeight = latest.Header.Height // overwrite special value height with fetched if req.StartHeight > req.EndHeight { - return nil, NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) + return nil, models.NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) } } @@ -93,7 +95,7 @@ func GetBlocksByHeight(r *request.Request, backend access.API, link models.LinkG func GetBlockPayloadByID(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockPayloadRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } blkProvider := NewBlockProvider(backend, forID(&req.ID)) @@ -194,7 +196,7 @@ func (blkProvider *blockProvider) getBlock(ctx context.Context) (*flow.Block, fl if blkProvider.id != nil { blk, _, err := blkProvider.backend.GetBlockByID(ctx, *blkProvider.id) if err != nil { // unfortunately backend returns internal error status if not found - return nil, flow.BlockStatusUnknown, NewNotFoundError( + return nil, flow.BlockStatusUnknown, models.NewNotFoundError( fmt.Sprintf("error looking up block with ID %s", blkProvider.id.String()), err, ) } @@ -205,14 +207,14 @@ func (blkProvider *blockProvider) getBlock(ctx context.Context) (*flow.Block, fl blk, status, err := blkProvider.backend.GetLatestBlock(ctx, blkProvider.sealed) if err != nil { // cannot be a 'not found' error since final and sealed block should always be found - return nil, flow.BlockStatusUnknown, NewRestError(http.StatusInternalServerError, "block lookup failed", err) + return nil, flow.BlockStatusUnknown, models.NewRestError(http.StatusInternalServerError, "block lookup failed", err) } return blk, status, nil } blk, status, err := blkProvider.backend.GetBlockByHeight(ctx, blkProvider.height) if err != nil { // unfortunately backend returns internal error status if not found - return nil, flow.BlockStatusUnknown, NewNotFoundError( + return nil, flow.BlockStatusUnknown, models.NewNotFoundError( fmt.Sprintf("error looking up block at height %d", blkProvider.height), err, ) } diff --git a/engine/access/rest/blocks_test.go b/engine/access/rest/routes/blocks_test.go similarity index 96% rename from engine/access/rest/blocks_test.go rename to engine/access/rest/routes/blocks_test.go index 7f977b06d69..3abccc9c78a 100644 --- a/engine/access/rest/blocks_test.go +++ b/engine/access/rest/routes/blocks_test.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "fmt" @@ -31,12 +31,12 @@ type testVector struct { expectedResponse string } -// TestGetBlocks tests the get blocks by ID and get blocks by heights API -func TestGetBlocks(t *testing.T) { - backend := &mock.API{} - - blkCnt := 10 - blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt) +func prepareTestVectors(t *testing.T, + blockIDs []string, + heights []string, + blocks []*flow.Block, + executionResults []*flow.ExecutionResult, + blkCnt int) []testVector { singleBlockExpandedResponse := expectedBlockResponsesExpanded(blocks[:1], executionResults[:1], true, flow.BlockStatusUnknown) singleSealedBlockExpandedResponse := expectedBlockResponsesExpanded(blocks[:1], executionResults[:1], true, flow.BlockStatusSealed) @@ -137,6 +137,16 @@ func TestGetBlocks(t *testing.T) { expectedResponse: fmt.Sprintf(`{"code":400, "message": "at most %d IDs can be requested at a time"}`, request.MaxBlockRequestHeightRange), }, } + return testVectors +} + +// TestGetBlocks tests local get blocks by ID and get blocks by heights API +func TestAccessGetBlocks(t *testing.T) { + backend := &mock.API{} + + blkCnt := 10 + blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt) + testVectors := prepareTestVectors(t, blockIDs, heights, blocks, executionResults, blkCnt) for _, tv := range testVectors { responseRec, err := executeRequest(tv.request, backend) diff --git a/engine/access/rest/collections.go b/engine/access/rest/routes/collections.go similarity index 94% rename from engine/access/rest/collections.go rename to engine/access/rest/routes/collections.go index 807be2c0c41..47b6150f480 100644 --- a/engine/access/rest/collections.go +++ b/engine/access/rest/routes/collections.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "github.com/onflow/flow-go/access" @@ -11,7 +11,7 @@ import ( func GetCollectionByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetCollectionRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } collection, err := backend.GetCollectionByID(r.Context(), req.ID) diff --git a/engine/access/rest/collections_test.go b/engine/access/rest/routes/collections_test.go similarity index 99% rename from engine/access/rest/collections_test.go rename to engine/access/rest/routes/collections_test.go index 3981541f3a7..de05152b6d5 100644 --- a/engine/access/rest/collections_test.go +++ b/engine/access/rest/routes/collections_test.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "encoding/json" diff --git a/engine/access/rest/events.go b/engine/access/rest/routes/events.go similarity index 85% rename from engine/access/rest/events.go rename to engine/access/rest/routes/events.go index 2a79939bc21..4f03624c768 100644 --- a/engine/access/rest/events.go +++ b/engine/access/rest/routes/events.go @@ -1,22 +1,21 @@ -package rest +package routes import ( "fmt" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" - - "github.com/onflow/flow-go/access" ) -const blockQueryParam = "block_ids" -const eventTypeQuery = "type" +const BlockQueryParam = "block_ids" +const EventTypeQuery = "type" // GetEvents for the provided block range or list of block IDs filtered by type. func GetEvents(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { req, err := r.GetEventsRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } // if the request has block IDs provided then return events for block IDs @@ -41,7 +40,7 @@ func GetEvents(r *request.Request, backend access.API, _ models.LinkGenerator) ( req.EndHeight = latest.Height // special check after we resolve special height value if req.StartHeight > req.EndHeight { - return nil, NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) + return nil, models.NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) } } diff --git a/engine/access/rest/events_test.go b/engine/access/rest/routes/events_test.go similarity index 67% rename from engine/access/rest/events_test.go rename to engine/access/rest/routes/events_test.go index 560ca224968..47d4d89fd52 100644 --- a/engine/access/rest/events_test.go +++ b/engine/access/rest/routes/events_test.go @@ -1,23 +1,25 @@ -package rest +package routes import ( + "encoding/json" "fmt" + "net/http" "net/url" "strings" "testing" "time" - "github.com/onflow/flow-go/engine/access/rest/util" - - "github.com/onflow/flow-go/access/mock" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/unittest" - mocks "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/access/mock" + "github.com/onflow/flow-go/engine/access/rest/util" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" ) func TestGetEvents(t *testing.T) { @@ -28,8 +30,16 @@ func TestGetEvents(t *testing.T) { for i, e := range events { allBlockIDs[i] = e.BlockID.String() } - startHeight := fmt.Sprintf("%d", events[0].BlockHeight) - endHeight := fmt.Sprintf("%d", events[len(events)-1].BlockHeight) + startHeight := fmt.Sprint(events[0].BlockHeight) + endHeight := fmt.Sprint(events[len(events)-1].BlockHeight) + + // remove events from the last block to test that an empty BlockEvents is returned when the last + // block contains no events + truncatedEvents := append(events[:len(events)-1], flow.BlockEvents{ + BlockHeight: events[len(events)-1].BlockHeight, + BlockID: events[len(events)-1].BlockID, + BlockTimestamp: events[len(events)-1].BlockTimestamp, + }) testVectors := []testVector{ // valid @@ -37,25 +47,31 @@ func TestGetEvents(t *testing.T) { description: "Get events for a single block by ID", request: getEventReq(t, "A.179b6b1cb6755e31.Foo.Bar", "", "", []string{events[0].BlockID.String()}), expectedStatus: http.StatusOK, - expectedResponse: testBlockEventResponse([]flow.BlockEvents{events[0]}), + expectedResponse: testBlockEventResponse(t, []flow.BlockEvents{events[0]}), }, { description: "Get events by all block IDs", request: getEventReq(t, "A.179b6b1cb6755e31.Foo.Bar", "", "", allBlockIDs), expectedStatus: http.StatusOK, - expectedResponse: testBlockEventResponse(events), + expectedResponse: testBlockEventResponse(t, events), }, { description: "Get events for height range", request: getEventReq(t, "A.179b6b1cb6755e31.Foo.Bar", startHeight, endHeight, nil), expectedStatus: http.StatusOK, - expectedResponse: testBlockEventResponse(events), + expectedResponse: testBlockEventResponse(t, events), }, { - description: "Get invalid - invalid height format", + description: "Get events range ending at sealed block", request: getEventReq(t, "A.179b6b1cb6755e31.Foo.Bar", "0", "sealed", nil), expectedStatus: http.StatusOK, - expectedResponse: testBlockEventResponse(events), + expectedResponse: testBlockEventResponse(t, events), + }, + { + description: "Get events range ending after last block", + request: getEventReq(t, "A.179b6b1cb6755e31.Foo.Bar", "0", fmt.Sprint(events[len(events)-1].BlockHeight+5), nil), + expectedStatus: http.StatusOK, + expectedResponse: testBlockEventResponse(t, truncatedEvents), }, // invalid { @@ -121,7 +137,7 @@ func getEventReq(t *testing.T, eventType string, start string, end string, block q := u.Query() if len(blockIDs) > 0 { - q.Add(blockQueryParam, strings.Join(blockIDs, ",")) + q.Add(BlockQueryParam, strings.Join(blockIDs, ",")) } if start != "" && end != "" { @@ -129,7 +145,7 @@ func getEventReq(t *testing.T, eventType string, start string, end string, block q.Add(endHeightQueryParam, end) } - q.Add(eventTypeQuery, eventType) + q.Add(EventTypeQuery, eventType) u.RawQuery = q.Encode() @@ -143,6 +159,7 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { events := make([]flow.BlockEvents, n) ids := make([]flow.Identifier, n) + var lastHeader *flow.Header for i := 0; i < n; i++ { header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(uint64(i))) ids[i] = header.ID() @@ -152,12 +169,15 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { backend.Mock. On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, []flow.Identifier{header.ID()}). Return([]flow.BlockEvents{events[i]}, nil) + + lastHeader = header } backend.Mock. On("GetEventsForBlockIDs", mocks.Anything, mocks.Anything, ids). Return(events, nil) + // range from first to last block backend.Mock.On( "GetEventsForHeightRange", mocks.Anything, @@ -166,6 +186,15 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { events[len(events)-1].BlockHeight, ).Return(events, nil) + // range from first to last block + 5 + backend.Mock.On( + "GetEventsForHeightRange", + mocks.Anything, + mocks.Anything, + events[0].BlockHeight, + events[len(events)-1].BlockHeight+5, + ).Return(append(events[:len(events)-1], unittest.BlockEventsFixture(lastHeader, 0)), nil) + latestBlock := unittest.BlockHeaderFixture() latestBlock.Height = uint64(n - 1) @@ -185,34 +214,48 @@ func generateEventsMocks(backend *mock.API, n int) []flow.BlockEvents { return events } -func testBlockEventResponse(events []flow.BlockEvents) string { - res := make([]string, len(events)) +func testBlockEventResponse(t *testing.T, events []flow.BlockEvents) string { + + type eventResponse struct { + Type flow.EventType `json:"type"` + TransactionID flow.Identifier `json:"transaction_id"` + TransactionIndex string `json:"transaction_index"` + EventIndex string `json:"event_index"` + Payload string `json:"payload"` + } + + type blockEventsResponse struct { + BlockID flow.Identifier `json:"block_id"` + BlockHeight string `json:"block_height"` + BlockTimestamp string `json:"block_timestamp"` + Events []eventResponse `json:"events,omitempty"` + } + + res := make([]blockEventsResponse, len(events)) for i, e := range events { - events := make([]string, len(e.Events)) + events := make([]eventResponse, len(e.Events)) for i, ev := range e.Events { - events[i] = fmt.Sprintf(`{ - "type": "%s", - "transaction_id": "%s", - "transaction_index": "%d", - "event_index": "%d", - "payload": "%s" - }`, ev.Type, ev.TransactionID, ev.TransactionIndex, ev.EventIndex, util.ToBase64(ev.Payload)) + events[i] = eventResponse{ + Type: ev.Type, + TransactionID: ev.TransactionID, + TransactionIndex: fmt.Sprint(ev.TransactionIndex), + EventIndex: fmt.Sprint(ev.EventIndex), + Payload: util.ToBase64(ev.Payload), + } } - res[i] = fmt.Sprintf(`{ - "block_id": "%s", - "block_height": "%d", - "block_timestamp": "%s", - "events": [%s] - }`, - e.BlockID.String(), - e.BlockHeight, - e.BlockTimestamp.Format(time.RFC3339Nano), - strings.Join(events, ","), - ) + res[i] = blockEventsResponse{ + BlockID: e.BlockID, + BlockHeight: fmt.Sprint(e.BlockHeight), + BlockTimestamp: e.BlockTimestamp.Format(time.RFC3339Nano), + Events: events, + } } - return fmt.Sprintf(`[%s]`, strings.Join(res, ",")) + data, err := json.Marshal(res) + require.NoError(t, err) + + return string(data) } diff --git a/engine/access/rest/execution_result.go b/engine/access/rest/routes/execution_result.go similarity index 89% rename from engine/access/rest/execution_result.go rename to engine/access/rest/routes/execution_result.go index b0583d43b0d..b999665b26b 100644 --- a/engine/access/rest/execution_result.go +++ b/engine/access/rest/routes/execution_result.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "fmt" @@ -12,7 +12,7 @@ import ( func GetExecutionResultsByBlockIDs(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetExecutionResultByBlockIDsRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } // for each block ID we retrieve execution result @@ -38,7 +38,7 @@ func GetExecutionResultsByBlockIDs(r *request.Request, backend access.API, link func GetExecutionResultByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetExecutionResultRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } res, err := backend.GetExecutionResultByID(r.Context(), req.ID) @@ -48,7 +48,7 @@ func GetExecutionResultByID(r *request.Request, backend access.API, link models. if res == nil { err := fmt.Errorf("execution result with ID: %s not found", req.ID.String()) - return nil, NewNotFoundError(err.Error(), err) + return nil, models.NewNotFoundError(err.Error(), err) } var response models.ExecutionResult diff --git a/engine/access/rest/execution_result_test.go b/engine/access/rest/routes/execution_result_test.go similarity index 99% rename from engine/access/rest/execution_result_test.go rename to engine/access/rest/routes/execution_result_test.go index adb3852c668..ba74974af1a 100644 --- a/engine/access/rest/execution_result_test.go +++ b/engine/access/rest/routes/execution_result_test.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "fmt" @@ -37,7 +37,6 @@ func getResultByIDReq(id string, blockIDs []string) *http.Request { } func TestGetResultByID(t *testing.T) { - t.Run("get by ID", func(t *testing.T) { backend := &mock.API{} result := unittest.ExecutionResultFixture() @@ -68,6 +67,7 @@ func TestGetResultByID(t *testing.T) { } func TestGetResultBlockID(t *testing.T) { + t.Run("get by block ID", func(t *testing.T) { backend := &mock.API{} blockID := unittest.IdentifierFixture() diff --git a/engine/access/rest/handler.go b/engine/access/rest/routes/handler.go similarity index 95% rename from engine/access/rest/handler.go rename to engine/access/rest/routes/handler.go index 028176fc9e0..e323843e50e 100644 --- a/engine/access/rest/handler.go +++ b/engine/access/rest/routes/handler.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "encoding/json" @@ -6,17 +6,17 @@ import ( "fmt" "net/http" - "github.com/onflow/flow-go/engine/access/rest/models" - "github.com/onflow/flow-go/engine/access/rest/request" - "github.com/onflow/flow-go/engine/access/rest/util" - fvmErrors "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/model/flow" - "github.com/rs/zerolog" + "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/models" + "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/engine/access/rest/util" + fvmErrors "github.com/onflow/flow-go/fvm/errors" + "github.com/onflow/flow-go/model/flow" ) const MaxRequestSize = 2 << 20 // 2MB @@ -93,7 +93,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *Handler) errorHandler(w http.ResponseWriter, err error, errorLogger zerolog.Logger) { // rest status type error should be returned with status and user message provided - var statusErr StatusError + var statusErr models.StatusError if errors.As(err, &statusErr) { h.errorResponse(w, statusErr.Status(), statusErr.UserMessage(), errorLogger) return @@ -124,6 +124,11 @@ func (h *Handler) errorHandler(w http.ResponseWriter, err error, errorLogger zer h.errorResponse(w, http.StatusBadRequest, msg, errorLogger) return } + if se.Code() == codes.Unavailable { + msg := fmt.Sprintf("Failed to process request: %s", se.Message()) + h.errorResponse(w, http.StatusServiceUnavailable, msg, errorLogger) + return + } } // stop going further - catch all error diff --git a/engine/access/rest/network.go b/engine/access/rest/routes/network.go similarity index 87% rename from engine/access/rest/network.go rename to engine/access/rest/routes/network.go index 6100bc765d5..82abcbb6d49 100644 --- a/engine/access/rest/network.go +++ b/engine/access/rest/routes/network.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "github.com/onflow/flow-go/access" @@ -7,7 +7,7 @@ import ( ) // GetNetworkParameters returns network-wide parameters of the blockchain -func GetNetworkParameters(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { +func GetNetworkParameters(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { params := backend.GetNetworkParameters(r.Context()) var response models.NetworkParameters diff --git a/engine/access/rest/network_test.go b/engine/access/rest/routes/network_test.go similarity index 98% rename from engine/access/rest/network_test.go rename to engine/access/rest/routes/network_test.go index c4ce7492476..00d0ca03944 100644 --- a/engine/access/rest/network_test.go +++ b/engine/access/rest/routes/network_test.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "fmt" diff --git a/engine/access/rest/routes/node_version_info.go b/engine/access/rest/routes/node_version_info.go new file mode 100644 index 00000000000..31e172bba9f --- /dev/null +++ b/engine/access/rest/routes/node_version_info.go @@ -0,0 +1,19 @@ +package routes + +import ( + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/models" + "github.com/onflow/flow-go/engine/access/rest/request" +) + +// GetNodeVersionInfo returns node version information +func GetNodeVersionInfo(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { + params, err := backend.GetNodeVersionInfo(r.Context()) + if err != nil { + return nil, err + } + + var response models.NodeVersionInfo + response.Build(params) + return response, nil +} diff --git a/engine/access/rest/routes/node_version_info_test.go b/engine/access/rest/routes/node_version_info_test.go new file mode 100644 index 00000000000..179d339f94f --- /dev/null +++ b/engine/access/rest/routes/node_version_info_test.go @@ -0,0 +1,62 @@ +package routes + +import ( + "fmt" + "net/http" + "net/url" + "testing" + + mocktestify "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/access/mock" + "github.com/onflow/flow-go/cmd/build" + "github.com/onflow/flow-go/utils/unittest" +) + +func nodeVersionInfoURL(t *testing.T) string { + u, err := url.ParseRequestURI("/v1/node_version_info") + require.NoError(t, err) + + return u.String() +} + +func TestGetNodeVersionInfo(t *testing.T) { + backend := mock.NewAPI(t) + + t.Run("get node version info", func(t *testing.T) { + req := getNodeVersionInfoRequest(t) + + params := &access.NodeVersionInfo{ + Semver: build.Version(), + Commit: build.Commit(), + SporkId: unittest.IdentifierFixture(), + ProtocolVersion: unittest.Uint64InRange(10, 30), + } + + backend.Mock. + On("GetNodeVersionInfo", mocktestify.Anything). + Return(params, nil) + + expected := nodeVersionInfoExpectedStr(params) + + assertOKResponse(t, req, expected, backend) + mocktestify.AssertExpectationsForObjects(t, backend) + }) +} + +func nodeVersionInfoExpectedStr(nodeVersionInfo *access.NodeVersionInfo) string { + return fmt.Sprintf(`{ + "semver": "%s", + "commit": "%s", + "spork_id": "%s", + "protocol_version": "%d" + }`, nodeVersionInfo.Semver, nodeVersionInfo.Commit, nodeVersionInfo.SporkId.String(), nodeVersionInfo.ProtocolVersion) +} + +func getNodeVersionInfoRequest(t *testing.T) *http.Request { + req, err := http.NewRequest("GET", nodeVersionInfoURL(t), nil) + require.NoError(t, err) + return req +} diff --git a/engine/access/rest/router.go b/engine/access/rest/routes/router.go similarity index 57% rename from engine/access/rest/router.go rename to engine/access/rest/routes/router.go index d750c000578..a2185e4e9a3 100644 --- a/engine/access/rest/router.go +++ b/engine/access/rest/routes/router.go @@ -1,7 +1,10 @@ -package rest +package routes import ( + "fmt" "net/http" + "regexp" + "strings" "github.com/gorilla/mux" "github.com/rs/zerolog" @@ -10,9 +13,10 @@ import ( "github.com/onflow/flow-go/engine/access/rest/middleware" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" ) -func newRouter(backend access.API, logger zerolog.Logger, chain flow.Chain) (*mux.Router, error) { +func NewRouter(backend access.API, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*mux.Router, error) { router := mux.NewRouter().StrictSlash(true) v1SubRouter := router.PathPrefix("/v1").Subrouter() @@ -20,7 +24,7 @@ func newRouter(backend access.API, logger zerolog.Logger, chain flow.Chain) (*mu v1SubRouter.Use(middleware.LoggingMiddleware(logger)) v1SubRouter.Use(middleware.QueryExpandable()) v1SubRouter.Use(middleware.QuerySelect()) - v1SubRouter.Use(middleware.MetricsMiddleware()) + v1SubRouter.Use(middleware.MetricsMiddleware(restCollector)) linkGenerator := models.NewLinkGeneratorImpl(v1SubRouter) @@ -107,4 +111,65 @@ var Routes = []route{{ Pattern: "/network/parameters", Name: "getNetworkParameters", Handler: GetNetworkParameters, +}, { + Method: http.MethodGet, + Pattern: "/node_version_info", + Name: "getNodeVersionInfo", + Handler: GetNodeVersionInfo, }} + +var routeUrlMap = map[string]string{} +var routeRE = regexp.MustCompile(`(?i)/v1/(\w+)(/(\w+)(/(\w+))?)?`) + +func init() { + for _, r := range Routes { + routeUrlMap[r.Pattern] = r.Name + } +} + +func URLToRoute(url string) (string, error) { + normalized, err := normalizeURL(url) + if err != nil { + return "", err + } + + name, ok := routeUrlMap[normalized] + if !ok { + return "", fmt.Errorf("invalid url") + } + return name, nil +} + +func normalizeURL(url string) (string, error) { + matches := routeRE.FindAllStringSubmatch(url, -1) + if len(matches) != 1 || len(matches[0]) != 6 { + return "", fmt.Errorf("invalid url") + } + + // given a URL like + // /v1/blocks/1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef/payload + // groups [ 1 ] [ 3 ] [ 5 ] + // normalized form like /v1/blocks/{id}/payload + + parts := []string{matches[0][1]} + + switch len(matches[0][3]) { + case 0: + // top level resource. e.g. /v1/blocks + case 64: + // id based resource. e.g. /v1/blocks/1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef + parts = append(parts, "{id}") + case 16: + // address based resource. e.g. /v1/accounts/1234567890abcdef + parts = append(parts, "{address}") + default: + // named resource. e.g. /v1/network/parameters + parts = append(parts, matches[0][3]) + } + + if matches[0][5] != "" { + parts = append(parts, matches[0][5]) + } + + return "/" + strings.Join(parts, "/"), nil +} diff --git a/engine/access/rest/routes/router_test.go b/engine/access/rest/routes/router_test.go new file mode 100644 index 00000000000..e3c2a2c3fdd --- /dev/null +++ b/engine/access/rest/routes/router_test.go @@ -0,0 +1,185 @@ +package routes + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseURL(t *testing.T) { + tests := []struct { + name string + url string + expected string + }{ + { + name: "/v1/transactions", + url: "/v1/transactions", + expected: "createTransaction", + }, + { + name: "/v1/transactions/{id}", + url: "/v1/transactions/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getTransactionByID", + }, + { + name: "/v1/transaction_results/{id}", + url: "/v1/transaction_results/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getTransactionResultByID", + }, + { + name: "/v1/blocks", + url: "/v1/blocks", + expected: "getBlocksByHeight", + }, + { + name: "/v1/blocks/{id}", + url: "/v1/blocks/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getBlocksByIDs", + }, + { + name: "/v1/blocks/{id}/payload", + url: "/v1/blocks/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76/payload", + expected: "getBlockPayloadByID", + }, + { + name: "/v1/execution_results/{id}", + url: "/v1/execution_results/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getExecutionResultByID", + }, + { + name: "/v1/execution_results", + url: "/v1/execution_results", + expected: "getExecutionResultByBlockID", + }, + { + name: "/v1/collections/{id}", + url: "/v1/collections/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getCollectionByID", + }, + { + name: "/v1/scripts", + url: "/v1/scripts", + expected: "executeScript", + }, + { + name: "/v1/accounts/{address}", + url: "/v1/accounts/6a587be304c1224c", + expected: "getAccount", + }, + { + name: "/v1/events", + url: "/v1/events", + expected: "getEvents", + }, + { + name: "/v1/network/parameters", + url: "/v1/network/parameters", + expected: "getNetworkParameters", + }, + { + name: "/v1/node_version_info", + url: "/v1/node_version_info", + expected: "getNodeVersionInfo", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := URLToRoute(tt.url) + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} + +func TestBenchmarkParseURL(t *testing.T) { + tests := []struct { + name string + url string + expected string + }{ + { + name: "/v1/transactions", + url: "/v1/transactions", + expected: "createTransaction", + }, + { + name: "/v1/transactions/{id}", + url: "/v1/transactions/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getTransactionByID", + }, + { + name: "/v1/transaction_results/{id}", + url: "/v1/transaction_results/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getTransactionResultByID", + }, + { + name: "/v1/blocks", + url: "/v1/blocks", + expected: "getBlocksByHeight", + }, + { + name: "/v1/blocks/{id}", + url: "/v1/blocks/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getBlocksByIDs", + }, + { + name: "/v1/blocks/{id}/payload", + url: "/v1/blocks/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76/payload", + expected: "getBlockPayloadByID", + }, + { + name: "/v1/execution_results/{id}", + url: "/v1/execution_results/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getExecutionResultByID", + }, + { + name: "/v1/execution_results", + url: "/v1/execution_results", + expected: "getExecutionResultByBlockID", + }, + { + name: "/v1/collections/{id}", + url: "/v1/collections/53730d3f3d2d2f46cb910b16db817d3a62adaaa72fdb3a92ee373c37c5b55a76", + expected: "getCollectionByID", + }, + { + name: "/v1/scripts", + url: "/v1/scripts", + expected: "executeScript", + }, + { + name: "/v1/accounts/{address}", + url: "/v1/accounts/6a587be304c1224c", + expected: "getAccount", + }, + { + name: "/v1/events", + url: "/v1/events", + expected: "getEvents", + }, + { + name: "/v1/network/parameters", + url: "/v1/network/parameters", + expected: "getNetworkParameters", + }, + { + name: "/v1/node_version_info", + url: "/v1/node_version_info", + expected: "getNodeVersionInfo", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + start := time.Now() + for i := 0; i < 100_000; i++ { + _, _ = URLToRoute(tt.url) + } + t.Logf("%s: %v", tt.name, time.Since(start)/100_000) + }) + } +} diff --git a/engine/access/rest/scripts.go b/engine/access/rest/routes/scripts.go similarity index 94% rename from engine/access/rest/scripts.go rename to engine/access/rest/routes/scripts.go index 8bd86bae54f..8627470ab88 100644 --- a/engine/access/rest/scripts.go +++ b/engine/access/rest/routes/scripts.go @@ -1,18 +1,17 @@ -package rest +package routes import ( + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/model/flow" - - "github.com/onflow/flow-go/access" ) // ExecuteScript handler sends the script from the request to be executed. func ExecuteScript(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { req, err := r.GetScriptRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } if req.BlockID != flow.ZeroID { diff --git a/engine/access/rest/scripts_test.go b/engine/access/rest/routes/scripts_test.go similarity index 99% rename from engine/access/rest/scripts_test.go rename to engine/access/rest/routes/scripts_test.go index 7e3271c1d81..8a6a63cc819 100644 --- a/engine/access/rest/scripts_test.go +++ b/engine/access/rest/routes/scripts_test.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "bytes" diff --git a/engine/access/rest/test_helpers.go b/engine/access/rest/routes/test_helpers.go similarity index 76% rename from engine/access/rest/test_helpers.go rename to engine/access/rest/routes/test_helpers.go index eb63376da4e..e512cc94434 100644 --- a/engine/access/rest/test_helpers.go +++ b/engine/access/rest/routes/test_helpers.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "bytes" @@ -11,8 +11,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/access/mock" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" ) const ( @@ -25,10 +26,11 @@ const ( heightQueryParam = "height" ) -func executeRequest(req *http.Request, backend *mock.API) (*httptest.ResponseRecorder, error) { +func executeRequest(req *http.Request, backend access.API) (*httptest.ResponseRecorder, error) { var b bytes.Buffer logger := zerolog.New(&b) - router, err := newRouter(backend, logger, flow.Testnet.Chain()) + + router, err := NewRouter(backend, logger, flow.Testnet.Chain(), metrics.NewNoopCollector()) if err != nil { return nil, err } @@ -38,14 +40,13 @@ func executeRequest(req *http.Request, backend *mock.API) (*httptest.ResponseRec return rr, nil } -func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, backend *mock.API) { +func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, backend access.API) { assertResponse(t, req, http.StatusOK, expectedRespBody, backend) } -func assertResponse(t *testing.T, req *http.Request, status int, expectedRespBody string, backend *mock.API) { +func assertResponse(t *testing.T, req *http.Request, status int, expectedRespBody string, backend access.API) { rr, err := executeRequest(req, backend) assert.NoError(t, err) - actualResponseBody := rr.Body.String() require.JSONEq(t, expectedRespBody, diff --git a/engine/access/rest/transactions.go b/engine/access/rest/routes/transactions.go similarity index 82% rename from engine/access/rest/transactions.go rename to engine/access/rest/routes/transactions.go index 21b6c300c95..b77aead82b4 100644 --- a/engine/access/rest/transactions.go +++ b/engine/access/rest/routes/transactions.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "github.com/onflow/flow-go/access" @@ -10,7 +10,7 @@ import ( func GetTransactionByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetTransactionRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } tx, err := backend.GetTransaction(r.Context(), req.ID) @@ -21,7 +21,7 @@ func GetTransactionByID(r *request.Request, backend access.API, link models.Link var txr *access.TransactionResult // only lookup result if transaction result is to be expanded if req.ExpandsResult { - txr, err = backend.GetTransactionResult(r.Context(), req.ID) + txr, err = backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID) if err != nil { return nil, err } @@ -36,10 +36,10 @@ func GetTransactionByID(r *request.Request, backend access.API, link models.Link func GetTransactionResultByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetTransactionResultRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } - txr, err := backend.GetTransactionResult(r.Context(), req.ID) + txr, err := backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID) if err != nil { return nil, err } @@ -53,7 +53,7 @@ func GetTransactionResultByID(r *request.Request, backend access.API, link model func CreateTransaction(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.CreateTransactionRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } err = backend.SendTransaction(r.Context(), &req.Transaction) diff --git a/engine/access/rest/transactions_test.go b/engine/access/rest/routes/transactions_test.go similarity index 74% rename from engine/access/rest/transactions_test.go rename to engine/access/rest/routes/transactions_test.go index f41c4d44787..3b02c4d5de5 100644 --- a/engine/access/rest/transactions_test.go +++ b/engine/access/rest/routes/transactions_test.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "bytes" @@ -23,21 +23,43 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -func getTransactionReq(id string, expandResult bool) *http.Request { +func getTransactionReq(id string, expandResult bool, blockIdQuery string, collectionIdQuery string) *http.Request { u, _ := url.Parse(fmt.Sprintf("/v1/transactions/%s", id)) + q := u.Query() + if expandResult { - q := u.Query() // by default expand all since we test expanding with converters q.Add("expand", "result") - u.RawQuery = q.Encode() } + if blockIdQuery != "" { + q.Add("block_id", blockIdQuery) + } + + if collectionIdQuery != "" { + q.Add("collection_id", collectionIdQuery) + } + + u.RawQuery = q.Encode() + req, _ := http.NewRequest("GET", u.String(), nil) return req } -func getTransactionResultReq(id string) *http.Request { - req, _ := http.NewRequest("GET", fmt.Sprintf("/v1/transaction_results/%s", id), nil) +func getTransactionResultReq(id string, blockIdQuery string, collectionIdQuery string) *http.Request { + u, _ := url.Parse(fmt.Sprintf("/v1/transaction_results/%s", id)) + q := u.Query() + if blockIdQuery != "" { + q.Add("block_id", blockIdQuery) + } + + if collectionIdQuery != "" { + q.Add("collection_id", collectionIdQuery) + } + + u.RawQuery = q.Encode() + + req, _ := http.NewRequest("GET", u.String(), nil) return req } @@ -47,44 +69,11 @@ func createTransactionReq(body interface{}) *http.Request { return req } -func validCreateBody(tx flow.TransactionBody) map[string]interface{} { - tx.Arguments = [][]uint8{} // fix how fixture creates nil values - auth := make([]string, len(tx.Authorizers)) - for i, a := range tx.Authorizers { - auth[i] = a.String() - } - - return map[string]interface{}{ - "script": util.ToBase64(tx.Script), - "arguments": tx.Arguments, - "reference_block_id": tx.ReferenceBlockID.String(), - "gas_limit": fmt.Sprintf("%d", tx.GasLimit), - "payer": tx.Payer.String(), - "proposal_key": map[string]interface{}{ - "address": tx.ProposalKey.Address.String(), - "key_index": fmt.Sprintf("%d", tx.ProposalKey.KeyIndex), - "sequence_number": fmt.Sprintf("%d", tx.ProposalKey.SequenceNumber), - }, - "authorizers": auth, - "payload_signatures": []map[string]interface{}{{ - "address": tx.PayloadSignatures[0].Address.String(), - "key_index": fmt.Sprintf("%d", tx.PayloadSignatures[0].KeyIndex), - "signature": util.ToBase64(tx.PayloadSignatures[0].Signature), - }}, - "envelope_signatures": []map[string]interface{}{{ - "address": tx.EnvelopeSignatures[0].Address.String(), - "key_index": fmt.Sprintf("%d", tx.EnvelopeSignatures[0].KeyIndex), - "signature": util.ToBase64(tx.EnvelopeSignatures[0].Signature), - }}, - } -} - func TestGetTransactions(t *testing.T) { - t.Run("get by ID without results", func(t *testing.T) { backend := &mock.API{} tx := unittest.TransactionFixture() - req := getTransactionReq(tx.ID().String(), false) + req := getTransactionReq(tx.ID().String(), false, "", "") backend.Mock. On("GetTransaction", mocks.Anything, tx.ID()). @@ -128,6 +117,7 @@ func TestGetTransactions(t *testing.T) { t.Run("Get by ID with results", func(t *testing.T) { backend := &mock.API{} + tx := unittest.TransactionFixture() txr := transactionResultFixture(tx) @@ -136,10 +126,10 @@ func TestGetTransactions(t *testing.T) { Return(&tx.TransactionBody, nil) backend.Mock. - On("GetTransactionResult", mocks.Anything, tx.ID()). + On("GetTransactionResult", mocks.Anything, tx.ID(), flow.ZeroID, flow.ZeroID). Return(txr, nil) - req := getTransactionReq(tx.ID().String(), true) + req := getTransactionReq(tx.ID().String(), true, "", "") expected := fmt.Sprintf(` { @@ -167,6 +157,7 @@ func TestGetTransactions(t *testing.T) { ], "result": { "block_id": "%s", + "collection_id": "%s", "execution": "Success", "status": "Sealed", "status_code": 1, @@ -190,22 +181,23 @@ func TestGetTransactions(t *testing.T) { "_self":"/v1/transactions/%s" } }`, - tx.ID(), tx.ReferenceBlockID, util.ToBase64(tx.EnvelopeSignatures[0].Signature), tx.ReferenceBlockID, tx.ID(), tx.ID(), tx.ID()) + tx.ID(), tx.ReferenceBlockID, util.ToBase64(tx.EnvelopeSignatures[0].Signature), tx.ReferenceBlockID, txr.CollectionID, tx.ID(), tx.ID(), tx.ID()) assertOKResponse(t, req, expected, backend) }) t.Run("get by ID Invalid", func(t *testing.T) { backend := &mock.API{} - req := getTransactionReq("invalid", false) + req := getTransactionReq("invalid", false, "", "") expected := `{"code":400, "message":"invalid ID format"}` assertResponse(t, req, http.StatusBadRequest, expected, backend) }) t.Run("get by ID non-existing", func(t *testing.T) { backend := &mock.API{} + tx := unittest.TransactionFixture() - req := getTransactionReq(tx.ID().String(), false) + req := getTransactionReq(tx.ID().String(), false, "", "") backend.Mock. On("GetTransaction", mocks.Anything, tx.ID()). @@ -217,30 +209,23 @@ func TestGetTransactions(t *testing.T) { } func TestGetTransactionResult(t *testing.T) { - - t.Run("get by ID", func(t *testing.T) { - backend := &mock.API{} - id := unittest.IdentifierFixture() - bid := unittest.IdentifierFixture() - txr := &access.TransactionResult{ - Status: flow.TransactionStatusSealed, - StatusCode: 10, - Events: []flow.Event{ - unittest.EventFixture(flow.EventAccountCreated, 1, 0, id, 200), - }, - ErrorMessage: "", - BlockID: bid, - } - txr.Events[0].Payload = []byte(`test payload`) - - req := getTransactionResultReq(id.String()) - - backend.Mock. - On("GetTransactionResult", mocks.Anything, id). - Return(txr, nil) - - expected := fmt.Sprintf(`{ + id := unittest.IdentifierFixture() + bid := unittest.IdentifierFixture() + cid := unittest.IdentifierFixture() + txr := &access.TransactionResult{ + Status: flow.TransactionStatusSealed, + StatusCode: 10, + Events: []flow.Event{ + unittest.EventFixture(flow.EventAccountCreated, 1, 0, id, 200), + }, + ErrorMessage: "", + BlockID: bid, + CollectionID: cid, + } + txr.Events[0].Payload = []byte(`test payload`) + expected := fmt.Sprintf(`{ "block_id": "%s", + "collection_id": "%s", "execution": "Success", "status": "Sealed", "status_code": 10, @@ -258,14 +243,46 @@ func TestGetTransactionResult(t *testing.T) { "_links": { "_self": "/v1/transaction_results/%s" } - }`, bid.String(), id.String(), util.ToBase64(txr.Events[0].Payload), id.String()) + }`, bid.String(), cid.String(), id.String(), util.ToBase64(txr.Events[0].Payload), id.String()) + + t.Run("get by transaction ID", func(t *testing.T) { + backend := &mock.API{} + + req := getTransactionResultReq(id.String(), "", "") + + backend.Mock. + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID). + Return(txr, nil) + + assertOKResponse(t, req, expected, backend) + }) + + t.Run("get by block ID", func(t *testing.T) { + backend := &mock.API{} + + req := getTransactionResultReq(id.String(), bid.String(), "") + + backend.Mock. + On("GetTransactionResult", mocks.Anything, id, bid, flow.ZeroID). + Return(txr, nil) + + assertOKResponse(t, req, expected, backend) + }) + + t.Run("get by collection ID", func(t *testing.T) { + backend := &mock.API{} + + req := getTransactionResultReq(id.String(), "", cid.String()) + + backend.Mock. + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, cid). + Return(txr, nil) + assertOKResponse(t, req, expected, backend) }) t.Run("get execution statuses", func(t *testing.T) { backend := &mock.API{} - id := unittest.IdentifierFixture() - bid := unittest.IdentifierFixture() testVectors := map[*access.TransactionResult]string{{ Status: flow.TransactionStatusExpired, @@ -287,16 +304,18 @@ func TestGetTransactionResult(t *testing.T) { ErrorMessage: "", }: string(models.SUCCESS_RESULT)} - for txr, err := range testVectors { - txr.BlockID = bid - req := getTransactionResultReq(id.String()) + for txResult, err := range testVectors { + txResult.BlockID = bid + txResult.CollectionID = cid + req := getTransactionResultReq(id.String(), "", "") backend.Mock. - On("GetTransactionResult", mocks.Anything, id). - Return(txr, nil). + On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID). + Return(txResult, nil). Once() - expected := fmt.Sprintf(`{ + expectedResp := fmt.Sprintf(`{ "block_id": "%s", + "collection_id": "%s", "execution": "%s", "status": "%s", "status_code": 0, @@ -306,14 +325,15 @@ func TestGetTransactionResult(t *testing.T) { "_links": { "_self": "/v1/transaction_results/%s" } - }`, bid.String(), err, cases.Title(language.English).String(strings.ToLower(txr.Status.String())), txr.ErrorMessage, id.String()) - assertOKResponse(t, req, expected, backend) + }`, bid.String(), cid.String(), err, cases.Title(language.English).String(strings.ToLower(txResult.Status.String())), txResult.ErrorMessage, id.String()) + assertOKResponse(t, req, expectedResp, backend) } }) t.Run("get by ID Invalid", func(t *testing.T) { backend := &mock.API{} - req := getTransactionResultReq("invalid") + + req := getTransactionResultReq("invalid", "", "") expected := `{"code":400, "message":"invalid ID format"}` assertResponse(t, req, http.StatusBadRequest, expected, backend) @@ -321,13 +341,13 @@ func TestGetTransactionResult(t *testing.T) { } func TestCreateTransaction(t *testing.T) { + backend := &mock.API{} t.Run("create", func(t *testing.T) { - backend := &mock.API{} tx := unittest.TransactionBodyFixture() tx.PayloadSignatures = []flow.TransactionSignature{unittest.TransactionSignatureFixture()} tx.Arguments = [][]uint8{} - req := createTransactionReq(validCreateBody(tx)) + req := createTransactionReq(unittest.CreateSendTxHttpPayload(tx)) backend.Mock. On("SendTransaction", mocks.Anything, &tx). @@ -375,7 +395,6 @@ func TestCreateTransaction(t *testing.T) { }) t.Run("post invalid transaction", func(t *testing.T) { - backend := &mock.API{} tests := []struct { inputField string inputValue string @@ -395,7 +414,7 @@ func TestCreateTransaction(t *testing.T) { for _, test := range tests { tx := unittest.TransactionBodyFixture() tx.PayloadSignatures = []flow.TransactionSignature{unittest.TransactionSignatureFixture()} - testTx := validCreateBody(tx) + testTx := unittest.CreateSendTxHttpPayload(tx) testTx[test.inputField] = test.inputValue req := createTransactionReq(testTx) @@ -405,6 +424,7 @@ func TestCreateTransaction(t *testing.T) { } func transactionResultFixture(tx flow.Transaction) *access.TransactionResult { + cid := unittest.IdentifierFixture() return &access.TransactionResult{ Status: flow.TransactionStatusSealed, StatusCode: 1, @@ -413,5 +433,6 @@ func transactionResultFixture(tx flow.Transaction) *access.TransactionResult { }, ErrorMessage: "", BlockID: tx.ReferenceBlockID, + CollectionID: cid, } } diff --git a/engine/access/rest/server.go b/engine/access/rest/server.go index b7f45bb8645..4a4b1be6f0e 100644 --- a/engine/access/rest/server.go +++ b/engine/access/rest/server.go @@ -8,13 +8,14 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" ) // NewServer returns an HTTP server initialized with the REST API handler -func NewServer(backend access.API, listenAddress string, logger zerolog.Logger, chain flow.Chain) (*http.Server, error) { - - router, err := newRouter(backend, logger, chain) +func NewServer(serverAPI access.API, listenAddress string, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*http.Server, error) { + router, err := routes.NewRouter(serverAPI, logger, chain, restCollector) if err != nil { return nil, err } diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 69bde45c23b..091c5e2e3ad 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -3,7 +3,6 @@ package access import ( "context" "fmt" - "math/rand" "net/http" "os" @@ -11,6 +10,11 @@ import ( "testing" "time" + "google.golang.org/grpc/credentials" + + "github.com/onflow/flow-go/module/grpcserver" + "github.com/onflow/flow-go/utils/grpcutils" + "github.com/antihax/optional" restclient "github.com/onflow/flow/openapi/go-client-generated" "github.com/rs/zerolog" @@ -20,10 +24,12 @@ import ( "github.com/stretchr/testify/suite" accessmock "github.com/onflow/flow-go/engine/access/mock" - "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/engine/access/rpc" + "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network" @@ -50,6 +56,8 @@ type RestAPITestSuite struct { chainID flow.ChainID metrics *metrics.NoopCollector rpcEng *rpc.Engine + sealedBlock *flow.Header + finalizedBlock *flow.Header // storage blocks *storagemock.Blocks @@ -58,6 +66,13 @@ type RestAPITestSuite struct { transactions *storagemock.Transactions receipts *storagemock.ExecutionReceipts executionResults *storagemock.ExecutionResults + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer } func (suite *RestAPITestSuite) SetupTest() { @@ -66,9 +81,23 @@ func (suite *RestAPITestSuite) SetupTest() { suite.state = new(protocol.State) suite.sealedSnaphost = new(protocol.Snapshot) suite.finalizedSnapshot = new(protocol.Snapshot) + suite.sealedBlock = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(0)) + suite.finalizedBlock = unittest.BlockHeaderWithParentFixture(suite.sealedBlock) suite.state.On("Sealed").Return(suite.sealedSnaphost, nil) suite.state.On("Final").Return(suite.finalizedSnapshot, nil) + suite.sealedSnaphost.On("Head").Return( + func() *flow.Header { + return suite.sealedBlock + }, + nil, + ).Maybe() + suite.finalizedSnapshot.On("Head").Return( + func() *flow.Header { + return suite.finalizedBlock + }, + nil, + ).Maybe() suite.blocks = new(storagemock.Blocks) suite.headers = new(storagemock.Headers) suite.transactions = new(storagemock.Transactions) @@ -99,18 +128,87 @@ func (suite *RestAPITestSuite) SetupTest() { RESTListenAddr: unittest.DefaultAddress, } - rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, - nil, suite.executionResults, suite.chainID, suite.metrics, suite.metrics, 0, 0, false, - false, nil, nil) + // generate a server certificate that will be served by the GRPC server + networkingKey := unittest.NetworkingPrivKeyFixture() + x509Certificate, err := grpcutils.X509Certificate(networkingKey) + assert.NoError(suite.T(), err) + tlsConfig := grpcutils.DefaultServerTLSConfig(x509Certificate) + // set the transport credentials for the server to use + config.TransportCredentials = credentials.NewTLS(tlsConfig) + + suite.secureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.SecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + nil, + nil, + grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() + + suite.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.UnsecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + nil, + nil).Build() + + backend := backend.New(suite.state, + suite.collClient, + nil, + suite.blocks, + suite.headers, + suite.collections, + suite.transactions, + nil, + suite.executionResults, + suite.chainID, + suite.metrics, + nil, + false, + 0, + nil, + nil, + suite.log, + 0, + nil, + false) + + rpcEngBuilder, err := rpc.NewBuilder( + suite.log, + suite.state, + config, + suite.chainID, + suite.metrics, + false, + suite.me, + backend, + backend, + suite.secureGrpcServer, + suite.unsecureGrpcServer, + ) assert.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() assert.NoError(suite.T(), err) + + suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + + suite.rpcEng.Start(suite.ctx) + + suite.secureGrpcServer.Start(suite.ctx) + suite.unsecureGrpcServer.Start(suite.ctx) + + // wait for the servers to startup + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) + + // wait for the engine to startup unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second) +} - // wait for the server to startup - assert.Eventually(suite.T(), func() bool { - return suite.rpcEng.RestApiAddress() != nil - }, 5*time.Second, 10*time.Millisecond) +func (suite *RestAPITestSuite) TearDownTest() { + suite.cancel() + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Done(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Done(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) } func TestRestAPI(t *testing.T) { @@ -136,10 +234,8 @@ func (suite *RestAPITestSuite) TestGetBlock() { suite.executionResults.On("ByBlockID", block.ID()).Return(execResult, nil) } - sealedBlock := testBlocks[len(testBlocks)-1] - finalizedBlock := testBlocks[len(testBlocks)-2] - suite.sealedSnaphost.On("Head").Return(sealedBlock.Header, nil) - suite.finalizedSnapshot.On("Head").Return(finalizedBlock.Header, nil) + suite.sealedBlock = testBlocks[len(testBlocks)-1].Header + suite.finalizedBlock = testBlocks[len(testBlocks)-2].Header client := suite.restAPIClient() @@ -227,7 +323,7 @@ func (suite *RestAPITestSuite) TestGetBlock() { require.NoError(suite.T(), err) assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) assert.Len(suite.T(), actualBlocks, 1) - assert.Equal(suite.T(), finalizedBlock.ID().String(), actualBlocks[0].Header.Id) + assert.Equal(suite.T(), suite.finalizedBlock.ID().String(), actualBlocks[0].Header.Id) }) suite.Run("GetBlockByHeight for height=sealed happy path", func() { @@ -239,7 +335,7 @@ func (suite *RestAPITestSuite) TestGetBlock() { require.NoError(suite.T(), err) assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) assert.Len(suite.T(), actualBlocks, 1) - assert.Equal(suite.T(), sealedBlock.ID().String(), actualBlocks[0].Header.Id) + assert.Equal(suite.T(), suite.sealedBlock.ID().String(), actualBlocks[0].Header.Id) }) suite.Run("GetBlockByID with a non-existing block ID", func() { @@ -285,7 +381,6 @@ func (suite *RestAPITestSuite) TestGetBlock() { defer cancel() // replace one ID with a block ID for which the storage returns a not found error - rand.Seed(time.Now().Unix()) invalidBlockIndex := rand.Intn(len(testBlocks)) invalidID := unittest.IdentifierFixture() suite.blocks.On("ByID", invalidID).Return(nil, storage.ErrNotFound).Once() @@ -317,7 +412,7 @@ func (suite *RestAPITestSuite) TestRequestSizeRestriction() { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() // make a request of size larger than the max permitted size - requestBytes := make([]byte, rest.MaxRequestSize+1) + requestBytes := make([]byte, routes.MaxRequestSize+1) script := restclient.ScriptsBody{ Script: string(requestBytes), } @@ -325,13 +420,6 @@ func (suite *RestAPITestSuite) TestRequestSizeRestriction() { assertError(suite.T(), resp, err, http.StatusBadRequest, "request body too large") } -func (suite *RestAPITestSuite) TearDownTest() { - // close the server - if suite.rpcEng != nil { - unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) - } -} - // restAPIClient creates a REST API client func (suite *RestAPITestSuite) restAPIClient() *restclient.APIClient { config := restclient.NewConfiguration() @@ -351,13 +439,13 @@ func assertError(t *testing.T, resp *http.Response, err error, expectedCode int, func optionsForBlockByID() *restclient.BlocksApiBlocksIdGetOpts { return &restclient.BlocksApiBlocksIdGetOpts{ - Expand: optional.NewInterface([]string{rest.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{routes.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id"}), } } func optionsForBlockByStartEndHeight(startHeight, endHeight uint64) *restclient.BlocksApiBlocksGetOpts { return &restclient.BlocksApiBlocksGetOpts{ - Expand: optional.NewInterface([]string{rest.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{routes.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id", "header.height"}), StartHeight: optional.NewInterface(startHeight), EndHeight: optional.NewInterface(endHeight), @@ -366,7 +454,7 @@ func optionsForBlockByStartEndHeight(startHeight, endHeight uint64) *restclient. func optionsForBlockByHeights(heights []uint64) *restclient.BlocksApiBlocksGetOpts { return &restclient.BlocksApiBlocksGetOpts{ - Expand: optional.NewInterface([]string{rest.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{routes.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id", "header.height"}), Height: optional.NewInterface(heights), } @@ -374,7 +462,7 @@ func optionsForBlockByHeights(heights []uint64) *restclient.BlocksApiBlocksGetOp func optionsForFinalizedBlock(finalOrSealed string) *restclient.BlocksApiBlocksGetOpts { return &restclient.BlocksApiBlocksGetOpts{ - Expand: optional.NewInterface([]string{rest.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{routes.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id", "header.height"}), Height: optional.NewInterface(finalOrSealed), } diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 23c1df6420d..9b10cc5b539 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -3,13 +3,18 @@ package backend import ( "context" "fmt" + "net" + "strconv" "time" lru "github.com/hashicorp/golang-lru" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/cmd/build" + "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" @@ -17,10 +22,9 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" -) -// maxExecutionNodesCnt is the max number of execution nodes that will be contacted to complete an execution api request -const maxExecutionNodesCnt = 3 + accessproto "github.com/onflow/flow/protobuf/go/flow/access" +) // minExecutionNodesCnt is the minimum number of execution nodes expected to have sent the execution receipt for a block const minExecutionNodesCnt = 2 @@ -71,7 +75,19 @@ type Backend struct { chainID flow.ChainID collections storage.Collections executionReceipts storage.ExecutionReceipts - connFactory ConnectionFactory + connFactory connection.ConnectionFactory +} + +// Config defines the configurable options for creating Backend +type Config struct { + ExecutionClientTimeout time.Duration // execution API GRPC client timeout + CollectionClientTimeout time.Duration // collection API GRPC client timeout + ConnectionPoolSize uint // size of the cache for storing collection and execution connections + MaxHeightRange uint // max size of height range requests + PreferredExecutionNodeIDs []string // preferred list of upstream execution node IDs + FixedExecutionNodeIDs []string // fixed list of execution node IDs to choose from if no node ID can be chosen from the PreferredExecutionNodeIDs + ArchiveAddressList []string // the archive node address list to send script executions. when configured, script executions will be all sent to the archive node + CircuitBreakerConfig connection.CircuitBreakerConfig // the configuration for circuit breaker } func New( @@ -85,14 +101,16 @@ func New( executionReceipts storage.ExecutionReceipts, executionResults storage.ExecutionResults, chainID flow.ChainID, - transactionMetrics module.TransactionMetrics, - connFactory ConnectionFactory, + accessMetrics module.AccessMetrics, + connFactory connection.ConnectionFactory, retryEnabled bool, maxHeightRange uint, preferredExecutionNodeIDs []string, fixedExecutionNodeIDs []string, log zerolog.Logger, snapshotHistoryLimit int, + archiveAddressList []string, + circuitBreakerEnabled bool, ) *Backend { retry := newRetry() if retryEnabled { @@ -104,17 +122,32 @@ func New( log.Fatal().Err(err).Msg("failed to initialize script logging cache") } + archivePorts := make([]uint, len(archiveAddressList)) + for idx, addr := range archiveAddressList { + port, err := findPortFromAddress(addr) + if err != nil { + log.Fatal().Err(err).Msg("failed to find archive node port") + } + archivePorts[idx] = port + } + + // create node communicator, that will be used in sub-backend logic for interacting with API calls + nodeCommunicator := NewNodeCommunicator(circuitBreakerEnabled) + b := &Backend{ state: state, // create the sub-backends backendScripts: backendScripts{ - headers: headers, - executionReceipts: executionReceipts, - connFactory: connFactory, - state: state, - log: log, - metrics: transactionMetrics, - loggedScripts: loggedScripts, + headers: headers, + executionReceipts: executionReceipts, + connFactory: connFactory, + state: state, + log: log, + metrics: accessMetrics, + loggedScripts: loggedScripts, + archiveAddressList: archiveAddressList, + archivePorts: archivePorts, + nodeCommunicator: nodeCommunicator, }, backendTransactions: backendTransactions{ staticCollectionRPC: collectionRPC, @@ -125,11 +158,12 @@ func New( transactions: transactions, executionReceipts: executionReceipts, transactionValidator: configureTransactionValidator(state, chainID), - transactionMetrics: transactionMetrics, + transactionMetrics: accessMetrics, retry: retry, connFactory: connFactory, previousAccessNodes: historicalAccessNodes, log: log, + nodeCommunicator: nodeCommunicator, }, backendEvents: backendEvents{ state: state, @@ -138,6 +172,7 @@ func New( connFactory: connFactory, log: log, maxHeightRange: maxHeightRange, + nodeCommunicator: nodeCommunicator, }, backendBlockHeaders: backendBlockHeaders{ headers: headers, @@ -153,6 +188,7 @@ func New( executionReceipts: executionReceipts, connFactory: connFactory, log: log, + nodeCommunicator: nodeCommunicator, }, backendExecutionResults: backendExecutionResults{ executionResults: executionResults, @@ -183,6 +219,33 @@ func New( return b } +// NewCache constructs cache for storing connections to other nodes. +// No errors are expected during normal operations. +func NewCache( + log zerolog.Logger, + accessMetrics module.AccessMetrics, + connectionPoolSize uint, +) (*lru.Cache, uint, error) { + + var cache *lru.Cache + cacheSize := connectionPoolSize + if cacheSize > 0 { + var err error + cache, err = lru.NewWithEvict(int(cacheSize), func(_, evictedValue interface{}) { + store := evictedValue.(*connection.CachedClient) + store.Close() + log.Debug().Str("grpc_conn_evicted", store.Address).Msg("closing grpc connection evicted from pool") + if accessMetrics != nil { + accessMetrics.ConnectionFromPoolEvicted() + } + }) + if err != nil { + return nil, 0, fmt.Errorf("could not initialize connection pool cache: %w", err) + } + } + return cache, cacheSize, nil +} + func identifierList(ids []string) (flow.IdentifierList, error) { idList := make(flow.IdentifierList, len(ids)) for i, idStr := range ids { @@ -226,6 +289,27 @@ func (b *Backend) Ping(ctx context.Context) error { return nil } +// GetNodeVersionInfo returns node version information such as semver, commit, sporkID, protocolVersion, etc +func (b *Backend) GetNodeVersionInfo(ctx context.Context) (*access.NodeVersionInfo, error) { + stateParams := b.state.Params() + sporkId, err := stateParams.SporkID() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to read spork ID: %v", err) + } + + protocolVersion, err := stateParams.ProtocolVersion() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to read protocol version: %v", err) + } + + return &access.NodeVersionInfo{ + Semver: build.Version(), + Commit: build.Commit(), + SporkId: sporkId, + ProtocolVersion: uint64(protocolVersion), + }, nil +} + func (b *Backend) GetCollectionByID(_ context.Context, colID flow.Identifier) (*flow.LightCollection, error) { // retrieve the collection from the collection storage col, err := b.collections.LightByID(colID) @@ -259,7 +343,7 @@ func (b *Backend) GetLatestProtocolStateSnapshot(_ context.Context) ([]byte, err return convert.SnapshotToBytes(validSnapshot) } -// executionNodesForBlockID returns upto maxExecutionNodesCnt number of randomly chosen execution node identities +// executionNodesForBlockID returns upto maxNodesCnt number of randomly chosen execution node identities // which have executed the given block ID. // If no such execution node is found, an InsufficientExecutionReceipts error is returned. func executionNodesForBlockID( @@ -273,7 +357,7 @@ func executionNodesForBlockID( // check if the block ID is of the root block. If it is then don't look for execution receipts since they // will not be present for the root block. - rootBlock, err := state.Params().Root() + rootBlock, err := state.Params().FinalizedRoot() if err != nil { return nil, fmt.Errorf("failed to retreive execution IDs for block ID %v: %w", blockID, err) } @@ -330,14 +414,11 @@ func executionNodesForBlockID( return nil, fmt.Errorf("failed to retreive execution IDs for block ID %v: %w", blockID, err) } - // randomly choose upto maxExecutionNodesCnt identities - executionIdentitiesRandom := subsetENs.Sample(maxExecutionNodesCnt) - - if len(executionIdentitiesRandom) == 0 { + if len(subsetENs) == 0 { return nil, fmt.Errorf("no matching execution node found for block ID %v", blockID) } - return executionIdentitiesRandom, nil + return subsetENs, nil } // findAllExecutionNodes find all the execution nodes ids from the execution receipts that have been received for the @@ -435,3 +516,22 @@ func chooseExecutionNodes(state protocol.State, executorIDs flow.IdentifierList) // If no preferred or fixed ENs have been specified, then return all executor IDs i.e. no preference at all return allENs.Filter(filter.HasNodeID(executorIDs...)), nil } + +// Find ports from supplied Address +func findPortFromAddress(address string) (uint, error) { + _, portStr, err := net.SplitHostPort(address) + if err != nil { + return 0, fmt.Errorf("fail to extract port from address %v: %w", address, err) + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return 0, fmt.Errorf("fail to convert port string %v to port from address %v", portStr, address) + } + + if port < 0 { + return 0, fmt.Errorf("invalid port: %v in address %v", port, address) + } + + return uint(port), nil +} diff --git a/engine/access/rpc/backend/backend_accounts.go b/engine/access/rpc/backend/backend_accounts.go index a3a41053c61..35f8f0bf4df 100644 --- a/engine/access/rpc/backend/backend_accounts.go +++ b/engine/access/rpc/backend/backend_accounts.go @@ -4,12 +4,13 @@ import ( "context" "time" - "github.com/hashicorp/go-multierror" - execproto "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/rs/zerolog" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" + + "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" @@ -21,8 +22,9 @@ type backendAccounts struct { state protocol.State headers storage.Headers executionReceipts storage.ExecutionReceipts - connFactory ConnectionFactory + connFactory connection.ConnectionFactory log zerolog.Logger + nodeCommunicator *NodeCommunicator } func (b *backendAccounts) GetAccount(ctx context.Context, address flow.Address) (*flow.Account, error) { @@ -107,34 +109,39 @@ func (b *backendAccounts) getAccountAtBlockID( // other ENs are logged and swallowed. If all ENs fail to return a valid response, then an // error aggregating all failures is returned. func (b *backendAccounts) getAccountFromAnyExeNode(ctx context.Context, execNodes flow.IdentityList, req *execproto.GetAccountAtBlockIDRequest) (*execproto.GetAccountAtBlockIDResponse, error) { - var errors *multierror.Error - for _, execNode := range execNodes { - // TODO: use the GRPC Client interceptor - start := time.Now() - - resp, err := b.tryGetAccount(ctx, execNode, req) - duration := time.Since(start) - if err == nil { - // return if any execution node replied successfully - b.log.Debug(). - Str("execution_node", execNode.String()). + var resp *execproto.GetAccountAtBlockIDResponse + errToReturn := b.nodeCommunicator.CallAvailableNode( + execNodes, + func(node *flow.Identity) error { + var err error + // TODO: use the GRPC Client interceptor + start := time.Now() + + resp, err = b.tryGetAccount(ctx, node, req) + duration := time.Since(start) + if err == nil { + // return if any execution node replied successfully + b.log.Debug(). + Str("execution_node", node.String()). + Hex("block_id", req.GetBlockId()). + Hex("address", req.GetAddress()). + Int64("rtt_ms", duration.Milliseconds()). + Msg("Successfully got account info") + return nil + } + b.log.Error(). + Str("execution_node", node.String()). Hex("block_id", req.GetBlockId()). Hex("address", req.GetAddress()). Int64("rtt_ms", duration.Milliseconds()). - Msg("Successfully got account info") - return resp, nil - } - b.log.Error(). - Str("execution_node", execNode.String()). - Hex("block_id", req.GetBlockId()). - Hex("address", req.GetAddress()). - Int64("rtt_ms", duration.Milliseconds()). - Err(err). - Msg("failed to execute GetAccount") - errors = multierror.Append(errors, err) - } - - return nil, errors.ErrorOrNil() + Err(err). + Msg("failed to execute GetAccount") + return err + }, + nil, + ) + + return resp, errToReturn } func (b *backendAccounts) tryGetAccount(ctx context.Context, execNode *flow.Identity, req *execproto.GetAccountAtBlockIDRequest) (*execproto.GetAccountAtBlockIDResponse, error) { @@ -146,9 +153,6 @@ func (b *backendAccounts) tryGetAccount(ctx context.Context, execNode *flow.Iden resp, err := execRPCClient.GetAccountAtBlockID(ctx, req) if err != nil { - if status.Code(err) == codes.Unavailable { - b.connFactory.InvalidateExecutionAPIClient(execNode.Address) - } return nil, err } return resp, nil diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index e097843b933..d0b52820ee4 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -7,12 +7,12 @@ import ( "fmt" "time" - "github.com/hashicorp/go-multierror" execproto "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/rs/zerolog" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" @@ -24,9 +24,10 @@ type backendEvents struct { headers storage.Headers executionReceipts storage.ExecutionReceipts state protocol.State - connFactory ConnectionFactory + connFactory connection.ConnectionFactory log zerolog.Logger maxHeightRange uint + nodeCommunicator *NodeCommunicator } // GetEventsForHeightRange retrieves events for all sealed blocks between the start block height and @@ -148,7 +149,7 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( Msg("successfully got events") // convert execution node api result to access node api result - results, err := verifyAndConvertToAccessEvents(resp.GetResults(), blockHeaders) + results, err := verifyAndConvertToAccessEvents(resp.GetResults(), blockHeaders, resp.GetEventEncodingVersion()) if err != nil { return nil, status.Errorf(codes.Internal, "failed to verify retrieved events from execution node: %v", err) } @@ -158,7 +159,11 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( // verifyAndConvertToAccessEvents converts execution node api result to access node api result, and verifies that the results contains // results from each block that was requested -func verifyAndConvertToAccessEvents(execEvents []*execproto.GetEventsForBlockIDsResponse_Result, requestedBlockHeaders []*flow.Header) ([]flow.BlockEvents, error) { +func verifyAndConvertToAccessEvents( + execEvents []*execproto.GetEventsForBlockIDsResponse_Result, + requestedBlockHeaders []*flow.Header, + version execproto.EventEncodingVersion, +) ([]flow.BlockEvents, error) { if len(execEvents) != len(requestedBlockHeaders) { return nil, errors.New("number of results does not match number of blocks requested") } @@ -181,11 +186,17 @@ func verifyAndConvertToAccessEvents(execEvents []*execproto.GetEventsForBlockIDs result.GetBlockId()) } + events, err := convert.MessagesToEventsFromVersion(result.GetEvents(), version) + if err != nil { + return nil, fmt.Errorf("failed to unmarshall events in event %d with encoding version %s: %w", + i, version.String(), err) + } + results[i] = flow.BlockEvents{ BlockID: header.ID(), BlockHeight: header.Height, BlockTimestamp: header.Timestamp, - Events: convert.MessagesToEvents(result.GetEvents()), + Events: events, } } @@ -199,31 +210,37 @@ func verifyAndConvertToAccessEvents(execEvents []*execproto.GetEventsForBlockIDs func (b *backendEvents) getEventsFromAnyExeNode(ctx context.Context, execNodes flow.IdentityList, req *execproto.GetEventsForBlockIDsRequest) (*execproto.GetEventsForBlockIDsResponse, *flow.Identity, error) { - var errors *multierror.Error - // try to get events from one of the execution nodes - for _, execNode := range execNodes { - start := time.Now() - resp, err := b.tryGetEvents(ctx, execNode, req) - duration := time.Since(start) - - logger := b.log.With(). - Str("execution_node", execNode.String()). - Str("event", req.GetType()). - Int("blocks", len(req.BlockIds)). - Int64("rtt_ms", duration.Milliseconds()). - Logger() - - if err == nil { - // return if any execution node replied successfully - logger.Debug().Msg("Successfully got events") - return resp, execNode, nil - } - - logger.Err(err).Msg("failed to execute GetEvents") - - errors = multierror.Append(errors, err) - } - return nil, nil, errors.ErrorOrNil() + var resp *execproto.GetEventsForBlockIDsResponse + var execNode *flow.Identity + errToReturn := b.nodeCommunicator.CallAvailableNode( + execNodes, + func(node *flow.Identity) error { + var err error + start := time.Now() + resp, err = b.tryGetEvents(ctx, node, req) + duration := time.Since(start) + + logger := b.log.With(). + Str("execution_node", node.String()). + Str("event", req.GetType()). + Int("blocks", len(req.BlockIds)). + Int64("rtt_ms", duration.Milliseconds()). + Logger() + + if err == nil { + // return if any execution node replied successfully + logger.Debug().Msg("Successfully got events") + execNode = node + return nil + } + + logger.Err(err).Msg("failed to execute GetEvents") + return err + }, + nil, + ) + + return resp, execNode, errToReturn } func (b *backendEvents) tryGetEvents(ctx context.Context, @@ -237,9 +254,6 @@ func (b *backendEvents) tryGetEvents(ctx context.Context, resp, err := execRPCClient.GetEventsForBlockIDs(ctx, req) if err != nil { - if status.Code(err) == codes.Unavailable { - b.connFactory.InvalidateExecutionAPIClient(execNode.Address) - } return nil, err } return resp, nil diff --git a/engine/access/rpc/backend/backend_network.go b/engine/access/rpc/backend/backend_network.go index 099cad9af90..577ebaa8a84 100644 --- a/engine/access/rpc/backend/backend_network.go +++ b/engine/access/rpc/backend/backend_network.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "github.com/onflow/flow-go/cmd/build" + "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -42,6 +44,26 @@ func (b *backendNetwork) GetNetworkParameters(_ context.Context) access.NetworkP } } +func (b *backendNetwork) GetNodeVersionInfo(ctx context.Context) (*access.NodeVersionInfo, error) { + stateParams := b.state.Params() + sporkId, err := stateParams.SporkID() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to read spork ID: %v", err) + } + + protocolVersion, err := stateParams.ProtocolVersion() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to read protocol version: %v", err) + } + + return &access.NodeVersionInfo{ + Semver: build.Version(), + Commit: build.Commit(), + SporkId: sporkId, + ProtocolVersion: uint64(protocolVersion), + }, nil +} + // GetLatestProtocolStateSnapshot returns the latest finalized snapshot func (b *backendNetwork) GetLatestProtocolStateSnapshot(_ context.Context) ([]byte, error) { snapshot := b.state.Final() diff --git a/engine/access/rpc/backend/backend_scripts.go b/engine/access/rpc/backend/backend_scripts.go index a8613dcd68b..62d32c56211 100644 --- a/engine/access/rpc/backend/backend_scripts.go +++ b/engine/access/rpc/backend/backend_scripts.go @@ -3,16 +3,18 @@ package backend import ( "context" "crypto/md5" //nolint:gosec + "io" "time" lru "github.com/hashicorp/golang-lru" + "github.com/onflow/flow/protobuf/go/flow/access" - "github.com/hashicorp/go-multierror" execproto "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/rs/zerolog" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -24,13 +26,16 @@ import ( const uniqueScriptLoggingTimeWindow = 10 * time.Minute type backendScripts struct { - headers storage.Headers - executionReceipts storage.ExecutionReceipts - state protocol.State - connFactory ConnectionFactory - log zerolog.Logger - metrics module.BackendScriptsMetrics - loggedScripts *lru.Cache + headers storage.Headers + executionReceipts storage.ExecutionReceipts + state protocol.State + connFactory connection.ConnectionFactory + log zerolog.Logger + metrics module.BackendScriptsMetrics + loggedScripts *lru.Cache + archiveAddressList []string + archivePorts []uint + nodeCommunicator *NodeCommunicator } func (b *backendScripts) ExecuteScriptAtLatestBlock( @@ -49,7 +54,7 @@ func (b *backendScripts) ExecuteScriptAtLatestBlock( latestBlockID := latestHeader.ID() // execute script on the execution node at that block id - return b.executeScriptOnExecutionNode(ctx, latestBlockID, script, arguments) + return b.executeScriptOnExecutor(ctx, latestBlockID, script, arguments) } func (b *backendScripts) ExecuteScriptAtBlockID( @@ -59,7 +64,7 @@ func (b *backendScripts) ExecuteScriptAtBlockID( arguments [][]byte, ) ([]byte, error) { // execute script on the execution node at that block id - return b.executeScriptOnExecutionNode(ctx, blockID, script, arguments) + return b.executeScriptOnExecutor(ctx, blockID, script, arguments) } func (b *backendScripts) ExecuteScriptAtBlockHeight( @@ -78,81 +83,121 @@ func (b *backendScripts) ExecuteScriptAtBlockHeight( blockID := header.ID() // execute script on the execution node at that block id - return b.executeScriptOnExecutionNode(ctx, blockID, script, arguments) + return b.executeScriptOnExecutor(ctx, blockID, script, arguments) } // executeScriptOnExecutionNode forwards the request to the execution node using the execution node // grpc client and converts the response back to the access node api response format -func (b *backendScripts) executeScriptOnExecutionNode( +func (b *backendScripts) executeScriptOnExecutor( ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte, ) ([]byte, error) { - - execReq := &execproto.ExecuteScriptAtBlockIDRequest{ - BlockId: blockID[:], - Script: script, - Arguments: arguments, - } - // find few execution nodes which have executed the block earlier and provided an execution receipt for it - execNodes, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) + executors, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to find execution nodes at blockId %v: %v", blockID.String(), err) + return nil, status.Errorf(codes.Internal, "failed to find script executors at blockId %v: %v", blockID.String(), err) } // encode to MD5 as low compute/memory lookup key // CAUTION: cryptographically insecure md5 is used here, but only to de-duplicate logs. // *DO NOT* use this hash for any protocol-related or cryptographic functions. insecureScriptHash := md5.Sum(script) //nolint:gosec - // try each of the execution nodes found - var errors *multierror.Error - // try to execute the script on one of the execution nodes - for _, execNode := range execNodes { - execStartTime := time.Now() // record start time - result, err := b.tryExecuteScript(ctx, execNode, execReq) - if err == nil { - if b.log.GetLevel() == zerolog.DebugLevel { - executionTime := time.Now() - if b.shouldLogScript(executionTime, insecureScriptHash) { - b.log.Debug(). - Str("execution_node", execNode.String()). + // try execution on Archive nodes + if len(b.archiveAddressList) > 0 { + startTime := time.Now() + for idx, rnAddr := range b.archiveAddressList { + rnPort := b.archivePorts[idx] + result, err := b.tryExecuteScriptOnArchiveNode(ctx, rnAddr, rnPort, blockID, script, arguments) + if err == nil { + // log execution time + b.metrics.ScriptExecuted( + time.Since(startTime), + len(script), + ) + return result, nil + } else { + errCode := status.Code(err) + switch errCode { + case codes.InvalidArgument: + // failure due to cadence script, no need to query further + b.log.Debug().Err(err). + Str("script_executor_addr", rnAddr). Hex("block_id", blockID[:]). Hex("script_hash", insecureScriptHash[:]). Str("script", string(script)). - Msg("Successfully executed script") - b.loggedScripts.Add(insecureScriptHash, executionTime) + Msg("script failed to execute on the execution node") + return nil, err + case codes.NotFound: + // failures due to unavailable blocks are explicitly marked Not found + b.metrics.ScriptExecutionErrorOnArchiveNode() + b.log.Error().Err(err).Msg("script execution failed for archive node") + default: + continue } } + } + } + + // try to execute the script on one of the execution nodes found + var result []byte + hasInvalidArgument := false + errToReturn := b.nodeCommunicator.CallAvailableNode( + executors, + func(node *flow.Identity) error { + execStartTime := time.Now() + result, err = b.tryExecuteScriptOnExecutionNode(ctx, node.Address, blockID, script, arguments) + if err == nil { + if b.log.GetLevel() == zerolog.DebugLevel { + executionTime := time.Now() + if b.shouldLogScript(executionTime, insecureScriptHash) { + b.log.Debug(). + Str("script_executor_addr", node.Address). + Hex("block_id", blockID[:]). + Hex("script_hash", insecureScriptHash[:]). + Str("script", string(script)). + Msg("Successfully executed script") + b.loggedScripts.Add(insecureScriptHash, executionTime) + } + } - // log execution time - b.metrics.ScriptExecuted( - time.Since(execStartTime), - len(script), - ) + // log execution time + b.metrics.ScriptExecuted( + time.Since(execStartTime), + len(script), + ) - return result, nil - } - // return if it's just a script failure as opposed to an EN failure and skip trying other ENs - if status.Code(err) == codes.InvalidArgument { - b.log.Debug().Err(err). - Str("execution_node", execNode.String()). - Hex("block_id", blockID[:]). - Hex("script_hash", insecureScriptHash[:]). - Str("script", string(script)). - Msg("script failed to execute on the execution node") - return nil, err - } - errors = multierror.Append(errors, err) + return nil + } + + return err + }, + func(node *flow.Identity, err error) bool { + hasInvalidArgument = status.Code(err) == codes.InvalidArgument + if hasInvalidArgument { + b.log.Debug().Err(err). + Str("script_executor_addr", node.Address). + Hex("block_id", blockID[:]). + Hex("script_hash", insecureScriptHash[:]). + Str("script", string(script)). + Msg("script failed to execute on the execution node") + } + return hasInvalidArgument + }, + ) + + if hasInvalidArgument { + return nil, errToReturn } - errToReturn := errors.ErrorOrNil() - if errToReturn != nil { + if errToReturn == nil { + return result, nil + } else { + b.metrics.ScriptExecutionErrorOnExecutionNode() b.log.Error().Err(err).Msg("script execution failed for execution node internal reasons") + return nil, rpc.ConvertError(errToReturn, "failed to execute script on execution nodes", codes.Internal) } - - return nil, rpc.ConvertMultiError(errors, "failed to execute script on execution nodes", codes.Internal) } // shouldLogScript checks if the script hash is unique in the time window @@ -167,19 +212,69 @@ func (b *backendScripts) shouldLogScript(execTime time.Time, scriptHash [16]byte } } -func (b *backendScripts) tryExecuteScript(ctx context.Context, execNode *flow.Identity, req *execproto.ExecuteScriptAtBlockIDRequest) ([]byte, error) { - execRPCClient, closer, err := b.connFactory.GetExecutionAPIClient(execNode.Address) +func (b *backendScripts) tryExecuteScriptOnExecutionNode( + ctx context.Context, + executorAddress string, + blockID flow.Identifier, + script []byte, + arguments [][]byte, +) ([]byte, error) { + req := &execproto.ExecuteScriptAtBlockIDRequest{ + BlockId: blockID[:], + Script: script, + Arguments: arguments, + } + execRPCClient, closer, err := b.connFactory.GetExecutionAPIClient(executorAddress) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to create client for execution node %s: %v", execNode.String(), err) + return nil, status.Errorf(codes.Internal, "failed to create client for execution node %s: %v", + executorAddress, err) } - defer closer.Close() + defer func(closer io.Closer) { + err := closer.Close() + if err != nil { + b.log.Error().Err(err).Msg("failed to close execution client") + } + }(closer) execResp, err := execRPCClient.ExecuteScriptAtBlockID(ctx, req) + if err != nil { + return nil, status.Errorf(status.Code(err), "failed to execute the script on the execution node %s: %v", executorAddress, err) + } + return execResp.GetValue(), nil +} + +func (b *backendScripts) tryExecuteScriptOnArchiveNode( + ctx context.Context, + executorAddress string, + port uint, + blockID flow.Identifier, + script []byte, + arguments [][]byte, +) ([]byte, error) { + req := &access.ExecuteScriptAtBlockIDRequest{ + BlockId: blockID[:], + Script: script, + Arguments: arguments, + } + + archiveClient, closer, err := b.connFactory.GetAccessAPIClientWithPort(executorAddress, port) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to create client for archive node %s: %v", + executorAddress, err) + } + defer func(closer io.Closer) { + err := closer.Close() + if err != nil { + b.log.Error().Err(err).Msg("failed to close archive client") + } + }(closer) + resp, err := archiveClient.ExecuteScriptAtBlockID(ctx, req) if err != nil { if status.Code(err) == codes.Unavailable { - b.connFactory.InvalidateExecutionAPIClient(execNode.Address) + b.connFactory.InvalidateAccessAPIClient(executorAddress) } - return nil, status.Errorf(status.Code(err), "failed to execute the script on the execution node %s: %v", execNode.String(), err) + return nil, status.Errorf(status.Code(err), "failed to execute the script on archive node %s: %v", + executorAddress, err) } - return execResp.GetValue(), nil + return resp.GetValue(), nil } diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index cc52ef54c6d..d40ff45890e 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -3,9 +3,10 @@ package backend import ( "context" "fmt" - "math/rand" + "strconv" "testing" - "time" + + "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/dgraph-io/badger/v2" accessproto "github.com/onflow/flow/protobuf/go/flow/access" @@ -48,6 +49,7 @@ type Suite struct { colClient *access.AccessAPIClient execClient *access.ExecutionAPIClient historicalAccessClient *access.AccessAPIClient + archiveClient *access.AccessAPIClient connectionFactory *backendmock.ConnectionFactory chainID flow.ChainID } @@ -57,13 +59,12 @@ func TestHandler(t *testing.T) { } func (suite *Suite) SetupTest() { - rand.Seed(time.Now().UnixNano()) suite.log = zerolog.New(zerolog.NewConsoleWriter()) suite.state = new(protocol.State) suite.snapshot = new(protocol.Snapshot) header := unittest.BlockHeaderFixture() params := new(protocol.Params) - params.On("Root").Return(header, nil) + params.On("FinalizedRoot").Return(header, nil) params.On("SporkRootBlockHeight").Return(header.Height, nil) suite.state.On("Params").Return(params).Maybe() suite.blocks = new(storagemock.Blocks) @@ -73,6 +74,7 @@ func (suite *Suite) SetupTest() { suite.receipts = new(storagemock.ExecutionReceipts) suite.results = new(storagemock.ExecutionResults) suite.colClient = new(access.AccessAPIClient) + suite.archiveClient = new(access.AccessAPIClient) suite.execClient = new(access.ExecutionAPIClient) suite.chainID = flow.Testnet suite.historicalAccessClient = new(access.AccessAPIClient) @@ -107,6 +109,8 @@ func (suite *Suite) TestPing() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) err := backend.Ping(context.Background()) @@ -141,6 +145,8 @@ func (suite *Suite) TestGetLatestFinalizedBlockHeader() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // query the handler for the latest finalized block @@ -205,6 +211,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_NoTransitionSpan() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // query the handler for the latest finalized snapshot @@ -276,6 +284,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_TransitionSpans() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // query the handler for the latest finalized snapshot @@ -340,6 +350,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_PhaseTransitionSpan() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // query the handler for the latest finalized snapshot @@ -415,6 +427,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_EpochTransitionSpan() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // query the handler for the latest finalized snapshot @@ -474,6 +488,8 @@ func (suite *Suite) TestGetLatestProtocolStateSnapshot_HistoryLimit() { nil, suite.log, snapshotHistoryLimit, + nil, + false, ) // the handler should return a snapshot history limit error @@ -511,6 +527,8 @@ func (suite *Suite) TestGetLatestSealedBlockHeader() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // query the handler for the latest sealed block @@ -556,6 +574,8 @@ func (suite *Suite) TestGetTransaction() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) actual, err := backend.GetTransaction(context.Background(), transaction.ID()) @@ -595,6 +615,8 @@ func (suite *Suite) TestGetCollection() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) actual, err := backend.GetCollectionByID(context.Background(), expected.ID()) @@ -657,6 +679,8 @@ func (suite *Suite) TestGetTransactionResultByIndex() { flow.IdentifierList(fixedENIDs.NodeIDs()).Strings(), suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) suite.execClient. On("GetTransactionResultByIndex", ctx, exeEventReq). @@ -719,6 +743,8 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { flow.IdentifierList(fixedENIDs.NodeIDs()).Strings(), suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) suite.execClient. On("GetTransactionResultsByBlockID", ctx, exeEventReq). @@ -743,12 +769,17 @@ func (suite *Suite) TestTransactionStatusTransition() { block.Header.Height = 2 headBlock := unittest.BlockFixture() headBlock.Header.Height = block.Header.Height - 1 // head is behind the current block + block.SetPayload( + unittest.PayloadFixture( + unittest.WithGuarantees( + unittest.CollectionGuaranteesWithCollectionIDFixture([]*flow.Collection{&collection})...))) suite.snapshot. On("Head"). Return(headBlock.Header, nil) light := collection.Light() + suite.collections.On("LightByID", light.ID()).Return(&light, nil) // transaction storage returns the corresponding transaction suite.transactions. @@ -804,6 +835,8 @@ func (suite *Suite) TestTransactionStatusTransition() { flow.IdentifierList(fixedENIDs.NodeIDs()).Strings(), suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // Successfully return empty event list @@ -813,7 +846,7 @@ func (suite *Suite) TestTransactionStatusTransition() { Once() // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) // status should be finalized since the sealed blocks is smaller in height @@ -828,7 +861,7 @@ func (suite *Suite) TestTransactionStatusTransition() { Return(exeEventResp, nil) // second call - when block under test's height is greater height than the sealed head - result, err = backend.GetTransactionResult(ctx, txID) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) // status should be executed since no `NotFound` error in the `GetTransactionResult` call @@ -838,7 +871,7 @@ func (suite *Suite) TestTransactionStatusTransition() { headBlock.Header.Height = block.Header.Height + 1 // third call - when block under test's height is less than sealed head's height - result, err = backend.GetTransactionResult(ctx, txID) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) // status should be sealed since the sealed blocks is greater in height @@ -849,7 +882,7 @@ func (suite *Suite) TestTransactionStatusTransition() { // fourth call - when block under test's height so much less than the head's height that it's considered expired, // but since there is a execution result, means it should retain it's sealed status - result, err = backend.GetTransactionResult(ctx, txID) + result, err = backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) // status should be expired since @@ -923,12 +956,14 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // should return pending status when we have not observed an expiry block suite.Run("pending", func() { // referenced block isn't known yet, so should return pending status - result, err := backend.GetTransactionResult(ctx, txID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) @@ -944,7 +979,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have NOT observed all intermediary collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry/2 - result, err := backend.GetTransactionResult(ctx, txID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) }) @@ -954,7 +989,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have observed all intermediary collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry + 1 - result, err := backend.GetTransactionResult(ctx, txID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) }) @@ -969,7 +1004,7 @@ func (suite *Suite) TestTransactionExpiredStatusTransition() { // we have observed all intermediary collections fullHeight = block.Header.Height + flow.DefaultTransactionExpiry + 1 - result, err := backend.GetTransactionResult(ctx, txID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusExpired, result.Status) }) @@ -985,7 +1020,12 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { transactionBody := collection.Transactions[0] // block which will eventually contain the transaction block := unittest.BlockFixture() + block.SetPayload( + unittest.PayloadFixture( + unittest.WithGuarantees( + unittest.CollectionGuaranteesWithCollectionIDFixture([]*flow.Collection{&collection})...))) blockID := block.ID() + // reference block to which the transaction points to refBlock := unittest.BlockFixture() refBlockID := refBlock.ID() @@ -1037,6 +1077,9 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { return nil }) + light := collection.Light() + suite.collections.On("LightByID", mock.Anything).Return(&light, nil) + // refBlock storage returns the corresponding refBlock suite.blocks. On("ByCollectionID", collection.ID()). @@ -1081,6 +1124,8 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { flow.IdentifierList(enIDs.NodeIDs()).Strings(), suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) preferredENIdentifiers = flow.IdentifierList{receipts[0].ExecutorID} @@ -1088,18 +1133,18 @@ func (suite *Suite) TestTransactionPendingToFinalizedStatusTransition() { // should return pending status when we have not observed collection for the transaction suite.Run("pending", func() { currentState = flow.TransactionStatusPending - result, err := backend.GetTransactionResult(ctx, txID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusPending, result.Status) // assert that no call to an execution node is made suite.execClient.AssertNotCalled(suite.T(), "GetTransactionResult", mock.Anything, mock.Anything) }) - // should return finalized status when we have have observed collection for the transaction (after observing the - // a preceding sealed refBlock) + // should return finalized status when we have observed collection for the transaction (after observing the + // preceding sealed refBlock) suite.Run("finalized", func() { currentState = flow.TransactionStatusFinalized - result, err := backend.GetTransactionResult(ctx, txID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) suite.Assert().Equal(flow.TransactionStatusFinalized, result.Status) }) @@ -1138,10 +1183,12 @@ func (suite *Suite) TestTransactionResultUnknown() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) // status should be reported as unknown @@ -1191,6 +1238,8 @@ func (suite *Suite) TestGetLatestFinalizedBlock() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // query the handler for the latest finalized header @@ -1320,6 +1369,8 @@ func (suite *Suite) TestGetEventsForBlockIDs() { validENIDs.Strings(), // set the fixed EN Identifiers to the generated execution IDs suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // execute request @@ -1351,6 +1402,8 @@ func (suite *Suite) TestGetEventsForBlockIDs() { validENIDs.Strings(), // set the fixed EN Identifiers to the generated execution IDs suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // execute request with an empty block id list and expect an empty list of events and no error @@ -1409,6 +1462,8 @@ func (suite *Suite) TestGetExecutionResultByID() { validENIDs.Strings(), // set the fixed EN Identifiers to the generated execution IDs suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // execute request @@ -1438,6 +1493,8 @@ func (suite *Suite) TestGetExecutionResultByID() { validENIDs.Strings(), // set the fixed EN Identifiers to the generated execution IDs suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // execute request @@ -1500,6 +1557,8 @@ func (suite *Suite) TestGetExecutionResultByBlockID() { validENIDs.Strings(), // set the fixed EN Identifiers to the generated execution IDs suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // execute request @@ -1530,6 +1589,8 @@ func (suite *Suite) TestGetExecutionResultByBlockID() { validENIDs.Strings(), // set the fixed EN Identifiers to the generated execution IDs suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // execute request @@ -1562,7 +1623,7 @@ func (suite *Suite) TestGetEventsForHeightRange() { rootHeader := unittest.BlockHeaderFixture() params := new(protocol.Params) - params.On("Root").Return(rootHeader, nil) + params.On("FinalizedRoot").Return(rootHeader, nil) state.On("Params").Return(params).Maybe() // mock snapshot to return head backend @@ -1679,6 +1740,8 @@ func (suite *Suite) TestGetEventsForHeightRange() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) _, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), maxHeight, minHeight) @@ -1717,6 +1780,8 @@ func (suite *Suite) TestGetEventsForHeightRange() { fixedENIdentifiersStr, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // execute request @@ -1754,6 +1819,8 @@ func (suite *Suite) TestGetEventsForHeightRange() { fixedENIdentifiersStr, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) actualResp, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight) @@ -1790,6 +1857,8 @@ func (suite *Suite) TestGetEventsForHeightRange() { fixedENIdentifiersStr, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) _, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, minHeight+1) @@ -1826,6 +1895,8 @@ func (suite *Suite) TestGetEventsForHeightRange() { fixedENIdentifiersStr, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) _, err := backend.GetEventsForHeightRange(ctx, string(flow.EventAccountCreated), minHeight, maxHeight) @@ -1902,6 +1973,8 @@ func (suite *Suite) TestGetAccount() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) preferredENIdentifiers = flow.IdentifierList{receipts[0].ExecutorID} @@ -1982,6 +2055,8 @@ func (suite *Suite) TestGetAccountAtBlockHeight() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) preferredENIdentifiers = flow.IdentifierList{receipts[0].ExecutorID} @@ -2001,7 +2076,8 @@ func (suite *Suite) TestGetNetworkParameters() { expectedChainID := flow.Mainnet - backend := New(nil, + backend := New( + nil, nil, nil, nil, @@ -2019,6 +2095,8 @@ func (suite *Suite) TestGetNetworkParameters() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) params := backend.GetNetworkParameters(context.Background()) @@ -2087,12 +2165,24 @@ func (suite *Suite) TestExecutionNodesForBlockID() { if fixedENs != nil { fixedENIdentifiers = fixedENs.NodeIDs() } - actualList, err := executionNodesForBlockID(context.Background(), block.ID(), suite.receipts, suite.state, suite.log) - require.NoError(suite.T(), err) + if expectedENs == nil { expectedENs = flow.IdentityList{} } - if len(expectedENs) > maxExecutionNodesCnt { + + allExecNodes, err := executionNodesForBlockID(context.Background(), block.ID(), suite.receipts, suite.state, suite.log) + require.NoError(suite.T(), err) + + execNodeSelectorFactory := NodeSelectorFactory{circuitBreakerEnabled: false} + execSelector, err := execNodeSelectorFactory.SelectNodes(allExecNodes) + require.NoError(suite.T(), err) + + actualList := flow.IdentityList{} + for actual := execSelector.Next(); actual != nil; actual = execSelector.Next() { + actualList = append(actualList, actual) + } + + if len(expectedENs) > maxNodesCnt { for _, actual := range actualList { require.Contains(suite.T(), expectedENs, actual) } @@ -2107,9 +2197,20 @@ func (suite *Suite) TestExecutionNodesForBlockID() { attempt2Receipts = flow.ExecutionReceiptList{} attempt3Receipts = flow.ExecutionReceiptList{} suite.state.On("AtBlockID", mock.Anything).Return(suite.snapshot) - actualList, err := executionNodesForBlockID(context.Background(), block.ID(), suite.receipts, suite.state, suite.log) + + allExecNodes, err := executionNodesForBlockID(context.Background(), block.ID(), suite.receipts, suite.state, suite.log) + require.NoError(suite.T(), err) + + execNodeSelectorFactory := NodeSelectorFactory{circuitBreakerEnabled: false} + execSelector, err := execNodeSelectorFactory.SelectNodes(allExecNodes) require.NoError(suite.T(), err) - require.Equal(suite.T(), len(actualList), maxExecutionNodesCnt) + + actualList := flow.IdentityList{} + for actual := execSelector.Next(); actual != nil; actual = execSelector.Next() { + actualList = append(actualList, actual) + } + + require.Equal(suite.T(), len(actualList), maxNodesCnt) }) // if no preferred or fixed ENs are specified, the ExecutionNodesForBlockID function should @@ -2197,6 +2298,8 @@ func (suite *Suite) TestExecuteScriptOnExecutionNode() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // mock parameters @@ -2217,7 +2320,7 @@ func (suite *Suite) TestExecuteScriptOnExecutionNode() { suite.Run("happy path script execution success", func() { suite.execClient.On("ExecuteScriptAtBlockID", ctx, execReq).Return(execRes, nil).Once() - res, err := backend.tryExecuteScript(ctx, executionNode, execReq) + res, err := backend.tryExecuteScriptOnExecutionNode(ctx, executionNode.Address, blockID, script, arguments) suite.execClient.AssertExpectations(suite.T()) suite.checkResponse(res, err) }) @@ -2225,7 +2328,7 @@ func (suite *Suite) TestExecuteScriptOnExecutionNode() { suite.Run("script execution failure returns status OK", func() { suite.execClient.On("ExecuteScriptAtBlockID", ctx, execReq). Return(nil, status.Error(codes.InvalidArgument, "execution failure!")).Once() - _, err := backend.tryExecuteScript(ctx, executionNode, execReq) + _, err := backend.tryExecuteScriptOnExecutionNode(ctx, executionNode.Address, blockID, script, arguments) suite.execClient.AssertExpectations(suite.T()) suite.Require().Error(err) suite.Require().Equal(status.Code(err), codes.InvalidArgument) @@ -2234,13 +2337,95 @@ func (suite *Suite) TestExecuteScriptOnExecutionNode() { suite.Run("execution node internal failure returns status code Internal", func() { suite.execClient.On("ExecuteScriptAtBlockID", ctx, execReq). Return(nil, status.Error(codes.Internal, "execution node internal error!")).Once() - _, err := backend.tryExecuteScript(ctx, executionNode, execReq) + _, err := backend.tryExecuteScriptOnExecutionNode(ctx, executionNode.Address, blockID, script, arguments) suite.execClient.AssertExpectations(suite.T()) suite.Require().Error(err) suite.Require().Equal(status.Code(err), codes.Internal) }) } +// TestExecuteScriptOnArchiveNode tests the method backend.scripts.executeScriptOnArchiveNode for script execution +func (suite *Suite) TestExecuteScriptOnArchiveNode() { + + // create a mock connection factory + var mockPort uint = 9000 + connFactory := new(backendmock.ConnectionFactory) + connFactory.On("GetAccessAPIClientWithPort", mock.Anything, mockPort).Return(suite.archiveClient, &mockCloser{}, nil) + connFactory.On("InvalidateAccessAPIClient", mock.Anything) + archiveNode := unittest.IdentityFixture(unittest.WithRole(flow.RoleAccess)) + fullArchiveAddress := archiveNode.Address + ":" + strconv.FormatUint(uint64(mockPort), 10) + + // create the handler with the mock + backend := New( + suite.state, + nil, + nil, + nil, + suite.headers, + nil, + nil, + suite.receipts, + suite.results, + flow.Mainnet, + metrics.NewNoopCollector(), + connFactory, // the connection factory should be used to get the execution node client + false, + DefaultMaxHeightRange, + nil, + nil, + suite.log, + DefaultSnapshotHistoryLimit, + []string{fullArchiveAddress}, + false, + ) + + // mock parameters + ctx := context.Background() + block := unittest.BlockFixture() + blockID := block.ID() + script := []byte("dummy script") + arguments := [][]byte(nil) + archiveRes := &accessproto.ExecuteScriptResponse{Value: []byte{4, 5, 6}} + archiveReq := &accessproto.ExecuteScriptAtBlockIDRequest{ + BlockId: blockID[:], + Script: script, + Arguments: arguments} + + suite.Run("happy path script execution success", func() { + suite.archiveClient.On("ExecuteScriptAtBlockID", ctx, archiveReq).Return(archiveRes, nil).Once() + res, err := backend.tryExecuteScriptOnArchiveNode(ctx, archiveNode.Address, mockPort, blockID, script, arguments) + suite.archiveClient.AssertExpectations(suite.T()) + suite.checkResponse(res, err) + }) + + suite.Run("script execution failure returns status OK", func() { + suite.archiveClient.On("ExecuteScriptAtBlockID", ctx, archiveReq). + Return(nil, status.Error(codes.InvalidArgument, "execution failure!")).Once() + _, err := backend.tryExecuteScriptOnArchiveNode(ctx, archiveNode.Address, mockPort, blockID, script, arguments) + suite.archiveClient.AssertExpectations(suite.T()) + suite.Require().Error(err) + suite.Require().Equal(status.Code(err), codes.InvalidArgument) + }) + + suite.Run("script execution due to missing block returns Not found", func() { + suite.archiveClient.On("ExecuteScriptAtBlockID", ctx, archiveReq). + Return(nil, status.Error(codes.NotFound, "missing block!")).Once() + _, err := backend.tryExecuteScriptOnArchiveNode(ctx, archiveNode.Address, mockPort, blockID, script, arguments) + suite.archiveClient.AssertExpectations(suite.T()) + suite.Require().Error(err) + suite.Require().Equal(status.Code(err), codes.NotFound) + }) + + suite.Run("archive node internal failure returns status code Internal", func() { + suite.archiveClient.On("ExecuteScriptAtBlockID", ctx, archiveReq). + Return(nil, status.Error(codes.Internal, "archive node internal error!")).Once() + _, err := backend.tryExecuteScriptOnArchiveNode(ctx, archiveNode.Address, mockPort, blockID, script, arguments) + suite.archiveClient.AssertExpectations(suite.T()) + suite.Require().Error(err) + suite.Require().Equal(status.Code(err), codes.Internal) + }) +} + func (suite *Suite) assertAllExpectations() { suite.snapshot.AssertExpectations(suite.T()) suite.state.AssertExpectations(suite.T()) @@ -2272,7 +2457,7 @@ func (suite *Suite) setupReceipts(block *flow.Block) ([]*flow.ExecutionReceipt, return receipts, ids } -func (suite *Suite) setupConnectionFactory() ConnectionFactory { +func (suite *Suite) setupConnectionFactory() connection.ConnectionFactory { // create a mock connection factory connFactory := new(backendmock.ConnectionFactory) connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 731b042477e..79579d420e6 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -6,7 +6,6 @@ import ( "fmt" "time" - "github.com/hashicorp/go-multierror" accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" execproto "github.com/onflow/flow/protobuf/go/flow/execution" @@ -15,6 +14,7 @@ import ( "google.golang.org/grpc/status" "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/fvm/blueprints" @@ -24,8 +24,6 @@ import ( "github.com/onflow/flow-go/storage" ) -const collectionNodesToTry uint = 3 - type backendTransactions struct { staticCollectionRPC accessproto.AccessAPIClient // rpc client tied to a fixed collection node transactions storage.Transactions @@ -37,10 +35,11 @@ type backendTransactions struct { transactionMetrics module.TransactionMetrics transactionValidator *access.TransactionValidator retry *Retry - connFactory ConnectionFactory + connFactory connection.ConnectionFactory previousAccessNodes []accessproto.AccessAPIClient log zerolog.Logger + nodeCommunicator *NodeCommunicator } // SendTransaction forwards the transaction to the collection node @@ -85,36 +84,39 @@ func (b *backendTransactions) trySendTransaction(ctx context.Context, tx *flow.T return b.grpcTxSend(ctx, b.staticCollectionRPC, tx) } - // otherwise choose a random set of collections nodes to try - collAddrs, err := b.chooseCollectionNodes(tx, collectionNodesToTry) + // otherwise choose all collection nodes to try + collNodes, err := b.chooseCollectionNodes(tx) if err != nil { return fmt.Errorf("failed to determine collection node for tx %x: %w", tx, err) } - var sendErrors *multierror.Error + var sendError error logAnyError := func() { - err = sendErrors.ErrorOrNil() - if err != nil { + if sendError != nil { b.log.Info().Err(err).Msg("failed to send transactions to collector nodes") } } defer logAnyError() // try sending the transaction to one of the chosen collection nodes - for _, addr := range collAddrs { - err = b.sendTransactionToCollector(ctx, tx, addr) - if err == nil { + sendError = b.nodeCommunicator.CallAvailableNode( + collNodes, + func(node *flow.Identity) error { + err = b.sendTransactionToCollector(ctx, tx, node.Address) + if err != nil { + return err + } return nil - } - sendErrors = multierror.Append(sendErrors, err) - } + }, + nil, + ) - return sendErrors.ErrorOrNil() + return sendError } // chooseCollectionNodes finds a random subset of size sampleSize of collection node addresses from the // collection node cluster responsible for the given tx -func (b *backendTransactions) chooseCollectionNodes(tx *flow.TransactionBody, sampleSize uint) ([]string, error) { +func (b *backendTransactions) chooseCollectionNodes(tx *flow.TransactionBody) (flow.IdentityList, error) { // retrieve the set of collector clusters clusters, err := b.state.Final().Epochs().Current().Clustering() @@ -123,21 +125,12 @@ func (b *backendTransactions) chooseCollectionNodes(tx *flow.TransactionBody, sa } // get the cluster responsible for the transaction - txCluster, ok := clusters.ByTxID(tx.ID()) + targetNodes, ok := clusters.ByTxID(tx.ID()) if !ok { return nil, fmt.Errorf("could not get local cluster by txID: %x", tx.ID()) } - // select a random subset of collection nodes from the cluster to be tried in order - targetNodes := txCluster.Sample(sampleSize) - - // collect the addresses of all the chosen collection nodes - var targetAddrs = make([]string, len(targetNodes)) - for i, id := range targetNodes { - targetAddrs[i] = id.Address - } - - return targetAddrs, nil + return targetNodes, nil } // sendTransactionToCollection sends the transaction to the given collection node via grpc @@ -153,9 +146,6 @@ func (b *backendTransactions) sendTransactionToCollector(ctx context.Context, err = b.grpcTxSend(ctx, collectionRPC, tx) if err != nil { - if status.Code(err) == codes.Unavailable { - b.connFactory.InvalidateAccessAPIClient(collectionNodeAddr) - } return fmt.Errorf("failed to send transaction to collection node at %s: %w", collectionNodeAddr, err) } return nil @@ -234,13 +224,15 @@ func (b *backendTransactions) GetTransactionsByBlockID( func (b *backendTransactions) GetTransactionResult( ctx context.Context, txID flow.Identifier, + blockID flow.Identifier, + collectionID flow.Identifier, ) (*access.TransactionResult, error) { // look up transaction from storage start := time.Now() tx, err := b.transactions.ByID(txID) - txErr := rpc.ConvertStorageError(err) - if txErr != nil { + if err != nil { + txErr := rpc.ConvertStorageError(err) if status.Code(txErr) == codes.NotFound { // Tx not found. If we have historical Sporks setup, lets look through those as well historicalTxResult, err := b.getHistoricalTransactionResult(ctx, txID) @@ -258,26 +250,49 @@ func (b *backendTransactions) GetTransactionResult( return nil, txErr } - // find the block for the transaction - block, err := b.lookupBlock(txID) - if err != nil && !errors.Is(err, storage.ErrNotFound) { + block, err := b.retrieveBlock(blockID, collectionID, txID) + + // an error occurred looking up the block or the requested block or collection was not found. + // If looking up the block based solely on the txID returns not found, then no error is + // returned since the block may not be finalized yet. + if err != nil { return nil, rpc.ConvertStorageError(err) } - var blockID flow.Identifier var transactionWasExecuted bool var events []flow.Event var txError string var statusCode uint32 var blockHeight uint64 + // access node may not have the block if it hasn't yet been finalized, hence block can be nil at this point if block != nil { - blockID = block.ID() - transactionWasExecuted, events, statusCode, txError, err = b.lookupTransactionResult(ctx, txID, blockID) - blockHeight = block.Header.Height + foundBlockID := block.ID() + transactionWasExecuted, events, statusCode, txError, err = b.lookupTransactionResult(ctx, txID, foundBlockID) if err != nil { return nil, rpc.ConvertError(err, "failed to retrieve result from any execution node", codes.Internal) } + + // an additional check to ensure the correctness of the collection ID. + expectedCollectionID, err := b.lookupCollectionIDInBlock(block, txID) + if err != nil { + // if the collection has not been indexed yet, the lookup will return a not found error. + // if the request included a blockID or collectionID in its the search criteria, not found + // should result in an error because it's not possible to guarantee that the result found + // is the correct one. + if blockID != flow.ZeroID || collectionID != flow.ZeroID { + return nil, rpc.ConvertStorageError(err) + } + } + + if collectionID == flow.ZeroID { + collectionID = expectedCollectionID + } else if collectionID != expectedCollectionID { + return nil, status.Error(codes.InvalidArgument, "transaction not found in provided collection") + } + + blockID = foundBlockID + blockHeight = block.Header.Height } // derive status of the transaction @@ -295,10 +310,60 @@ func (b *backendTransactions) GetTransactionResult( ErrorMessage: txError, BlockID: blockID, TransactionID: txID, + CollectionID: collectionID, BlockHeight: blockHeight, }, nil } +// lookupCollectionIDInBlock returns the collection ID based on the transaction ID. The lookup is performed in block +// collections. +func (b *backendTransactions) lookupCollectionIDInBlock( + block *flow.Block, + txID flow.Identifier, +) (flow.Identifier, error) { + for _, guarantee := range block.Payload.Guarantees { + collection, err := b.collections.LightByID(guarantee.ID()) + if err != nil { + return flow.ZeroID, err + } + + for _, collectionTxID := range collection.Transactions { + if collectionTxID == txID { + return collection.ID(), nil + } + } + } + return flow.ZeroID, status.Error(codes.NotFound, "transaction not found in block") +} + +// retrieveBlock function returns a block based on the input argument. The block ID lookup has the highest priority, +// followed by the collection ID lookup. If both are missing, the default lookup by transaction ID is performed. +func (b *backendTransactions) retrieveBlock( + + // the requested block or collection was not found. If looking up the block based solely on the txID returns + // not found, then no error is returned. + blockID flow.Identifier, + collectionID flow.Identifier, + txID flow.Identifier, +) (*flow.Block, error) { + if blockID != flow.ZeroID { + return b.blocks.ByID(blockID) + } + + if collectionID != flow.ZeroID { + return b.blocks.ByCollectionID(collectionID) + } + + // find the block for the transaction + block, err := b.lookupBlock(txID) + + if err != nil && !errors.Is(err, storage.ErrNotFound) { + return nil, err + } + + return block, nil +} + func (b *backendTransactions) GetTransactionResultsByBlockID( ctx context.Context, blockID flow.Identifier, @@ -312,6 +377,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( req := &execproto.GetTransactionsByBlockIDRequest{ BlockId: blockID[:], } + execNodes, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) if err != nil { if IsInsufficientExecutionReceipts(err) { @@ -351,10 +417,16 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, rpc.ConvertStorageError(err) } + events, err := convert.MessagesToEventsFromVersion(txResult.GetEvents(), resp.GetEventEncodingVersion()) + if err != nil { + return nil, status.Errorf(codes.Internal, + "failed to convert events to message in txID %x: %v", txID, err) + } + results = append(results, &access.TransactionResult{ Status: txStatus, StatusCode: uint(txResult.GetStatusCode()), - Events: convert.MessagesToEvents(txResult.GetEvents()), + Events: events, ErrorMessage: txResult.GetErrorMessage(), BlockID: blockID, TransactionID: txID, @@ -400,10 +472,15 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( return nil, rpc.ConvertStorageError(err) } + events, err := convert.MessagesToEventsFromVersion(systemTxResult.GetEvents(), resp.GetEventEncodingVersion()) + if err != nil { + return nil, rpc.ConvertError(err, "failed to convert events from system tx result", codes.Internal) + } + results = append(results, &access.TransactionResult{ Status: systemTxStatus, StatusCode: uint(systemTxResult.GetStatusCode()), - Events: convert.MessagesToEvents(systemTxResult.GetEvents()), + Events: events, ErrorMessage: systemTxResult.GetErrorMessage(), BlockID: blockID, TransactionID: systemTx.ID(), @@ -432,6 +509,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( BlockId: blockID[:], Index: index, } + execNodes, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) if err != nil { if IsInsufficientExecutionReceipts(err) { @@ -451,11 +529,16 @@ func (b *backendTransactions) GetTransactionResultByIndex( return nil, rpc.ConvertStorageError(err) } + events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), resp.GetEventEncodingVersion()) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to convert events in blockID %x: %v", blockID, err) + } + // convert to response, cache and return return &access.TransactionResult{ Status: txStatus, StatusCode: uint(resp.GetStatusCode()), - Events: convert.MessagesToEvents(resp.GetEvents()), + Events: events, ErrorMessage: resp.GetErrorMessage(), BlockID: blockID, BlockHeight: block.Header.Height, @@ -670,7 +753,10 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( return nil, 0, "", err } - events := convert.MessagesToEvents(resp.GetEvents()) + events, err := convert.MessagesToEventsFromVersion(resp.GetEvents(), resp.GetEventEncodingVersion()) + if err != nil { + return nil, 0, "", rpc.ConvertError(err, "failed to convert events to message", codes.Internal) + } return events, resp.GetStatusCode(), resp.GetErrorMessage(), nil } @@ -684,32 +770,36 @@ func (b *backendTransactions) getTransactionResultFromAnyExeNode( execNodes flow.IdentityList, req *execproto.GetTransactionResultRequest, ) (*execproto.GetTransactionResultResponse, error) { - var errs *multierror.Error - logAnyError := func() { - errToReturn := errs.ErrorOrNil() + var errToReturn error + + defer func() { if errToReturn != nil { b.log.Info().Err(errToReturn).Msg("failed to get transaction result from execution nodes") } - } - defer logAnyError() - // try to execute the script on one of the execution nodes - for _, execNode := range execNodes { - resp, err := b.tryGetTransactionResult(ctx, execNode, req) - if err == nil { - b.log.Debug(). - Str("execution_node", execNode.String()). - Hex("block_id", req.GetBlockId()). - Hex("transaction_id", req.GetTransactionId()). - Msg("Successfully got transaction results from any node") - return resp, nil - } - if status.Code(err) == codes.NotFound { - return nil, err - } - errs = multierror.Append(errs, err) - } + }() + + var resp *execproto.GetTransactionResultResponse + errToReturn = b.nodeCommunicator.CallAvailableNode( + execNodes, + func(node *flow.Identity) error { + var err error + resp, err = b.tryGetTransactionResult(ctx, node, req) + if err == nil { + b.log.Debug(). + Str("execution_node", node.String()). + Hex("block_id", req.GetBlockId()). + Hex("transaction_id", req.GetTransactionId()). + Msg("Successfully got transaction results from any node") + return nil + } + return err + }, + func(_ *flow.Identity, err error) bool { + return status.Code(err) == codes.NotFound + }, + ) - return nil, errs.ErrorOrNil() + return resp, errToReturn } func (b *backendTransactions) tryGetTransactionResult( @@ -725,9 +815,6 @@ func (b *backendTransactions) tryGetTransactionResult( resp, err := execRPCClient.GetTransactionResult(ctx, req) if err != nil { - if status.Code(err) == codes.Unavailable { - b.connFactory.InvalidateExecutionAPIClient(execNode.Address) - } return nil, err } @@ -739,12 +826,12 @@ func (b *backendTransactions) getTransactionResultsByBlockIDFromAnyExeNode( execNodes flow.IdentityList, req *execproto.GetTransactionsByBlockIDRequest, ) (*execproto.GetTransactionResultsResponse, error) { - var errs *multierror.Error + var errToReturn error defer func() { // log the errors - if err := errs.ErrorOrNil(); err != nil { - b.log.Err(errs).Msg("failed to get transaction results from execution nodes") + if errToReturn != nil { + b.log.Err(errToReturn).Msg("failed to get transaction results from execution nodes") } }() @@ -753,22 +840,27 @@ func (b *backendTransactions) getTransactionResultsByBlockIDFromAnyExeNode( return nil, errors.New("zero execution nodes") } - for _, execNode := range execNodes { - resp, err := b.tryGetTransactionResultsByBlockID(ctx, execNode, req) - if err == nil { - b.log.Debug(). - Str("execution_node", execNode.String()). - Hex("block_id", req.GetBlockId()). - Msg("Successfully got transaction results from any node") - return resp, nil - } - if status.Code(err) == codes.NotFound { - return nil, err - } - errs = multierror.Append(errs, err) - } + var resp *execproto.GetTransactionResultsResponse + errToReturn = b.nodeCommunicator.CallAvailableNode( + execNodes, + func(node *flow.Identity) error { + var err error + resp, err = b.tryGetTransactionResultsByBlockID(ctx, node, req) + if err == nil { + b.log.Debug(). + Str("execution_node", node.String()). + Hex("block_id", req.GetBlockId()). + Msg("Successfully got transaction results from any node") + return nil + } + return err + }, + func(_ *flow.Identity, err error) bool { + return status.Code(err) == codes.NotFound + }, + ) - return nil, errs.ErrorOrNil() + return resp, errToReturn } func (b *backendTransactions) tryGetTransactionResultsByBlockID( @@ -784,9 +876,6 @@ func (b *backendTransactions) tryGetTransactionResultsByBlockID( resp, err := execRPCClient.GetTransactionResultsByBlockID(ctx, req) if err != nil { - if status.Code(err) == codes.Unavailable { - b.connFactory.InvalidateExecutionAPIClient(execNode.Address) - } return nil, err } @@ -798,37 +887,39 @@ func (b *backendTransactions) getTransactionResultByIndexFromAnyExeNode( execNodes flow.IdentityList, req *execproto.GetTransactionByIndexRequest, ) (*execproto.GetTransactionResultResponse, error) { - var errs *multierror.Error - logAnyError := func() { - errToReturn := errs.ErrorOrNil() + var errToReturn error + defer func() { if errToReturn != nil { b.log.Info().Err(errToReturn).Msg("failed to get transaction result from execution nodes") } - } - defer logAnyError() + }() if len(execNodes) == 0 { return nil, errors.New("zero execution nodes provided") } - // try to execute the script on one of the execution nodes - for _, execNode := range execNodes { - resp, err := b.tryGetTransactionResultByIndex(ctx, execNode, req) - if err == nil { - b.log.Debug(). - Str("execution_node", execNode.String()). - Hex("block_id", req.GetBlockId()). - Uint32("index", req.GetIndex()). - Msg("Successfully got transaction results from any node") - return resp, nil - } - if status.Code(err) == codes.NotFound { - return nil, err - } - errs = multierror.Append(errs, err) - } + var resp *execproto.GetTransactionResultResponse + errToReturn = b.nodeCommunicator.CallAvailableNode( + execNodes, + func(node *flow.Identity) error { + var err error + resp, err = b.tryGetTransactionResultByIndex(ctx, node, req) + if err == nil { + b.log.Debug(). + Str("execution_node", node.String()). + Hex("block_id", req.GetBlockId()). + Uint32("index", req.GetIndex()). + Msg("Successfully got transaction results from any node") + return nil + } + return err + }, + func(_ *flow.Identity, err error) bool { + return status.Code(err) == codes.NotFound + }, + ) - return nil, errs.ErrorOrNil() + return resp, errToReturn } func (b *backendTransactions) tryGetTransactionResultByIndex( @@ -844,9 +935,6 @@ func (b *backendTransactions) tryGetTransactionResultByIndex( resp, err := execRPCClient.GetTransactionResultByIndex(ctx, req) if err != nil { - if status.Code(err) == codes.Unavailable { - b.connFactory.InvalidateExecutionAPIClient(execNode.Address) - } return nil, err } diff --git a/engine/access/rpc/backend/connection_factory.go b/engine/access/rpc/backend/connection_factory.go deleted file mode 100644 index 63ead3d3e32..00000000000 --- a/engine/access/rpc/backend/connection_factory.go +++ /dev/null @@ -1,276 +0,0 @@ -package backend - -import ( - "context" - "fmt" - "io" - "net" - "sync" - "time" - - lru "github.com/hashicorp/golang-lru" - "github.com/onflow/flow/protobuf/go/flow/access" - "github.com/onflow/flow/protobuf/go/flow/execution" - "github.com/rs/zerolog" - "google.golang.org/grpc" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/keepalive" - - "github.com/onflow/flow-go/module" -) - -// DefaultClientTimeout is used when making a GRPC request to a collection node or an execution node -const DefaultClientTimeout = 3 * time.Second - -// ConnectionFactory is used to create an access api client -type ConnectionFactory interface { - GetAccessAPIClient(address string) (access.AccessAPIClient, io.Closer, error) - InvalidateAccessAPIClient(address string) - GetExecutionAPIClient(address string) (execution.ExecutionAPIClient, io.Closer, error) - InvalidateExecutionAPIClient(address string) -} - -type ProxyConnectionFactory struct { - ConnectionFactory - targetAddress string -} - -type noopCloser struct{} - -func (c *noopCloser) Close() error { - return nil -} - -func (p *ProxyConnectionFactory) GetAccessAPIClient(address string) (access.AccessAPIClient, io.Closer, error) { - return p.ConnectionFactory.GetAccessAPIClient(p.targetAddress) -} - -func (p *ProxyConnectionFactory) GetExecutionAPIClient(address string) (execution.ExecutionAPIClient, io.Closer, error) { - return p.ConnectionFactory.GetExecutionAPIClient(p.targetAddress) -} - -type ConnectionFactoryImpl struct { - CollectionGRPCPort uint - ExecutionGRPCPort uint - CollectionNodeGRPCTimeout time.Duration - ExecutionNodeGRPCTimeout time.Duration - ConnectionsCache *lru.Cache - CacheSize uint - MaxMsgSize uint - AccessMetrics module.AccessMetrics - Log zerolog.Logger - mutex sync.Mutex -} - -type CachedClient struct { - ClientConn *grpc.ClientConn - Address string - mutex sync.Mutex - timeout time.Duration -} - -// createConnection creates new gRPC connections to remote node -func (cf *ConnectionFactoryImpl) createConnection(address string, timeout time.Duration) (*grpc.ClientConn, error) { - - if timeout == 0 { - timeout = DefaultClientTimeout - } - - keepaliveParams := keepalive.ClientParameters{ - // how long the client will wait before sending a keepalive to the server if there is no activity - Time: 10 * time.Second, - // how long the client will wait for a response from the keepalive before closing - Timeout: timeout, - } - - // ClientConn's default KeepAlive on connections is indefinite, assuming the timeout isn't reached - // The connections should be safe to be persisted and reused - // https://pkg.go.dev/google.golang.org/grpc#WithKeepaliveParams - // https://grpc.io/blog/grpc-on-http2/#keeping-connections-alive - conn, err := grpc.Dial( - address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(cf.MaxMsgSize))), - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithKeepaliveParams(keepaliveParams), - WithClientUnaryInterceptor(timeout)) - if err != nil { - return nil, fmt.Errorf("failed to connect to address %s: %w", address, err) - } - return conn, nil -} - -func (cf *ConnectionFactoryImpl) retrieveConnection(grpcAddress string, timeout time.Duration) (*grpc.ClientConn, error) { - var conn *grpc.ClientConn - var store *CachedClient - cacheHit := false - cf.mutex.Lock() - if res, ok := cf.ConnectionsCache.Get(grpcAddress); ok { - cacheHit = true - store = res.(*CachedClient) - conn = store.ClientConn - } else { - store = &CachedClient{ - ClientConn: nil, - Address: grpcAddress, - timeout: timeout, - } - cf.Log.Debug().Str("cached_client_added", grpcAddress).Msg("adding new cached client to pool") - cf.ConnectionsCache.Add(grpcAddress, store) - if cf.AccessMetrics != nil { - cf.AccessMetrics.ConnectionAddedToPool() - } - } - cf.mutex.Unlock() - store.mutex.Lock() - defer store.mutex.Unlock() - - if conn == nil || conn.GetState() == connectivity.Shutdown { - var err error - conn, err = cf.createConnection(grpcAddress, timeout) - if err != nil { - return nil, err - } - store.ClientConn = conn - if cf.AccessMetrics != nil { - if cacheHit { - cf.AccessMetrics.ConnectionFromPoolUpdated() - } - cf.AccessMetrics.NewConnectionEstablished() - cf.AccessMetrics.TotalConnectionsInPool(uint(cf.ConnectionsCache.Len()), cf.CacheSize) - } - } else if cf.AccessMetrics != nil { - cf.AccessMetrics.ConnectionFromPoolReused() - } - return conn, nil -} - -func (cf *ConnectionFactoryImpl) GetAccessAPIClient(address string) (access.AccessAPIClient, io.Closer, error) { - - grpcAddress, err := getGRPCAddress(address, cf.CollectionGRPCPort) - if err != nil { - return nil, nil, err - } - - var conn *grpc.ClientConn - if cf.ConnectionsCache != nil { - conn, err = cf.retrieveConnection(grpcAddress, cf.CollectionNodeGRPCTimeout) - if err != nil { - return nil, nil, err - } - return access.NewAccessAPIClient(conn), &noopCloser{}, err - } - - conn, err = cf.createConnection(grpcAddress, cf.CollectionNodeGRPCTimeout) - if err != nil { - return nil, nil, err - } - - accessAPIClient := access.NewAccessAPIClient(conn) - closer := io.Closer(conn) - return accessAPIClient, closer, nil -} - -func (cf *ConnectionFactoryImpl) InvalidateAccessAPIClient(address string) { - if cf.ConnectionsCache != nil { - cf.Log.Debug().Str("cached_access_client_invalidated", address).Msg("invalidating cached access client") - cf.invalidateAPIClient(address, cf.CollectionGRPCPort) - } -} - -func (cf *ConnectionFactoryImpl) GetExecutionAPIClient(address string) (execution.ExecutionAPIClient, io.Closer, error) { - - grpcAddress, err := getGRPCAddress(address, cf.ExecutionGRPCPort) - if err != nil { - return nil, nil, err - } - - var conn *grpc.ClientConn - if cf.ConnectionsCache != nil { - conn, err = cf.retrieveConnection(grpcAddress, cf.ExecutionNodeGRPCTimeout) - if err != nil { - return nil, nil, err - } - return execution.NewExecutionAPIClient(conn), &noopCloser{}, nil - } - - conn, err = cf.createConnection(grpcAddress, cf.ExecutionNodeGRPCTimeout) - if err != nil { - return nil, nil, err - } - - executionAPIClient := execution.NewExecutionAPIClient(conn) - closer := io.Closer(conn) - return executionAPIClient, closer, nil -} - -func (cf *ConnectionFactoryImpl) InvalidateExecutionAPIClient(address string) { - if cf.ConnectionsCache != nil { - cf.Log.Debug().Str("cached_execution_client_invalidated", address).Msg("invalidating cached execution client") - cf.invalidateAPIClient(address, cf.ExecutionGRPCPort) - } -} - -func (cf *ConnectionFactoryImpl) invalidateAPIClient(address string, port uint) { - grpcAddress, _ := getGRPCAddress(address, port) - if res, ok := cf.ConnectionsCache.Get(grpcAddress); ok { - store := res.(*CachedClient) - store.Close() - if cf.AccessMetrics != nil { - cf.AccessMetrics.ConnectionFromPoolInvalidated() - } - } -} - -func (s *CachedClient) Close() { - s.mutex.Lock() - conn := s.ClientConn - s.ClientConn = nil - s.mutex.Unlock() - if conn == nil { - return - } - // allow time for any existing requests to finish before closing the connection - time.Sleep(s.timeout + 1*time.Second) - conn.Close() -} - -// getExecutionNodeAddress translates flow.Identity address to the GRPC address of the node by switching the port to the -// GRPC port from the libp2p port -func getGRPCAddress(address string, grpcPort uint) (string, error) { - // split hostname and port - hostnameOrIP, _, err := net.SplitHostPort(address) - if err != nil { - return "", err - } - // use the hostname from identity list and port number as the one passed in as argument - grpcAddress := fmt.Sprintf("%s:%d", hostnameOrIP, grpcPort) - - return grpcAddress, nil -} - -func WithClientUnaryInterceptor(timeout time.Duration) grpc.DialOption { - - clientTimeoutInterceptor := func( - ctx context.Context, - method string, - req interface{}, - reply interface{}, - cc *grpc.ClientConn, - invoker grpc.UnaryInvoker, - opts ...grpc.CallOption, - ) error { - - // create a context that expires after timeout - ctxWithTimeout, cancel := context.WithTimeout(ctx, timeout) - - defer cancel() - - // call the remote GRPC using the short context - err := invoker(ctxWithTimeout, method, req, reply, cc, opts...) - - return err - } - - return grpc.WithUnaryInterceptor(clientTimeoutInterceptor) -} diff --git a/engine/access/rpc/backend/connection_factory_test.go b/engine/access/rpc/backend/connection_factory_test.go deleted file mode 100644 index fa4801a5897..00000000000 --- a/engine/access/rpc/backend/connection_factory_test.go +++ /dev/null @@ -1,484 +0,0 @@ -package backend - -import ( - "context" - "fmt" - "net" - "strconv" - "strings" - "sync" - "testing" - "time" - - lru "github.com/hashicorp/golang-lru" - "github.com/onflow/flow/protobuf/go/flow/access" - "github.com/onflow/flow/protobuf/go/flow/execution" - "github.com/stretchr/testify/assert" - testifymock "github.com/stretchr/testify/mock" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/onflow/flow-go/engine/access/mock" - "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/utils/unittest" -) - -func TestProxyAccessAPI(t *testing.T) { - // create a collection node - cn := new(collectionNode) - cn.start(t) - defer cn.stop(t) - - req := &access.PingRequest{} - expected := &access.PingResponse{} - cn.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) - - // create the factory - connectionFactory := new(ConnectionFactoryImpl) - // set the collection grpc port - connectionFactory.CollectionGRPCPort = cn.port - // set metrics reporting - connectionFactory.AccessMetrics = metrics.NewNoopCollector() - - proxyConnectionFactory := ProxyConnectionFactory{ - ConnectionFactory: connectionFactory, - targetAddress: cn.listener.Addr().String(), - } - - // get a collection API client - client, conn, err := proxyConnectionFactory.GetAccessAPIClient("foo") - defer conn.Close() - assert.NoError(t, err) - - ctx := context.Background() - // make the call to the collection node - resp, err := client.Ping(ctx, req) - assert.NoError(t, err) - assert.Equal(t, resp, expected) -} - -func TestProxyExecutionAPI(t *testing.T) { - // create an execution node - en := new(executionNode) - en.start(t) - defer en.stop(t) - - req := &execution.PingRequest{} - expected := &execution.PingResponse{} - en.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) - - // create the factory - connectionFactory := new(ConnectionFactoryImpl) - // set the execution grpc port - connectionFactory.ExecutionGRPCPort = en.port - // set metrics reporting - connectionFactory.AccessMetrics = metrics.NewNoopCollector() - - proxyConnectionFactory := ProxyConnectionFactory{ - ConnectionFactory: connectionFactory, - targetAddress: en.listener.Addr().String(), - } - - // get an execution API client - client, _, err := proxyConnectionFactory.GetExecutionAPIClient("foo") - assert.NoError(t, err) - - ctx := context.Background() - // make the call to the execution node - resp, err := client.Ping(ctx, req) - assert.NoError(t, err) - assert.Equal(t, resp, expected) -} - -func TestProxyAccessAPIConnectionReuse(t *testing.T) { - // create a collection node - cn := new(collectionNode) - cn.start(t) - defer cn.stop(t) - - req := &access.PingRequest{} - expected := &access.PingResponse{} - cn.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) - - // create the factory - connectionFactory := new(ConnectionFactoryImpl) - // set the collection grpc port - connectionFactory.CollectionGRPCPort = cn.port - // set the connection pool cache size - cacheSize := 5 - cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { - evictedValue.(*CachedClient).Close() - }) - connectionFactory.ConnectionsCache = cache - connectionFactory.CacheSize = uint(cacheSize) - // set metrics reporting - connectionFactory.AccessMetrics = metrics.NewNoopCollector() - - proxyConnectionFactory := ProxyConnectionFactory{ - ConnectionFactory: connectionFactory, - targetAddress: cn.listener.Addr().String(), - } - - // get a collection API client - _, closer, err := proxyConnectionFactory.GetAccessAPIClient("foo") - assert.Equal(t, connectionFactory.ConnectionsCache.Len(), 1) - assert.NoError(t, err) - assert.Nil(t, closer.Close()) - - var conn *grpc.ClientConn - res, ok := connectionFactory.ConnectionsCache.Get(proxyConnectionFactory.targetAddress) - assert.True(t, ok) - conn = res.(*CachedClient).ClientConn - - // check if api client can be rebuilt with retrieved connection - accessAPIClient := access.NewAccessAPIClient(conn) - ctx := context.Background() - resp, err := accessAPIClient.Ping(ctx, req) - assert.NoError(t, err) - assert.Equal(t, resp, expected) -} - -func TestProxyExecutionAPIConnectionReuse(t *testing.T) { - // create an execution node - en := new(executionNode) - en.start(t) - defer en.stop(t) - - req := &execution.PingRequest{} - expected := &execution.PingResponse{} - en.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) - - // create the factory - connectionFactory := new(ConnectionFactoryImpl) - // set the execution grpc port - connectionFactory.ExecutionGRPCPort = en.port - // set the connection pool cache size - cacheSize := 5 - cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { - evictedValue.(*CachedClient).Close() - }) - connectionFactory.ConnectionsCache = cache - connectionFactory.CacheSize = uint(cacheSize) - // set metrics reporting - connectionFactory.AccessMetrics = metrics.NewNoopCollector() - - proxyConnectionFactory := ProxyConnectionFactory{ - ConnectionFactory: connectionFactory, - targetAddress: en.listener.Addr().String(), - } - - // get an execution API client - _, closer, err := proxyConnectionFactory.GetExecutionAPIClient("foo") - assert.Equal(t, connectionFactory.ConnectionsCache.Len(), 1) - assert.NoError(t, err) - assert.Nil(t, closer.Close()) - - var conn *grpc.ClientConn - res, ok := connectionFactory.ConnectionsCache.Get(proxyConnectionFactory.targetAddress) - assert.True(t, ok) - conn = res.(*CachedClient).ClientConn - - // check if api client can be rebuilt with retrieved connection - executionAPIClient := execution.NewExecutionAPIClient(conn) - ctx := context.Background() - resp, err := executionAPIClient.Ping(ctx, req) - assert.NoError(t, err) - assert.Equal(t, resp, expected) -} - -// TestExecutionNodeClientTimeout tests that the execution API client times out after the timeout duration -func TestExecutionNodeClientTimeout(t *testing.T) { - - timeout := 10 * time.Millisecond - - // create an execution node - en := new(executionNode) - en.start(t) - defer en.stop(t) - - // setup the handler mock to not respond within the timeout - req := &execution.PingRequest{} - resp := &execution.PingResponse{} - en.handler.On("Ping", testifymock.Anything, req).After(timeout+time.Second).Return(resp, nil) - - // create the factory - connectionFactory := new(ConnectionFactoryImpl) - // set the execution grpc port - connectionFactory.ExecutionGRPCPort = en.port - // set the execution grpc client timeout - connectionFactory.ExecutionNodeGRPCTimeout = timeout - // set the connection pool cache size - cacheSize := 5 - cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { - evictedValue.(*CachedClient).Close() - }) - connectionFactory.ConnectionsCache = cache - connectionFactory.CacheSize = uint(cacheSize) - // set metrics reporting - connectionFactory.AccessMetrics = metrics.NewNoopCollector() - - // create the execution API client - client, _, err := connectionFactory.GetExecutionAPIClient(en.listener.Addr().String()) - assert.NoError(t, err) - - ctx := context.Background() - // make the call to the execution node - _, err = client.Ping(ctx, req) - - // assert that the client timed out - assert.Equal(t, codes.DeadlineExceeded, status.Code(err)) -} - -// TestCollectionNodeClientTimeout tests that the collection API client times out after the timeout duration -func TestCollectionNodeClientTimeout(t *testing.T) { - - timeout := 10 * time.Millisecond - - // create a collection node - cn := new(collectionNode) - cn.start(t) - defer cn.stop(t) - - // setup the handler mock to not respond within the timeout - req := &access.PingRequest{} - resp := &access.PingResponse{} - cn.handler.On("Ping", testifymock.Anything, req).After(timeout+time.Second).Return(resp, nil) - - // create the factory - connectionFactory := new(ConnectionFactoryImpl) - // set the collection grpc port - connectionFactory.CollectionGRPCPort = cn.port - // set the collection grpc client timeout - connectionFactory.CollectionNodeGRPCTimeout = timeout - // set the connection pool cache size - cacheSize := 5 - cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { - evictedValue.(*CachedClient).Close() - }) - connectionFactory.ConnectionsCache = cache - connectionFactory.CacheSize = uint(cacheSize) - // set metrics reporting - connectionFactory.AccessMetrics = metrics.NewNoopCollector() - - // create the collection API client - client, _, err := connectionFactory.GetAccessAPIClient(cn.listener.Addr().String()) - assert.NoError(t, err) - - ctx := context.Background() - // make the call to the execution node - _, err = client.Ping(ctx, req) - - // assert that the client timed out - assert.Equal(t, codes.DeadlineExceeded, status.Code(err)) -} - -// TestConnectionPoolFull tests that the LRU cache replaces connections when full -func TestConnectionPoolFull(t *testing.T) { - // create a collection node - cn1, cn2, cn3 := new(collectionNode), new(collectionNode), new(collectionNode) - cn1.start(t) - cn2.start(t) - cn3.start(t) - defer cn1.stop(t) - defer cn2.stop(t) - defer cn3.stop(t) - - req := &access.PingRequest{} - expected := &access.PingResponse{} - cn1.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) - cn2.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) - cn3.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) - - // create the factory - connectionFactory := new(ConnectionFactoryImpl) - // set the collection grpc port - connectionFactory.CollectionGRPCPort = cn1.port - // set the connection pool cache size - cacheSize := 2 - cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { - evictedValue.(*CachedClient).Close() - }) - connectionFactory.ConnectionsCache = cache - connectionFactory.CacheSize = uint(cacheSize) - // set metrics reporting - connectionFactory.AccessMetrics = metrics.NewNoopCollector() - - cn1Address := "foo1:123" - cn2Address := "foo2:123" - cn3Address := "foo3:123" - - // get a collection API client - _, _, err := connectionFactory.GetAccessAPIClient(cn1Address) - assert.Equal(t, connectionFactory.ConnectionsCache.Len(), 1) - assert.NoError(t, err) - - _, _, err = connectionFactory.GetAccessAPIClient(cn2Address) - assert.Equal(t, connectionFactory.ConnectionsCache.Len(), 2) - assert.NoError(t, err) - - _, _, err = connectionFactory.GetAccessAPIClient(cn1Address) - assert.Equal(t, connectionFactory.ConnectionsCache.Len(), 2) - assert.NoError(t, err) - - // Expecting to replace cn2 because cn1 was accessed more recently - _, _, err = connectionFactory.GetAccessAPIClient(cn3Address) - assert.Equal(t, connectionFactory.ConnectionsCache.Len(), 2) - assert.NoError(t, err) - - var hostnameOrIP string - hostnameOrIP, _, err = net.SplitHostPort(cn1Address) - assert.NoError(t, err) - grpcAddress1 := fmt.Sprintf("%s:%d", hostnameOrIP, connectionFactory.CollectionGRPCPort) - hostnameOrIP, _, err = net.SplitHostPort(cn2Address) - assert.NoError(t, err) - grpcAddress2 := fmt.Sprintf("%s:%d", hostnameOrIP, connectionFactory.CollectionGRPCPort) - hostnameOrIP, _, err = net.SplitHostPort(cn3Address) - assert.NoError(t, err) - grpcAddress3 := fmt.Sprintf("%s:%d", hostnameOrIP, connectionFactory.CollectionGRPCPort) - - contains1 := connectionFactory.ConnectionsCache.Contains(grpcAddress1) - contains2 := connectionFactory.ConnectionsCache.Contains(grpcAddress2) - contains3 := connectionFactory.ConnectionsCache.Contains(grpcAddress3) - - assert.True(t, contains1) - assert.False(t, contains2) - assert.True(t, contains3) -} - -// TestConnectionPoolStale tests that a new connection will be established if the old one cached is stale -func TestConnectionPoolStale(t *testing.T) { - // create a collection node - cn := new(collectionNode) - cn.start(t) - defer cn.stop(t) - - req := &access.PingRequest{} - expected := &access.PingResponse{} - cn.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) - - // create the factory - connectionFactory := new(ConnectionFactoryImpl) - // set the collection grpc port - connectionFactory.CollectionGRPCPort = cn.port - // set the connection pool cache size - cacheSize := 5 - cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { - evictedValue.(*CachedClient).Close() - }) - connectionFactory.ConnectionsCache = cache - connectionFactory.CacheSize = uint(cacheSize) - // set metrics reporting - connectionFactory.AccessMetrics = metrics.NewNoopCollector() - - proxyConnectionFactory := ProxyConnectionFactory{ - ConnectionFactory: connectionFactory, - targetAddress: cn.listener.Addr().String(), - } - - // get a collection API client - client, _, err := proxyConnectionFactory.GetAccessAPIClient("foo") - assert.Equal(t, connectionFactory.ConnectionsCache.Len(), 1) - assert.NoError(t, err) - // close connection to simulate something "going wrong" with our stored connection - res, _ := connectionFactory.ConnectionsCache.Get(proxyConnectionFactory.targetAddress) - - res.(*CachedClient).Close() - - ctx := context.Background() - // make the call to the collection node (should fail, connection closed) - _, err = client.Ping(ctx, req) - assert.Error(t, err) - - // re-access, should replace stale connection in cache with new one - _, _, _ = proxyConnectionFactory.GetAccessAPIClient("foo") - assert.Equal(t, connectionFactory.ConnectionsCache.Len(), 1) - - var conn *grpc.ClientConn - res, ok := connectionFactory.ConnectionsCache.Get(proxyConnectionFactory.targetAddress) - assert.True(t, ok) - conn = res.(*CachedClient).ClientConn - - // check if api client can be rebuilt with retrieved connection - accessAPIClient := access.NewAccessAPIClient(conn) - ctx = context.Background() - resp, err := accessAPIClient.Ping(ctx, req) - assert.NoError(t, err) - assert.Equal(t, resp, expected) -} - -// node mocks a flow node that runs a GRPC server -type node struct { - server *grpc.Server - listener net.Listener - port uint -} - -func (n *node) setupNode(t *testing.T) { - n.server = grpc.NewServer() - listener, err := net.Listen("tcp4", unittest.DefaultAddress) - assert.NoError(t, err) - n.listener = listener - assert.Eventually(t, func() bool { - return !strings.HasSuffix(listener.Addr().String(), ":0") - }, time.Second*4, 10*time.Millisecond) - - _, port, err := net.SplitHostPort(listener.Addr().String()) - assert.NoError(t, err) - portAsUint, err := strconv.ParseUint(port, 10, 32) - assert.NoError(t, err) - n.port = uint(portAsUint) -} - -func (n *node) start(t *testing.T) { - // using a wait group here to ensure the goroutine has started before returning. Otherwise, - // there's a race condition where the server is sometimes stopped before it has started - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - wg.Done() - err := n.server.Serve(n.listener) - assert.NoError(t, err) - }() - unittest.RequireReturnsBefore(t, wg.Wait, 10*time.Millisecond, "could not start goroutine on time") -} - -func (n *node) stop(t *testing.T) { - if n.server != nil { - n.server.Stop() - } -} - -type executionNode struct { - node - handler *mock.ExecutionAPIServer -} - -func (en *executionNode) start(t *testing.T) { - en.setupNode(t) - handler := new(mock.ExecutionAPIServer) - execution.RegisterExecutionAPIServer(en.server, handler) - en.handler = handler - en.node.start(t) -} - -func (en *executionNode) stop(t *testing.T) { - en.node.stop(t) -} - -type collectionNode struct { - node - handler *mock.AccessAPIServer -} - -func (cn *collectionNode) start(t *testing.T) { - cn.setupNode(t) - handler := new(mock.AccessAPIServer) - access.RegisterAccessAPIServer(cn.server, handler) - cn.handler = handler - cn.node.start(t) -} - -func (cn *collectionNode) stop(t *testing.T) { - cn.node.stop(t) -} diff --git a/engine/access/rpc/backend/historical_access_test.go b/engine/access/rpc/backend/historical_access_test.go index 6971bb6298d..42dd829dbbc 100644 --- a/engine/access/rpc/backend/historical_access_test.go +++ b/engine/access/rpc/backend/historical_access_test.go @@ -55,6 +55,8 @@ func (suite *Suite) TestHistoricalTransactionResult() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // Successfully return the transaction from the historical node @@ -64,7 +66,7 @@ func (suite *Suite) TestHistoricalTransactionResult() { Once() // Make the call for the transaction result - result, err := backend.GetTransactionResult(ctx, txID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) // status should be sealed @@ -112,6 +114,8 @@ func (suite *Suite) TestHistoricalTransaction() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) // Successfully return the transaction from the historical node diff --git a/engine/access/rpc/backend/mock/connection_factory.go b/engine/access/rpc/backend/mock/connection_factory.go index 5dfd657ec7e..eebc006485d 100644 --- a/engine/access/rpc/backend/mock/connection_factory.go +++ b/engine/access/rpc/backend/mock/connection_factory.go @@ -52,6 +52,41 @@ func (_m *ConnectionFactory) GetAccessAPIClient(address string) (access.AccessAP return r0, r1, r2 } +// GetAccessAPIClientWithPort provides a mock function with given fields: address, port +func (_m *ConnectionFactory) GetAccessAPIClientWithPort(address string, port uint) (access.AccessAPIClient, io.Closer, error) { + ret := _m.Called(address, port) + + var r0 access.AccessAPIClient + var r1 io.Closer + var r2 error + if rf, ok := ret.Get(0).(func(string, uint) (access.AccessAPIClient, io.Closer, error)); ok { + return rf(address, port) + } + if rf, ok := ret.Get(0).(func(string, uint) access.AccessAPIClient); ok { + r0 = rf(address, port) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPIClient) + } + } + + if rf, ok := ret.Get(1).(func(string, uint) io.Closer); ok { + r1 = rf(address, port) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(io.Closer) + } + } + + if rf, ok := ret.Get(2).(func(string, uint) error); ok { + r2 = rf(address, port) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + // GetExecutionAPIClient provides a mock function with given fields: address func (_m *ConnectionFactory) GetExecutionAPIClient(address string) (execution.ExecutionAPIClient, io.Closer, error) { ret := _m.Called(address) diff --git a/engine/access/rpc/backend/node_communicator.go b/engine/access/rpc/backend/node_communicator.go new file mode 100644 index 00000000000..d75432b0b29 --- /dev/null +++ b/engine/access/rpc/backend/node_communicator.go @@ -0,0 +1,75 @@ +package backend + +import ( + "github.com/hashicorp/go-multierror" + "github.com/sony/gobreaker" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/model/flow" +) + +// maxFailedRequestCount represents the maximum number of failed requests before returning errors. +const maxFailedRequestCount = 3 + +// NodeAction is a callback function type that represents an action to be performed on a node. +// It takes a node as input and returns an error indicating the result of the action. +type NodeAction func(node *flow.Identity) error + +// ErrorTerminator is a callback function that determines whether an error should terminate further execution. +// It takes an error as input and returns a boolean value indicating whether the error should be considered terminal. +type ErrorTerminator func(node *flow.Identity, err error) bool + +// NodeCommunicator is responsible for calling available nodes in the backend. +type NodeCommunicator struct { + nodeSelectorFactory NodeSelectorFactory +} + +// NewNodeCommunicator creates a new instance of NodeCommunicator. +func NewNodeCommunicator(circuitBreakerEnabled bool) *NodeCommunicator { + return &NodeCommunicator{ + nodeSelectorFactory: NodeSelectorFactory{circuitBreakerEnabled: circuitBreakerEnabled}, + } +} + +// CallAvailableNode calls the provided function on the available nodes. +// It iterates through the nodes and executes the function. +// If an error occurs, it applies the custom error terminator (if provided) and keeps track of the errors. +// If the error occurs in circuit breaker, it continues to the next node. +// If the maximum failed request count is reached, it returns the accumulated errors. +func (b *NodeCommunicator) CallAvailableNode( + nodes flow.IdentityList, + call NodeAction, + shouldTerminateOnError ErrorTerminator, +) error { + var errs *multierror.Error + nodeSelector, err := b.nodeSelectorFactory.SelectNodes(nodes) + if err != nil { + return err + } + + for node := nodeSelector.Next(); node != nil; node = nodeSelector.Next() { + err := call(node) + if err == nil { + return nil + } + + if shouldTerminateOnError != nil && shouldTerminateOnError(node, err) { + return err + } + + if err == gobreaker.ErrOpenState { + if !nodeSelector.HasNext() && len(errs.Errors) == 0 { + errs = multierror.Append(errs, status.Error(codes.Unavailable, "there are no available nodes")) + } + continue + } + + errs = multierror.Append(errs, err) + if len(errs.Errors) >= maxFailedRequestCount { + return errs.ErrorOrNil() + } + } + + return errs.ErrorOrNil() +} diff --git a/engine/access/rpc/backend/node_selector.go b/engine/access/rpc/backend/node_selector.go new file mode 100644 index 00000000000..f90f8271b2d --- /dev/null +++ b/engine/access/rpc/backend/node_selector.go @@ -0,0 +1,70 @@ +package backend + +import ( + "fmt" + + "github.com/onflow/flow-go/model/flow" +) + +// maxNodesCnt is the maximum number of nodes that will be contacted to complete an API request. +const maxNodesCnt = 3 + +// NodeSelector is an interface that represents the ability to select node identities that the access node is trying to reach. +// It encapsulates the internal logic of node selection and provides a way to change implementations for different types +// of nodes. Implementations of this interface should define the Next method, which returns the next node identity to be +// selected. HasNext checks if there is next node available. +type NodeSelector interface { + Next() *flow.Identity + HasNext() bool +} + +// NodeSelectorFactory is a factory for creating node selectors based on factory configuration and node type. +// Supported configurations: +// circuitBreakerEnabled = true - nodes will be pseudo-randomly sampled and picked in-order. +// circuitBreakerEnabled = false - nodes will be picked from proposed list in-order without any changes. +type NodeSelectorFactory struct { + circuitBreakerEnabled bool +} + +// SelectNodes selects the configured number of node identities from the provided list of nodes +// and returns the node selector to iterate through them. +func (n *NodeSelectorFactory) SelectNodes(nodes flow.IdentityList) (NodeSelector, error) { + var err error + // If the circuit breaker is disabled, the legacy logic should be used, which selects only a specified number of nodes. + if !n.circuitBreakerEnabled { + nodes, err = nodes.Sample(maxNodesCnt) + if err != nil { + return nil, fmt.Errorf("sampling failed: %w", err) + } + } + + return NewMainNodeSelector(nodes), nil +} + +// MainNodeSelector is a specific implementation of the node selector. +// Which performs in-order node selection using fixed list of pre-defined nodes. +type MainNodeSelector struct { + nodes flow.IdentityList + index int +} + +var _ NodeSelector = (*MainNodeSelector)(nil) + +func NewMainNodeSelector(nodes flow.IdentityList) *MainNodeSelector { + return &MainNodeSelector{nodes: nodes, index: 0} +} + +// HasNext returns true if next node is available. +func (e *MainNodeSelector) HasNext() bool { + return e.index < len(e.nodes) +} + +// Next returns the next node in the selector. +func (e *MainNodeSelector) Next() *flow.Identity { + if e.index < len(e.nodes) { + next := e.nodes[e.index] + e.index++ + return next + } + return nil +} diff --git a/engine/access/rpc/backend/retry_test.go b/engine/access/rpc/backend/retry_test.go index cfa338dedc8..2189223118a 100644 --- a/engine/access/rpc/backend/retry_test.go +++ b/engine/access/rpc/backend/retry_test.go @@ -60,6 +60,8 @@ func (suite *Suite) TestTransactionRetry() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) retry := newRetry().SetBackend(backend).Activate() backend.retry = retry @@ -96,7 +98,14 @@ func (suite *Suite) TestSuccessfulTransactionsDontRetry() { block := unittest.BlockFixture() // Height needs to be at least DefaultTransactionExpiry before we start doing retries block.Header.Height = flow.DefaultTransactionExpiry + 1 - transactionBody.SetReferenceBlockID(block.ID()) + refBlock := unittest.BlockFixture() + refBlock.Header.Height = 2 + transactionBody.SetReferenceBlockID(refBlock.ID()) + + block.SetPayload( + unittest.PayloadFixture( + unittest.WithGuarantees( + unittest.CollectionGuaranteesWithCollectionIDFixture([]*flow.Collection{&collection})...))) light := collection.Light() suite.state.On("Final").Return(suite.snapshot, nil).Maybe() @@ -104,6 +113,7 @@ func (suite *Suite) TestSuccessfulTransactionsDontRetry() { suite.transactions.On("ByID", transactionBody.ID()).Return(transactionBody, nil) // collection storage returns the corresponding collection suite.collections.On("LightByTransactionID", transactionBody.ID()).Return(&light, nil) + suite.collections.On("LightByID", light.ID()).Return(&light, nil) // block storage returns the corresponding block suite.blocks.On("ByCollectionID", collection.ID()).Return(&block, nil) @@ -140,6 +150,8 @@ func (suite *Suite) TestSuccessfulTransactionsDontRetry() { nil, suite.log, DefaultSnapshotHistoryLimit, + nil, + false, ) retry := newRetry().SetBackend(backend).Activate() backend.retry = retry @@ -151,7 +163,7 @@ func (suite *Suite) TestSuccessfulTransactionsDontRetry() { // return not found to return finalized status suite.execClient.On("GetTransactionResult", ctx, &exeEventReq).Return(&exeEventResp, status.Errorf(codes.NotFound, "not found")).Once() // first call - when block under test is greater height than the sealed head, but execution node does not know about Tx - result, err := backend.GetTransactionResult(ctx, txID) + result, err := backend.GetTransactionResult(ctx, txID, flow.ZeroID, flow.ZeroID) suite.checkResponse(result, err) // status should be finalized since the sealed blocks is smaller in height diff --git a/engine/access/rpc/connection/cache.go b/engine/access/rpc/connection/cache.go new file mode 100644 index 00000000000..9aa53d3d251 --- /dev/null +++ b/engine/access/rpc/connection/cache.go @@ -0,0 +1,110 @@ +package connection + +import ( + "sync" + "time" + + lru "github.com/hashicorp/golang-lru" + "go.uber.org/atomic" + "google.golang.org/grpc" +) + +// CachedClient represents a gRPC client connection that is cached for reuse. +type CachedClient struct { + ClientConn *grpc.ClientConn + Address string + timeout time.Duration + closeRequested *atomic.Bool + wg sync.WaitGroup + mu sync.Mutex +} + +// Close closes the CachedClient connection. It marks the connection for closure and waits asynchronously for ongoing +// requests to complete before closing the connection. +func (s *CachedClient) Close() { + // Mark the connection for closure + if swapped := s.closeRequested.CompareAndSwap(false, true); !swapped { + return + } + + // If there are ongoing requests, wait for them to complete asynchronously + go func() { + s.wg.Wait() + + // Close the connection + s.ClientConn.Close() + }() +} + +// Cache represents a cache of CachedClient instances with a given maximum size. +type Cache struct { + cache *lru.Cache + size int +} + +// NewCache creates a new Cache with the specified maximum size and the underlying LRU cache. +func NewCache(cache *lru.Cache, size int) *Cache { + return &Cache{ + cache: cache, + size: size, + } +} + +// Get retrieves the CachedClient for the given address from the cache. +// It returns the CachedClient and a boolean indicating whether the entry exists in the cache. +func (c *Cache) Get(address string) (*CachedClient, bool) { + val, ok := c.cache.Get(address) + if !ok { + return nil, false + } + return val.(*CachedClient), true +} + +// GetOrAdd atomically gets the CachedClient for the given address from the cache, or adds a new one +// if none existed. +// New entries are added to the cache with their mutex locked. This ensures that the caller gets +// priority when working with the new client, allowing it to create the underlying connection. +// Clients retrieved from the cache are returned without modifying their lock. +func (c *Cache) GetOrAdd(address string, timeout time.Duration) (*CachedClient, bool) { + client := &CachedClient{} + client.mu.Lock() + + val, existed, _ := c.cache.PeekOrAdd(address, client) + if existed { + return val.(*CachedClient), true + } + + client.Address = address + client.timeout = timeout + client.closeRequested = atomic.NewBool(false) + + return client, false +} + +// Add adds a CachedClient to the cache with the given address. +// It returns a boolean indicating whether an existing entry was evicted. +func (c *Cache) Add(address string, client *CachedClient) (evicted bool) { + return c.cache.Add(address, client) +} + +// Remove removes the CachedClient entry from the cache with the given address. +// It returns a boolean indicating whether the entry was present and removed. +func (c *Cache) Remove(address string) (present bool) { + return c.cache.Remove(address) +} + +// Len returns the number of CachedClient entries in the cache. +func (c *Cache) Len() int { + return c.cache.Len() +} + +// MaxSize returns the maximum size of the cache. +func (c *Cache) MaxSize() int { + return c.size +} + +// Contains checks if the cache contains an entry with the given address. +// It returns a boolean indicating whether the address is present in the cache. +func (c *Cache) Contains(address string) (containKey bool) { + return c.cache.Contains(address) +} diff --git a/engine/access/rpc/connection/connection.go b/engine/access/rpc/connection/connection.go new file mode 100644 index 00000000000..de81319276f --- /dev/null +++ b/engine/access/rpc/connection/connection.go @@ -0,0 +1,136 @@ +package connection + +import ( + "fmt" + "io" + "net" + "time" + + "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/execution" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/module" +) + +// ConnectionFactory is an interface for creating access and execution API clients. +type ConnectionFactory interface { + GetAccessAPIClient(address string) (access.AccessAPIClient, io.Closer, error) + GetAccessAPIClientWithPort(address string, port uint) (access.AccessAPIClient, io.Closer, error) + InvalidateAccessAPIClient(address string) + GetExecutionAPIClient(address string) (execution.ExecutionAPIClient, io.Closer, error) + InvalidateExecutionAPIClient(address string) +} + +// ProxyConnectionFactory wraps an existing ConnectionFactory and allows getting API clients for a target address. +type ProxyConnectionFactory struct { + ConnectionFactory + targetAddress string +} + +func (p *ProxyConnectionFactory) GetAccessAPIClient(address string) (access.AccessAPIClient, io.Closer, error) { + return p.ConnectionFactory.GetAccessAPIClient(p.targetAddress) +} +func (p *ProxyConnectionFactory) GetExecutionAPIClient(address string) (execution.ExecutionAPIClient, io.Closer, error) { + return p.ConnectionFactory.GetExecutionAPIClient(p.targetAddress) +} + +var _ ConnectionFactory = (*ConnectionFactoryImpl)(nil) + +type ConnectionFactoryImpl struct { + CollectionGRPCPort uint + ExecutionGRPCPort uint + CollectionNodeGRPCTimeout time.Duration + ExecutionNodeGRPCTimeout time.Duration + AccessMetrics module.AccessMetrics + Log zerolog.Logger + Manager Manager +} + +// GetAccessAPIClient gets an access API client for the specified address using the default CollectionGRPCPort. +func (cf *ConnectionFactoryImpl) GetAccessAPIClient(address string) (access.AccessAPIClient, io.Closer, error) { + return cf.GetAccessAPIClientWithPort(address, cf.CollectionGRPCPort) +} + +// GetAccessAPIClientWithPort gets an access API client for the specified address and port. +func (cf *ConnectionFactoryImpl) GetAccessAPIClientWithPort(address string, port uint) (access.AccessAPIClient, io.Closer, error) { + grpcAddress, err := getGRPCAddress(address, port) + if err != nil { + return nil, nil, err + } + + conn, closer, err := cf.Manager.GetConnection(grpcAddress, cf.CollectionNodeGRPCTimeout, AccessClient) + if err != nil { + return nil, nil, err + } + + return access.NewAccessAPIClient(conn), closer, nil +} + +// InvalidateAccessAPIClient invalidates the access API client associated with the given address. +// It removes the cached client from the cache and closes the connection if a cache is used. +func (cf *ConnectionFactoryImpl) InvalidateAccessAPIClient(address string) { + if !cf.Manager.HasCache() { + return + } + + cf.Log.Debug().Str("cached_access_client_invalidated", address).Msg("invalidating cached access client") + cf.invalidateAPIClient(address, cf.CollectionGRPCPort) +} + +// GetExecutionAPIClient gets an execution API client for the specified address using the default ExecutionGRPCPort. +func (cf *ConnectionFactoryImpl) GetExecutionAPIClient(address string) (execution.ExecutionAPIClient, io.Closer, error) { + grpcAddress, err := getGRPCAddress(address, cf.ExecutionGRPCPort) + if err != nil { + return nil, nil, err + } + + conn, closer, err := cf.Manager.GetConnection(grpcAddress, cf.ExecutionNodeGRPCTimeout, ExecutionClient) + if err != nil { + return nil, nil, err + } + + return execution.NewExecutionAPIClient(conn), closer, nil +} + +// InvalidateExecutionAPIClient invalidates the execution API client associated with the given address. +// It removes the cached client from the cache and closes the connection if a cache is used. +func (cf *ConnectionFactoryImpl) InvalidateExecutionAPIClient(address string) { + if !cf.Manager.HasCache() { + return + } + + cf.Log.Debug().Str("cached_execution_client_invalidated", address).Msg("invalidating cached execution client") + cf.invalidateAPIClient(address, cf.ExecutionGRPCPort) +} + +// invalidateAPIClient invalidates the access or execution API client associated with the given address and port. +// It removes the cached client from the ConnectionsCache and closes the connection if a cache is used. +func (cf *ConnectionFactoryImpl) invalidateAPIClient(address string, port uint) { + grpcAddress, err := getGRPCAddress(address, port) + if err != nil { + panic(err) // TODO: return and handle the error + } + + if !cf.Manager.Remove(grpcAddress) { + return + } + + if cf.AccessMetrics != nil { + cf.AccessMetrics.ConnectionFromPoolInvalidated() + } +} + +// getGRPCAddress translates the flow.Identity address to the GRPC address of the node by switching the port to the +// GRPC port from the libp2p port. +func getGRPCAddress(address string, grpcPort uint) (string, error) { + // Split hostname and port + hostnameOrIP, _, err := net.SplitHostPort(address) + if err != nil { + return "", err + } + // Use the hostname from the identity list and the GRPC port number as the one passed in as an argument. + grpcAddress := fmt.Sprintf("%s:%d", hostnameOrIP, grpcPort) + + return grpcAddress, nil +} diff --git a/engine/access/rpc/connection/connection_test.go b/engine/access/rpc/connection/connection_test.go new file mode 100644 index 00000000000..a961816605e --- /dev/null +++ b/engine/access/rpc/connection/connection_test.go @@ -0,0 +1,890 @@ +package connection + +import ( + "context" + "fmt" + "net" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/sony/gobreaker" + + "go.uber.org/atomic" + "pgregory.net/rapid" + + lru "github.com/hashicorp/golang-lru" + "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/execution" + "github.com/stretchr/testify/assert" + testifymock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/engine/access/mock" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestProxyAccessAPI(t *testing.T) { + // create a collection node + cn := new(collectionNode) + cn.start(t) + defer cn.stop(t) + + req := &access.PingRequest{} + expected := &access.PingResponse{} + cn.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) + + // create the factory + connectionFactory := new(ConnectionFactoryImpl) + // set the collection grpc port + connectionFactory.CollectionGRPCPort = cn.port + // set metrics reporting + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + connectionFactory.Manager = NewManager( + nil, + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{}, + ) + + proxyConnectionFactory := ProxyConnectionFactory{ + ConnectionFactory: connectionFactory, + targetAddress: cn.listener.Addr().String(), + } + + // get a collection API client + client, conn, err := proxyConnectionFactory.GetAccessAPIClient("foo") + defer conn.Close() + assert.NoError(t, err) + + ctx := context.Background() + // make the call to the collection node + resp, err := client.Ping(ctx, req) + assert.NoError(t, err) + assert.Equal(t, resp, expected) +} + +func TestProxyExecutionAPI(t *testing.T) { + // create an execution node + en := new(executionNode) + en.start(t) + defer en.stop(t) + + req := &execution.PingRequest{} + expected := &execution.PingResponse{} + en.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) + + // create the factory + connectionFactory := new(ConnectionFactoryImpl) + // set the execution grpc port + connectionFactory.ExecutionGRPCPort = en.port + + // set metrics reporting + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + connectionFactory.Manager = NewManager( + nil, + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{}, + ) + + proxyConnectionFactory := ProxyConnectionFactory{ + ConnectionFactory: connectionFactory, + targetAddress: en.listener.Addr().String(), + } + + // get an execution API client + client, _, err := proxyConnectionFactory.GetExecutionAPIClient("foo") + assert.NoError(t, err) + + ctx := context.Background() + // make the call to the execution node + resp, err := client.Ping(ctx, req) + assert.NoError(t, err) + assert.Equal(t, resp, expected) +} + +func TestProxyAccessAPIConnectionReuse(t *testing.T) { + // create a collection node + cn := new(collectionNode) + cn.start(t) + defer cn.stop(t) + + req := &access.PingRequest{} + expected := &access.PingResponse{} + cn.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) + + // create the factory + connectionFactory := new(ConnectionFactoryImpl) + // set the collection grpc port + connectionFactory.CollectionGRPCPort = cn.port + // set the connection pool cache size + cacheSize := 1 + cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { + evictedValue.(*CachedClient).Close() + }) + connectionCache := NewCache(cache, cacheSize) + + // set metrics reporting + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + connectionFactory.Manager = NewManager( + connectionCache, + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{}, + ) + + proxyConnectionFactory := ProxyConnectionFactory{ + ConnectionFactory: connectionFactory, + targetAddress: cn.listener.Addr().String(), + } + + // get a collection API client + _, closer, err := proxyConnectionFactory.GetAccessAPIClient("foo") + assert.Equal(t, connectionCache.Len(), 1) + assert.NoError(t, err) + assert.Nil(t, closer.Close()) + + var conn *grpc.ClientConn + res, ok := connectionCache.Get(proxyConnectionFactory.targetAddress) + assert.True(t, ok) + conn = res.ClientConn + + // check if api client can be rebuilt with retrieved connection + accessAPIClient := access.NewAccessAPIClient(conn) + ctx := context.Background() + resp, err := accessAPIClient.Ping(ctx, req) + assert.NoError(t, err) + assert.Equal(t, resp, expected) +} + +func TestProxyExecutionAPIConnectionReuse(t *testing.T) { + // create an execution node + en := new(executionNode) + en.start(t) + defer en.stop(t) + + req := &execution.PingRequest{} + expected := &execution.PingResponse{} + en.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) + + // create the factory + connectionFactory := new(ConnectionFactoryImpl) + // set the execution grpc port + connectionFactory.ExecutionGRPCPort = en.port + // set the connection pool cache size + cacheSize := 5 + cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { + evictedValue.(*CachedClient).Close() + }) + connectionCache := NewCache(cache, cacheSize) + // set metrics reporting + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + connectionFactory.Manager = NewManager( + connectionCache, + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{}, + ) + + proxyConnectionFactory := ProxyConnectionFactory{ + ConnectionFactory: connectionFactory, + targetAddress: en.listener.Addr().String(), + } + + // get an execution API client + _, closer, err := proxyConnectionFactory.GetExecutionAPIClient("foo") + assert.Equal(t, connectionCache.Len(), 1) + assert.NoError(t, err) + assert.Nil(t, closer.Close()) + + var conn *grpc.ClientConn + res, ok := connectionCache.Get(proxyConnectionFactory.targetAddress) + assert.True(t, ok) + conn = res.ClientConn + + // check if api client can be rebuilt with retrieved connection + executionAPIClient := execution.NewExecutionAPIClient(conn) + ctx := context.Background() + resp, err := executionAPIClient.Ping(ctx, req) + assert.NoError(t, err) + assert.Equal(t, resp, expected) +} + +// TestExecutionNodeClientTimeout tests that the execution API client times out after the timeout duration +func TestExecutionNodeClientTimeout(t *testing.T) { + + timeout := 10 * time.Millisecond + + // create an execution node + en := new(executionNode) + en.start(t) + defer en.stop(t) + + // setup the handler mock to not respond within the timeout + req := &execution.PingRequest{} + resp := &execution.PingResponse{} + en.handler.On("Ping", testifymock.Anything, req).After(timeout+time.Second).Return(resp, nil) + + // create the factory + connectionFactory := new(ConnectionFactoryImpl) + // set the execution grpc port + connectionFactory.ExecutionGRPCPort = en.port + // set the execution grpc client timeout + connectionFactory.ExecutionNodeGRPCTimeout = timeout + // set the connection pool cache size + cacheSize := 5 + cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { + evictedValue.(*CachedClient).Close() + }) + connectionCache := NewCache(cache, cacheSize) + // set metrics reporting + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + connectionFactory.Manager = NewManager( + connectionCache, + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{}, + ) + + // create the execution API client + client, _, err := connectionFactory.GetExecutionAPIClient(en.listener.Addr().String()) + require.NoError(t, err) + + ctx := context.Background() + // make the call to the execution node + _, err = client.Ping(ctx, req) + + // assert that the client timed out + assert.Equal(t, codes.DeadlineExceeded, status.Code(err)) +} + +// TestCollectionNodeClientTimeout tests that the collection API client times out after the timeout duration +func TestCollectionNodeClientTimeout(t *testing.T) { + + timeout := 10 * time.Millisecond + + // create a collection node + cn := new(collectionNode) + cn.start(t) + defer cn.stop(t) + + // setup the handler mock to not respond within the timeout + req := &access.PingRequest{} + resp := &access.PingResponse{} + cn.handler.On("Ping", testifymock.Anything, req).After(timeout+time.Second).Return(resp, nil) + + // create the factory + connectionFactory := new(ConnectionFactoryImpl) + // set the collection grpc port + connectionFactory.CollectionGRPCPort = cn.port + // set the collection grpc client timeout + connectionFactory.CollectionNodeGRPCTimeout = timeout + // set the connection pool cache size + cacheSize := 5 + cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { + evictedValue.(*CachedClient).Close() + }) + connectionCache := NewCache(cache, cacheSize) + // set metrics reporting + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + connectionFactory.Manager = NewManager( + connectionCache, + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{}, + ) + + // create the collection API client + client, _, err := connectionFactory.GetAccessAPIClient(cn.listener.Addr().String()) + assert.NoError(t, err) + + ctx := context.Background() + // make the call to the execution node + _, err = client.Ping(ctx, req) + + // assert that the client timed out + assert.Equal(t, codes.DeadlineExceeded, status.Code(err)) +} + +// TestConnectionPoolFull tests that the LRU cache replaces connections when full +func TestConnectionPoolFull(t *testing.T) { + // create a collection node + cn1, cn2, cn3 := new(collectionNode), new(collectionNode), new(collectionNode) + cn1.start(t) + cn2.start(t) + cn3.start(t) + defer cn1.stop(t) + defer cn2.stop(t) + defer cn3.stop(t) + + req := &access.PingRequest{} + expected := &access.PingResponse{} + cn1.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) + cn2.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) + cn3.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) + + // create the factory + connectionFactory := new(ConnectionFactoryImpl) + // set the collection grpc port + connectionFactory.CollectionGRPCPort = cn1.port + // set the connection pool cache size + cacheSize := 2 + cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { + evictedValue.(*CachedClient).Close() + }) + connectionCache := NewCache(cache, cacheSize) + // set metrics reporting + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + connectionFactory.Manager = NewManager( + connectionCache, + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{}, + ) + + cn1Address := "foo1:123" + cn2Address := "foo2:123" + cn3Address := "foo3:123" + + // get a collection API client + // Create and add first client to cache + _, _, err := connectionFactory.GetAccessAPIClient(cn1Address) + assert.Equal(t, connectionCache.Len(), 1) + assert.NoError(t, err) + + // Create and add second client to cache + _, _, err = connectionFactory.GetAccessAPIClient(cn2Address) + assert.Equal(t, connectionCache.Len(), 2) + assert.NoError(t, err) + + // Peek first client from cache. "recently used"-ness will not be updated, so it will be wiped out first. + _, _, err = connectionFactory.GetAccessAPIClient(cn1Address) + assert.Equal(t, connectionCache.Len(), 2) + assert.NoError(t, err) + + // Create and add third client to cache, firs client will be removed from cache + _, _, err = connectionFactory.GetAccessAPIClient(cn3Address) + assert.Equal(t, connectionCache.Len(), 2) + assert.NoError(t, err) + + var hostnameOrIP string + hostnameOrIP, _, err = net.SplitHostPort(cn1Address) + assert.NoError(t, err) + grpcAddress1 := fmt.Sprintf("%s:%d", hostnameOrIP, connectionFactory.CollectionGRPCPort) + hostnameOrIP, _, err = net.SplitHostPort(cn2Address) + assert.NoError(t, err) + grpcAddress2 := fmt.Sprintf("%s:%d", hostnameOrIP, connectionFactory.CollectionGRPCPort) + hostnameOrIP, _, err = net.SplitHostPort(cn3Address) + assert.NoError(t, err) + grpcAddress3 := fmt.Sprintf("%s:%d", hostnameOrIP, connectionFactory.CollectionGRPCPort) + + contains1 := connectionCache.Contains(grpcAddress1) + contains2 := connectionCache.Contains(grpcAddress2) + contains3 := connectionCache.Contains(grpcAddress3) + + assert.False(t, contains1) + assert.True(t, contains2) + assert.True(t, contains3) +} + +// TestConnectionPoolStale tests that a new connection will be established if the old one cached is stale +func TestConnectionPoolStale(t *testing.T) { + // create a collection node + cn := new(collectionNode) + cn.start(t) + defer cn.stop(t) + + req := &access.PingRequest{} + expected := &access.PingResponse{} + cn.handler.On("Ping", testifymock.Anything, req).Return(expected, nil) + + // create the factory + connectionFactory := new(ConnectionFactoryImpl) + // set the collection grpc port + connectionFactory.CollectionGRPCPort = cn.port + // set the connection pool cache size + cacheSize := 5 + cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { + evictedValue.(*CachedClient).Close() + }) + connectionCache := NewCache(cache, cacheSize) + // set metrics reporting + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + connectionFactory.Manager = NewManager( + connectionCache, + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{}, + ) + + proxyConnectionFactory := ProxyConnectionFactory{ + ConnectionFactory: connectionFactory, + targetAddress: cn.listener.Addr().String(), + } + + // get a collection API client + client, _, err := proxyConnectionFactory.GetAccessAPIClient("foo") + assert.Equal(t, connectionCache.Len(), 1) + assert.NoError(t, err) + // close connection to simulate something "going wrong" with our stored connection + res, _ := connectionCache.Get(proxyConnectionFactory.targetAddress) + + connectionCache.Remove(proxyConnectionFactory.targetAddress) + res.Close() + + ctx := context.Background() + // make the call to the collection node (should fail, connection closed) + _, err = client.Ping(ctx, req) + assert.Error(t, err) + + // re-access, should replace stale connection in cache with new one + _, _, _ = proxyConnectionFactory.GetAccessAPIClient("foo") + assert.Equal(t, connectionCache.Len(), 1) + + var conn *grpc.ClientConn + res, ok := connectionCache.Get(proxyConnectionFactory.targetAddress) + assert.True(t, ok) + conn = res.ClientConn + + // check if api client can be rebuilt with retrieved connection + accessAPIClient := access.NewAccessAPIClient(conn) + ctx = context.Background() + resp, err := accessAPIClient.Ping(ctx, req) + assert.NoError(t, err) + assert.Equal(t, resp, expected) +} + +// TestExecutionNodeClientClosedGracefully tests the scenario where the execution node client is closed gracefully. +// +// Test Steps: +// - Generate a random number of requests and start goroutines to handle each request. +// - Invalidate the execution API client. +// - Wait for all goroutines to finish. +// - Verify that the number of completed requests matches the number of sent responses. +func TestExecutionNodeClientClosedGracefully(t *testing.T) { + // Add createExecNode function to recreate it each time for rapid test + createExecNode := func() (*executionNode, func()) { + en := new(executionNode) + en.start(t) + return en, func() { + en.stop(t) + } + } + + // Add rapid test, to check graceful close on different number of requests + rapid.Check(t, func(t *rapid.T) { + en, closer := createExecNode() + defer closer() + + // setup the handler mock + req := &execution.PingRequest{} + resp := &execution.PingResponse{} + respSent := atomic.NewUint64(0) + en.handler.On("Ping", testifymock.Anything, req).Run(func(_ testifymock.Arguments) { + respSent.Inc() + }).Return(resp, nil) + + // create the factory + connectionFactory := new(ConnectionFactoryImpl) + // set the execution grpc port + connectionFactory.ExecutionGRPCPort = en.port + // set the execution grpc client timeout + connectionFactory.ExecutionNodeGRPCTimeout = time.Second + // set the connection pool cache size + cacheSize := 1 + cache, _ := lru.NewWithEvict(cacheSize, func(_, evictedValue interface{}) { + evictedValue.(*CachedClient).Close() + }) + connectionCache := NewCache(cache, cacheSize) + // set metrics reporting + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + connectionFactory.Manager = NewManager( + connectionCache, + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{}, + ) + + clientAddress := en.listener.Addr().String() + // create the execution API client + client, _, err := connectionFactory.GetExecutionAPIClient(clientAddress) + assert.NoError(t, err) + + ctx := context.Background() + + // Generate random number of requests + nofRequests := rapid.IntRange(10, 100).Draw(t, "nofRequests").(int) + reqCompleted := atomic.NewUint64(0) + + var waitGroup sync.WaitGroup + + for i := 0; i < nofRequests; i++ { + waitGroup.Add(1) + + // call Ping request from different goroutines + go func() { + defer waitGroup.Done() + _, err := client.Ping(ctx, req) + + if err == nil { + reqCompleted.Inc() + } else { + require.Equalf(t, codes.Unavailable, status.Code(err), "unexpected error: %v", err) + } + }() + } + + // Close connection + connectionFactory.InvalidateExecutionAPIClient(clientAddress) + + waitGroup.Wait() + + assert.Equal(t, reqCompleted.Load(), respSent.Load()) + }) +} + +// TestExecutionEvictingCacheClients tests the eviction of cached clients in the execution flow. +// It verifies that when a client is evicted from the cache, subsequent requests are handled correctly. +// +// Test Steps: +// - Call the gRPC method Ping with a delayed response. +// - Invalidate the access API client during the Ping call and verify the expected behavior. +// - Call the gRPC method GetNetworkParameters on the client immediately after eviction and assert the expected +// error response. +// - Wait for the client state to change from "Ready" to "Shutdown", indicating that the client connection was closed. +func TestExecutionEvictingCacheClients(t *testing.T) { + // Create a new collection node for testing + cn := new(collectionNode) + cn.start(t) + defer cn.stop(t) + + // Set up mock handlers for Ping and GetNetworkParameters + pingReq := &access.PingRequest{} + pingResp := &access.PingResponse{} + cn.handler.On("Ping", testifymock.Anything, pingReq).After(time.Second).Return(pingResp, nil) + + netReq := &access.GetNetworkParametersRequest{} + netResp := &access.GetNetworkParametersResponse{} + cn.handler.On("GetNetworkParameters", testifymock.Anything).Return(netResp, nil) + + // Create the connection factory + connectionFactory := new(ConnectionFactoryImpl) + // Set the gRPC port + connectionFactory.CollectionGRPCPort = cn.port + // Set the gRPC client timeout + connectionFactory.CollectionNodeGRPCTimeout = 5 * time.Second + // Set the connection pool cache size + cacheSize := 1 + cache, err := lru.New(cacheSize) + require.NoError(t, err) + + connectionCache := NewCache(cache, cacheSize) + // set metrics reporting + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + connectionFactory.Manager = NewManager( + connectionCache, + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{}, + ) + + clientAddress := cn.listener.Addr().String() + // Create the execution API client + client, _, err := connectionFactory.GetAccessAPIClient(clientAddress) + require.NoError(t, err) + + ctx := context.Background() + + // Retrieve the cached client from the cache + result, _ := cache.Get(clientAddress) + cachedClient := result.(*CachedClient) + + // Schedule the invalidation of the access API client after a delay + time.AfterFunc(250*time.Millisecond, func() { + // Invalidate the access API client + connectionFactory.InvalidateAccessAPIClient(clientAddress) + + // Assert that the cached client is marked for closure but still waiting for previous request + assert.True(t, cachedClient.closeRequested.Load()) + assert.Equal(t, cachedClient.ClientConn.GetState(), connectivity.Ready) + + // Call a gRPC method on the client, that was already evicted + resp, err := client.GetNetworkParameters(ctx, netReq) + assert.Equal(t, status.Errorf(codes.Unavailable, "the connection to %s was closed", clientAddress), err) + assert.Nil(t, resp) + }) + + // Call a gRPC method on the client + _, err = client.Ping(ctx, pingReq) + // Check that Ping was called + cn.handler.AssertCalled(t, "Ping", testifymock.Anything, pingReq) + assert.NoError(t, err) + + // Wait for the client connection to change state from "Ready" to "Shutdown" as connection was closed. + changed := cachedClient.ClientConn.WaitForStateChange(ctx, connectivity.Ready) + assert.True(t, changed) + assert.Equal(t, connectivity.Shutdown, cachedClient.ClientConn.GetState()) + assert.Equal(t, 0, cache.Len()) +} + +// TestCircuitBreakerExecutionNode tests the circuit breaker state changes for execution nodes. +func TestCircuitBreakerExecutionNode(t *testing.T) { + requestTimeout := 500 * time.Millisecond + circuitBreakerRestoreTimeout := 1500 * time.Millisecond + + // Create an execution node for testing. + en := new(executionNode) + en.start(t) + defer en.stop(t) + + // Set up the handler mock to not respond within the requestTimeout. + req := &execution.PingRequest{} + resp := &execution.PingResponse{} + en.handler.On("Ping", testifymock.Anything, req).After(2*requestTimeout).Return(resp, nil) + + // Create the connection factory. + connectionFactory := new(ConnectionFactoryImpl) + + // Set the execution gRPC port. + connectionFactory.ExecutionGRPCPort = en.port + + // Set the execution gRPC client requestTimeout. + connectionFactory.ExecutionNodeGRPCTimeout = requestTimeout + + // Set the connection pool cache size. + cacheSize := 1 + connectionCache, _ := lru.New(cacheSize) + + connectionFactory.Manager = NewManager( + NewCache(connectionCache, cacheSize), + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{ + Enabled: true, + MaxFailures: 1, + MaxRequests: 1, + RestoreTimeout: circuitBreakerRestoreTimeout, + }, + ) + + // Set metrics reporting. + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + + // Create the execution API client. + client, _, err := connectionFactory.GetExecutionAPIClient(en.listener.Addr().String()) + require.NoError(t, err) + + ctx := context.Background() + + // Helper function to make the Ping call to the execution node and measure the duration. + callAndMeasurePingDuration := func() (time.Duration, error) { + start := time.Now() + + // Make the call to the execution node. + _, err = client.Ping(ctx, req) + en.handler.AssertCalled(t, "Ping", testifymock.Anything, req) + + return time.Since(start), err + } + + // Call and measure the duration for the first invocation. + duration, err := callAndMeasurePingDuration() + assert.Equal(t, codes.DeadlineExceeded, status.Code(err)) + assert.LessOrEqual(t, requestTimeout, duration) + + // Call and measure the duration for the second invocation (circuit breaker state is now "Open"). + duration, err = callAndMeasurePingDuration() + assert.Equal(t, gobreaker.ErrOpenState, err) + assert.Greater(t, requestTimeout, duration) + + // Reset the mock Ping for the next invocation to return response without delay + en.handler.On("Ping", testifymock.Anything, req).Unset() + en.handler.On("Ping", testifymock.Anything, req).Return(resp, nil) + + // Wait until the circuit breaker transitions to the "HalfOpen" state. + time.Sleep(circuitBreakerRestoreTimeout + (500 * time.Millisecond)) + + // Call and measure the duration for the third invocation (circuit breaker state is now "HalfOpen"). + duration, err = callAndMeasurePingDuration() + assert.Greater(t, requestTimeout, duration) + assert.Equal(t, nil, err) +} + +// TestCircuitBreakerCollectionNode tests the circuit breaker state changes for collection nodes. +func TestCircuitBreakerCollectionNode(t *testing.T) { + requestTimeout := 500 * time.Millisecond + circuitBreakerRestoreTimeout := 1500 * time.Millisecond + + // Create a collection node for testing. + cn := new(collectionNode) + cn.start(t) + defer cn.stop(t) + + // Set up the handler mock to not respond within the requestTimeout. + req := &access.PingRequest{} + resp := &access.PingResponse{} + cn.handler.On("Ping", testifymock.Anything, req).After(2*requestTimeout).Return(resp, nil) + + // Create the connection factory. + connectionFactory := new(ConnectionFactoryImpl) + + // Set the collection gRPC port. + connectionFactory.CollectionGRPCPort = cn.port + + // Set the collection gRPC client requestTimeout. + connectionFactory.CollectionNodeGRPCTimeout = requestTimeout + + // Set the connection pool cache size. + cacheSize := 1 + connectionCache, _ := lru.New(cacheSize) + + connectionFactory.Manager = NewManager( + NewCache(connectionCache, cacheSize), + unittest.Logger(), + connectionFactory.AccessMetrics, + 0, + CircuitBreakerConfig{ + Enabled: true, + MaxFailures: 1, + MaxRequests: 1, + RestoreTimeout: circuitBreakerRestoreTimeout, + }, + ) + + // Set metrics reporting. + connectionFactory.AccessMetrics = metrics.NewNoopCollector() + + // Create the collection API client. + client, _, err := connectionFactory.GetAccessAPIClient(cn.listener.Addr().String()) + assert.NoError(t, err) + + ctx := context.Background() + + // Helper function to make the Ping call to the collection node and measure the duration. + callAndMeasurePingDuration := func() (time.Duration, error) { + start := time.Now() + + // Make the call to the collection node. + _, err = client.Ping(ctx, req) + cn.handler.AssertCalled(t, "Ping", testifymock.Anything, req) + + return time.Since(start), err + } + + // Call and measure the duration for the first invocation. + duration, err := callAndMeasurePingDuration() + assert.Equal(t, codes.DeadlineExceeded, status.Code(err)) + assert.LessOrEqual(t, requestTimeout, duration) + + // Call and measure the duration for the second invocation (circuit breaker state is now "Open"). + duration, err = callAndMeasurePingDuration() + assert.Equal(t, gobreaker.ErrOpenState, err) + assert.Greater(t, requestTimeout, duration) + + // Reset the mock Ping for the next invocation to return response without delay + cn.handler.On("Ping", testifymock.Anything, req).Unset() + cn.handler.On("Ping", testifymock.Anything, req).Return(resp, nil) + + // Wait until the circuit breaker transitions to the "HalfOpen" state. + time.Sleep(circuitBreakerRestoreTimeout + (500 * time.Millisecond)) + + // Call and measure the duration for the third invocation (circuit breaker state is now "HalfOpen"). + duration, err = callAndMeasurePingDuration() + assert.Greater(t, requestTimeout, duration) + assert.Equal(t, nil, err) +} + +// node mocks a flow node that runs a GRPC server +type node struct { + server *grpc.Server + listener net.Listener + port uint +} + +func (n *node) setupNode(t *testing.T) { + n.server = grpc.NewServer() + listener, err := net.Listen("tcp4", unittest.DefaultAddress) + assert.NoError(t, err) + n.listener = listener + assert.Eventually(t, func() bool { + return !strings.HasSuffix(listener.Addr().String(), ":0") + }, time.Second*4, 10*time.Millisecond) + + _, port, err := net.SplitHostPort(listener.Addr().String()) + assert.NoError(t, err) + portAsUint, err := strconv.ParseUint(port, 10, 32) + assert.NoError(t, err) + n.port = uint(portAsUint) +} + +func (n *node) start(t *testing.T) { + // using a wait group here to ensure the goroutine has started before returning. Otherwise, + // there's a race condition where the server is sometimes stopped before it has started + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + wg.Done() + err := n.server.Serve(n.listener) + assert.NoError(t, err) + }() + unittest.RequireReturnsBefore(t, wg.Wait, 10*time.Millisecond, "could not start goroutine on time") +} + +func (n *node) stop(t *testing.T) { + if n.server != nil { + n.server.Stop() + } +} + +type executionNode struct { + node + handler *mock.ExecutionAPIServer +} + +func (en *executionNode) start(t *testing.T) { + en.setupNode(t) + handler := new(mock.ExecutionAPIServer) + execution.RegisterExecutionAPIServer(en.server, handler) + en.handler = handler + en.node.start(t) +} + +func (en *executionNode) stop(t *testing.T) { + en.node.stop(t) +} + +type collectionNode struct { + node + handler *mock.AccessAPIServer +} + +func (cn *collectionNode) start(t *testing.T) { + cn.setupNode(t) + handler := new(mock.AccessAPIServer) + access.RegisterAccessAPIServer(cn.server, handler) + cn.handler = handler + cn.node.start(t) +} + +func (cn *collectionNode) stop(t *testing.T) { + cn.node.stop(t) +} diff --git a/engine/access/rpc/connection/manager.go b/engine/access/rpc/connection/manager.go new file mode 100644 index 00000000000..018b08743c3 --- /dev/null +++ b/engine/access/rpc/connection/manager.go @@ -0,0 +1,387 @@ +package connection + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/sony/gobreaker" + + "github.com/rs/zerolog" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/module" +) + +// DefaultClientTimeout is used when making a GRPC request to a collection node or an execution node. +const DefaultClientTimeout = 3 * time.Second + +// clientType is an enumeration type used to differentiate between different types of gRPC clients. +type clientType int + +const ( + AccessClient clientType = iota + ExecutionClient +) + +type noopCloser struct{} + +func (c *noopCloser) Close() error { + return nil +} + +// Manager provides methods for getting and managing gRPC client connections. +type Manager struct { + cache *Cache + logger zerolog.Logger + metrics module.AccessMetrics + maxMsgSize uint + circuitBreakerConfig CircuitBreakerConfig +} + +// CircuitBreakerConfig is a configuration struct for the circuit breaker. +type CircuitBreakerConfig struct { + // Enabled specifies whether the circuit breaker is enabled for collection and execution API clients. + Enabled bool + // RestoreTimeout specifies the duration after which the circuit breaker will restore the connection to the client + // after closing it due to failures. + RestoreTimeout time.Duration + // MaxFailures specifies the maximum number of failed calls to the client that will cause the circuit breaker + // to close the connection. + MaxFailures uint32 + // MaxRequests specifies the maximum number of requests to check if connection restored after timeout. + MaxRequests uint32 +} + +// NewManager creates a new Manager with the specified parameters. +func NewManager( + cache *Cache, + logger zerolog.Logger, + metrics module.AccessMetrics, + maxMsgSize uint, + circuitBreakerConfig CircuitBreakerConfig, +) Manager { + return Manager{ + cache: cache, + logger: logger, + metrics: metrics, + maxMsgSize: maxMsgSize, + circuitBreakerConfig: circuitBreakerConfig, + } +} + +// GetConnection returns a gRPC client connection for the given grpcAddress and timeout. +// If a cache is used, it retrieves a cached connection, otherwise creates a new connection. +// It returns the client connection and an io.Closer to close the connection when done. +func (m *Manager) GetConnection(grpcAddress string, timeout time.Duration, clientType clientType) (*grpc.ClientConn, io.Closer, error) { + if m.cache != nil { + conn, err := m.retrieveConnection(grpcAddress, timeout, clientType) + if err != nil { + return nil, nil, err + } + return conn, &noopCloser{}, err + } + + conn, err := m.createConnection(grpcAddress, timeout, nil, clientType) + if err != nil { + return nil, nil, err + } + + return conn, io.Closer(conn), nil +} + +// Remove removes the gRPC client connection associated with the given grpcAddress from the cache. +// It returns true if the connection was removed successfully, false otherwise. +func (m *Manager) Remove(grpcAddress string) bool { + if m.cache == nil { + return false + } + + res, ok := m.cache.Get(grpcAddress) + if !ok { + return false + } + + if !m.cache.Remove(grpcAddress) { + return false + } + + // Obtain the lock here to ensure that ClientConn was initialized, avoiding a situation with a nil ClientConn. + res.mu.Lock() + defer res.mu.Unlock() + + // Close the connection only if it is successfully removed from the cache + res.Close() + return true +} + +// HasCache returns true if the Manager has a cache, false otherwise. +func (m *Manager) HasCache() bool { + return m.cache != nil +} + +// retrieveConnection retrieves the CachedClient for the given grpcAddress from the cache or adds a new one if not present. +// If the connection is already cached, it waits for the lock and returns the connection from the cache. +// Otherwise, it creates a new connection and caches it. +func (m *Manager) retrieveConnection(grpcAddress string, timeout time.Duration, clientType clientType) (*grpc.ClientConn, error) { + client, ok := m.cache.GetOrAdd(grpcAddress, timeout) + if ok { + // The client was retrieved from the cache, wait for the lock + client.mu.Lock() + if m.metrics != nil { + m.metrics.ConnectionFromPoolReused() + } + } else { + // The client is new, lock is already held + if m.metrics != nil { + m.metrics.ConnectionAddedToPool() + } + } + defer client.mu.Unlock() + + if client.ClientConn != nil && client.ClientConn.GetState() != connectivity.Shutdown { + // Return the client connection from the cache + return client.ClientConn, nil + } + + // The connection is not cached or is closed, create a new connection and cache it + conn, err := m.createConnection(grpcAddress, timeout, client, clientType) + if err != nil { + return nil, err + } + + client.ClientConn = conn + if m.metrics != nil { + m.metrics.NewConnectionEstablished() + m.metrics.TotalConnectionsInPool(uint(m.cache.Len()), uint(m.cache.MaxSize())) + } + + return client.ClientConn, nil +} + +// createConnection creates a new gRPC connection to the remote node at the given address with the specified timeout. +// If the cachedClient is not nil, it means a new entry in the cache is being created, so it's locked to give priority +// to the caller working with the new client, allowing it to create the underlying connection. +func (m *Manager) createConnection(address string, timeout time.Duration, cachedClient *CachedClient, clientType clientType) (*grpc.ClientConn, error) { + if timeout == 0 { + timeout = DefaultClientTimeout + } + + keepaliveParams := keepalive.ClientParameters{ + Time: 10 * time.Second, // How long the client will wait before sending a keepalive to the server if there is no activity. + Timeout: timeout, // How long the client will wait for a response from the keepalive before closing. + } + + // The order in which interceptors are added to the `connInterceptors` slice is important since they will be called + // in the opposite order during gRPC requests. See documentation for more info: + // https://grpc.io/blog/grpc-web-interceptor/#binding-interceptors + var connInterceptors []grpc.UnaryClientInterceptor + + if !m.circuitBreakerConfig.Enabled { + connInterceptors = append(connInterceptors, m.createClientInvalidationInterceptor(address, clientType)) + } + + connInterceptors = append(connInterceptors, createClientTimeoutInterceptor(timeout)) + + // This interceptor monitors ongoing requests before passing control to subsequent interceptors. + if cachedClient != nil { + connInterceptors = append(connInterceptors, createRequestWatcherInterceptor(cachedClient)) + } + + if m.circuitBreakerConfig.Enabled { + // If the circuit breaker interceptor is enabled, it should always be called first before passing control to + // subsequent interceptors. + connInterceptors = append(connInterceptors, m.createCircuitBreakerInterceptor()) + } + + // ClientConn's default KeepAlive on connections is indefinite, assuming the timeout isn't reached + // The connections should be safe to be persisted and reused. + // https://pkg.go.dev/google.golang.org/grpc#WithKeepaliveParams + // https://grpc.io/blog/grpc-on-http2/#keeping-connections-alive + conn, err := grpc.Dial( + address, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(m.maxMsgSize))), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithKeepaliveParams(keepaliveParams), + grpc.WithChainUnaryInterceptor(connInterceptors...), + ) + if err != nil { + return nil, fmt.Errorf("failed to connect to address %s: %w", address, err) + } + return conn, nil +} + +// createRequestWatcherInterceptor creates a request watcher interceptor to wait for unfinished requests before closing. +func createRequestWatcherInterceptor(cachedClient *CachedClient) grpc.UnaryClientInterceptor { + requestWatcherInterceptor := func( + ctx context.Context, + method string, + req interface{}, + reply interface{}, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + // Prevent new requests from being sent if the connection is marked for closure. + if cachedClient.closeRequested.Load() { + return status.Errorf(codes.Unavailable, "the connection to %s was closed", cachedClient.Address) + } + + // Increment the request counter to track ongoing requests, then decrement the request counter before returning. + cachedClient.wg.Add(1) + defer cachedClient.wg.Done() + + // Invoke the actual RPC method. + return invoker(ctx, method, req, reply, cc, opts...) + } + + return requestWatcherInterceptor +} + +// WithClientTimeoutOption is a helper function to create a GRPC dial option +// with the specified client timeout interceptor. +func WithClientTimeoutOption(timeout time.Duration) grpc.DialOption { + return grpc.WithUnaryInterceptor(createClientTimeoutInterceptor(timeout)) +} + +// createClientTimeoutInterceptor creates a client interceptor with a context that expires after the timeout. +func createClientTimeoutInterceptor(timeout time.Duration) grpc.UnaryClientInterceptor { + clientTimeoutInterceptor := func( + ctx context.Context, + method string, + req interface{}, + reply interface{}, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + // Create a context that expires after the specified timeout. + ctxWithTimeout, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // Call the remote GRPC using the short context. + err := invoker(ctxWithTimeout, method, req, reply, cc, opts...) + + return err + } + + return clientTimeoutInterceptor +} + +// createClientInvalidationInterceptor creates a client interceptor for client invalidation. It should only be created +// if the circuit breaker is disabled. If the response from the server indicates an unavailable status, it invalidates +// the corresponding client. +func (m *Manager) createClientInvalidationInterceptor( + address string, + clientType clientType, +) grpc.UnaryClientInterceptor { + if !m.circuitBreakerConfig.Enabled { + clientInvalidationInterceptor := func( + ctx context.Context, + method string, + req interface{}, + reply interface{}, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + err := invoker(ctx, method, req, reply, cc, opts...) + if status.Code(err) == codes.Unavailable { + switch clientType { + case AccessClient: + if m.Remove(address) { + m.logger.Debug().Str("cached_access_client_invalidated", address).Msg("invalidating cached access client") + if m.metrics != nil { + m.metrics.ConnectionFromPoolInvalidated() + } + } + case ExecutionClient: + if m.Remove(address) { + m.logger.Debug().Str("cached_execution_client_invalidated", address).Msg("invalidating cached execution client") + if m.metrics != nil { + m.metrics.ConnectionFromPoolInvalidated() + } + } + default: + m.logger.Info().Str("client_invalidation_interceptor", address).Msg(fmt.Sprintf("unexpected client type: %d", clientType)) + } + } + + return err + } + + return clientInvalidationInterceptor + } + + return nil +} + +// The simplified representation and description of circuit breaker pattern, that used to handle node connectivity: +// +// Circuit Open --> Circuit Half-Open --> Circuit Closed +// ^ | +// | | +// +--------------------------------------+ +// +// The "Circuit Open" state represents the circuit being open, indicating that the node is not available. +// This state is entered when the number of consecutive failures exceeds the maximum allowed failures. +// +// The "Circuit Half-Open" state represents the circuit transitioning from the open state to the half-open +// state after a configured restore timeout. In this state, the circuit allows a limited number of requests +// to test if the node has recovered. +// +// The "Circuit Closed" state represents the circuit being closed, indicating that the node is available. +// This state is initial or entered when the test requests in the half-open state succeed. + +// createCircuitBreakerInterceptor creates a client interceptor for circuit breaker functionality. It should only be +// created if the circuit breaker is enabled. All invocations will go through the circuit breaker to be tracked for +// success or failure of the call. +func (m *Manager) createCircuitBreakerInterceptor() grpc.UnaryClientInterceptor { + if m.circuitBreakerConfig.Enabled { + circuitBreaker := gobreaker.NewCircuitBreaker(gobreaker.Settings{ + // Timeout defines how long the circuit breaker will remain open before transitioning to the HalfClose state. + Timeout: m.circuitBreakerConfig.RestoreTimeout, + // ReadyToTrip returns true when the circuit breaker should trip and transition to the Open state + ReadyToTrip: func(counts gobreaker.Counts) bool { + // The number of maximum failures is checked before the circuit breaker goes to the Open state. + return counts.ConsecutiveFailures >= m.circuitBreakerConfig.MaxFailures + }, + // MaxRequests defines the max number of concurrent requests while the circuit breaker is in the HalfClosed + // state. + MaxRequests: m.circuitBreakerConfig.MaxRequests, + }) + + circuitBreakerInterceptor := func( + ctx context.Context, + method string, + req interface{}, + reply interface{}, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + // The circuit breaker integration occurs here, where all invoked calls to the node pass through the + // CircuitBreaker.Execute method. This method counts successful and failed invocations, and switches to the + // "StateOpen" when the maximum failure threshold is reached. When the circuit breaker is in the "StateOpen" + // it immediately rejects connections and returns without waiting for the call timeout. After the + // "RestoreTimeout" period elapses, the circuit breaker transitions to the "StateHalfOpen" and attempts the + // invocation again. If the invocation fails, it returns to the "StateOpen"; otherwise, it transitions to + // the "StateClosed" and handles invocations as usual. + _, err := circuitBreaker.Execute(func() (interface{}, error) { + err := invoker(ctx, method, req, reply, cc, opts...) + return nil, err + }) + return err + } + + return circuitBreakerInterceptor + } + + return nil +} diff --git a/engine/access/rpc/connection/mock/connection_factory.go b/engine/access/rpc/connection/mock/connection_factory.go new file mode 100644 index 00000000000..eebc006485d --- /dev/null +++ b/engine/access/rpc/connection/mock/connection_factory.go @@ -0,0 +1,148 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + access "github.com/onflow/flow/protobuf/go/flow/access" + + execution "github.com/onflow/flow/protobuf/go/flow/execution" + + io "io" + + mock "github.com/stretchr/testify/mock" +) + +// ConnectionFactory is an autogenerated mock type for the ConnectionFactory type +type ConnectionFactory struct { + mock.Mock +} + +// GetAccessAPIClient provides a mock function with given fields: address +func (_m *ConnectionFactory) GetAccessAPIClient(address string) (access.AccessAPIClient, io.Closer, error) { + ret := _m.Called(address) + + var r0 access.AccessAPIClient + var r1 io.Closer + var r2 error + if rf, ok := ret.Get(0).(func(string) (access.AccessAPIClient, io.Closer, error)); ok { + return rf(address) + } + if rf, ok := ret.Get(0).(func(string) access.AccessAPIClient); ok { + r0 = rf(address) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPIClient) + } + } + + if rf, ok := ret.Get(1).(func(string) io.Closer); ok { + r1 = rf(address) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(io.Closer) + } + } + + if rf, ok := ret.Get(2).(func(string) error); ok { + r2 = rf(address) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// GetAccessAPIClientWithPort provides a mock function with given fields: address, port +func (_m *ConnectionFactory) GetAccessAPIClientWithPort(address string, port uint) (access.AccessAPIClient, io.Closer, error) { + ret := _m.Called(address, port) + + var r0 access.AccessAPIClient + var r1 io.Closer + var r2 error + if rf, ok := ret.Get(0).(func(string, uint) (access.AccessAPIClient, io.Closer, error)); ok { + return rf(address, port) + } + if rf, ok := ret.Get(0).(func(string, uint) access.AccessAPIClient); ok { + r0 = rf(address, port) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(access.AccessAPIClient) + } + } + + if rf, ok := ret.Get(1).(func(string, uint) io.Closer); ok { + r1 = rf(address, port) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(io.Closer) + } + } + + if rf, ok := ret.Get(2).(func(string, uint) error); ok { + r2 = rf(address, port) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// GetExecutionAPIClient provides a mock function with given fields: address +func (_m *ConnectionFactory) GetExecutionAPIClient(address string) (execution.ExecutionAPIClient, io.Closer, error) { + ret := _m.Called(address) + + var r0 execution.ExecutionAPIClient + var r1 io.Closer + var r2 error + if rf, ok := ret.Get(0).(func(string) (execution.ExecutionAPIClient, io.Closer, error)); ok { + return rf(address) + } + if rf, ok := ret.Get(0).(func(string) execution.ExecutionAPIClient); ok { + r0 = rf(address) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(execution.ExecutionAPIClient) + } + } + + if rf, ok := ret.Get(1).(func(string) io.Closer); ok { + r1 = rf(address) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(io.Closer) + } + } + + if rf, ok := ret.Get(2).(func(string) error); ok { + r2 = rf(address) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// InvalidateAccessAPIClient provides a mock function with given fields: address +func (_m *ConnectionFactory) InvalidateAccessAPIClient(address string) { + _m.Called(address) +} + +// InvalidateExecutionAPIClient provides a mock function with given fields: address +func (_m *ConnectionFactory) InvalidateExecutionAPIClient(address string) { + _m.Called(address) +} + +type mockConstructorTestingTNewConnectionFactory interface { + mock.TestingT + Cleanup(func()) +} + +// NewConnectionFactory creates a new instance of ConnectionFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnectionFactory(t mockConstructorTestingTNewConnectionFactory) *ConnectionFactory { + mock := &ConnectionFactory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index cbe26a7daf9..eea6dc5d17c 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -7,194 +7,123 @@ import ( "net" "net/http" "sync" - "time" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - lru "github.com/hashicorp/golang-lru" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" - "google.golang.org/grpc" + "google.golang.org/grpc/credentials" - "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rpc/backend" - "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/events" + "github.com/onflow/flow-go/module/grpcserver" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/storage" ) // Config defines the configurable options for the access node server // A secure GRPC server here implies a server that presents a self-signed TLS certificate and a client that authenticates // the server via a pre-shared public key type Config struct { - UnsecureGRPCListenAddr string // the non-secure GRPC server address as ip:port - SecureGRPCListenAddr string // the secure GRPC server address as ip:port - TransportCredentials credentials.TransportCredentials // the secure GRPC credentials - HTTPListenAddr string // the HTTP web proxy address as ip:port - RESTListenAddr string // the REST server address as ip:port (if empty the REST server will not be started) - CollectionAddr string // the address of the upstream collection node - HistoricalAccessAddrs string // the list of all access nodes from previous spork - MaxMsgSize uint // GRPC max message size - ExecutionClientTimeout time.Duration // execution API GRPC client timeout - CollectionClientTimeout time.Duration // collection API GRPC client timeout - ConnectionPoolSize uint // size of the cache for storing collection and execution connections - MaxHeightRange uint // max size of height range requests - PreferredExecutionNodeIDs []string // preferred list of upstream execution node IDs - FixedExecutionNodeIDs []string // fixed list of execution node IDs to choose from if no node node ID can be chosen from the PreferredExecutionNodeIDs + UnsecureGRPCListenAddr string // the non-secure GRPC server address as ip:port + SecureGRPCListenAddr string // the secure GRPC server address as ip:port + TransportCredentials credentials.TransportCredentials // the secure GRPC credentials + HTTPListenAddr string // the HTTP web proxy address as ip:port + RESTListenAddr string // the REST server address as ip:port (if empty the REST server will not be started) + CollectionAddr string // the address of the upstream collection node + HistoricalAccessAddrs string // the list of all access nodes from previous spork + + BackendConfig backend.Config // configurable options for creating Backend + MaxMsgSize uint // GRPC max message size } // Engine exposes the server with a simplified version of the Access API. // An unsecured GRPC server (default port 9000), a secure GRPC server (default port 9001) and an HTTP Web proxy (default // port 8000) are brought up. type Engine struct { - unit *engine.Unit + component.Component + + finalizedHeaderCacheActor *events.FinalizationActor // consumes events to populate the finalized header cache + backendNotifierActor *events.FinalizationActor // consumes events to notify the backend of finalized heights + finalizedHeaderCache *events.FinalizedHeaderCache + log zerolog.Logger - backend *backend.Backend // the gRPC service implementation - unsecureGrpcServer *grpc.Server // the unsecure gRPC server - secureGrpcServer *grpc.Server // the secure gRPC server + restCollector module.RestMetrics + backend *backend.Backend // the gRPC service implementation + unsecureGrpcServer *grpcserver.GrpcServer // the unsecure gRPC server + secureGrpcServer *grpcserver.GrpcServer // the secure gRPC server httpServer *http.Server restServer *http.Server config Config chain flow.Chain - addrLock sync.RWMutex - unsecureGrpcAddress net.Addr - secureGrpcAddress net.Addr - restAPIAddress net.Addr + restHandler access.API + + addrLock sync.RWMutex + restAPIAddress net.Addr } +type Option func(*RPCEngineBuilder) // NewBuilder returns a new RPC engine builder. func NewBuilder(log zerolog.Logger, state protocol.State, config Config, - collectionRPC accessproto.AccessAPIClient, - historicalAccessNodes []accessproto.AccessAPIClient, - blocks storage.Blocks, - headers storage.Headers, - collections storage.Collections, - transactions storage.Transactions, - executionReceipts storage.ExecutionReceipts, - executionResults storage.ExecutionResults, chainID flow.ChainID, - transactionMetrics module.TransactionMetrics, accessMetrics module.AccessMetrics, - collectionGRPCPort uint, - executionGRPCPort uint, - retryEnabled bool, rpcMetricsEnabled bool, - apiRatelimits map[string]int, // the api rate limit (max calls per second) for each of the Access API e.g. Ping->100, GetTransaction->300 - apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the Access API e.g. Ping->50, GetTransaction->10 + me module.Local, + backend *backend.Backend, + restHandler access.API, + secureGrpcServer *grpcserver.GrpcServer, + unsecureGrpcServer *grpcserver.GrpcServer, ) (*RPCEngineBuilder, error) { - log = log.With().Str("engine", "rpc").Logger() - // create a GRPC server to serve GRPC clients - grpcOpts := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(int(config.MaxMsgSize)), - grpc.MaxSendMsgSize(int(config.MaxMsgSize)), - } - - var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors - // if rpc metrics is enabled, first create the grpc metrics interceptor - if rpcMetricsEnabled { - interceptors = append(interceptors, grpc_prometheus.UnaryServerInterceptor) - } - - if len(apiRatelimits) > 0 { - // create a rate limit interceptor - rateLimitInterceptor := rpc.NewRateLimiterInterceptor(log, apiRatelimits, apiBurstLimits).UnaryServerInterceptor - // append the rate limit interceptor to the list of interceptors - interceptors = append(interceptors, rateLimitInterceptor) - } - - // add the logging interceptor, ensure it is innermost wrapper - interceptors = append(interceptors, rpc.LoggingInterceptor(log)...) - - // create a chained unary interceptor - chainedInterceptors := grpc.ChainUnaryInterceptor(interceptors...) - grpcOpts = append(grpcOpts, chainedInterceptors) - - // create an unsecured grpc server - unsecureGrpcServer := grpc.NewServer(grpcOpts...) - - // create a secure server server by using the secure grpc credentials that are passed in as part of config - grpcOpts = append(grpcOpts, grpc.Creds(config.TransportCredentials)) - secureGrpcServer := grpc.NewServer(grpcOpts...) - // wrap the unsecured server with an HTTP proxy server to serve HTTP clients - httpServer := NewHTTPServer(unsecureGrpcServer, config.HTTPListenAddr) - - var cache *lru.Cache - cacheSize := config.ConnectionPoolSize - if cacheSize > 0 { - // TODO: remove this fallback after fixing issues with evictions - // It was observed that evictions cause connection errors for in flight requests. This works around - // the issue by forcing hte pool size to be greater than the number of ENs + LNs - if cacheSize < backend.DefaultConnectionPoolSize { - log.Warn().Msg("connection pool size below threshold, setting pool size to default value ") - cacheSize = backend.DefaultConnectionPoolSize - } - var err error - cache, err = lru.NewWithEvict(int(cacheSize), func(_, evictedValue interface{}) { - store := evictedValue.(*backend.CachedClient) - store.Close() - log.Debug().Str("grpc_conn_evicted", store.Address).Msg("closing grpc connection evicted from pool") - if accessMetrics != nil { - accessMetrics.ConnectionFromPoolEvicted() - } - }) - if err != nil { - return nil, fmt.Errorf("could not initialize connection pool cache: %w", err) - } - } + httpServer := newHTTPProxyServer(unsecureGrpcServer.Server) - connectionFactory := &backend.ConnectionFactoryImpl{ - CollectionGRPCPort: collectionGRPCPort, - ExecutionGRPCPort: executionGRPCPort, - CollectionNodeGRPCTimeout: config.CollectionClientTimeout, - ExecutionNodeGRPCTimeout: config.ExecutionClientTimeout, - ConnectionsCache: cache, - CacheSize: cacheSize, - MaxMsgSize: config.MaxMsgSize, - AccessMetrics: accessMetrics, - Log: log, + finalizedCache, finalizedCacheWorker, err := events.NewFinalizedHeaderCache(state) + if err != nil { + return nil, fmt.Errorf("could not create header cache: %w", err) } - backend := backend.New(state, - collectionRPC, - historicalAccessNodes, - blocks, - headers, - collections, - transactions, - executionReceipts, - executionResults, - chainID, - transactionMetrics, - connectionFactory, - retryEnabled, - config.MaxHeightRange, - config.PreferredExecutionNodeIDs, - config.FixedExecutionNodeIDs, - log, - backend.DefaultSnapshotHistoryLimit, - ) - eng := &Engine{ - log: log, - unit: engine.NewUnit(), - backend: backend, - unsecureGrpcServer: unsecureGrpcServer, - secureGrpcServer: secureGrpcServer, - httpServer: httpServer, - config: config, - chain: chainID.Chain(), - } - - builder := NewRPCEngineBuilder(eng) + finalizedHeaderCache: finalizedCache, + finalizedHeaderCacheActor: finalizedCache.FinalizationActor, + log: log, + backend: backend, + unsecureGrpcServer: unsecureGrpcServer, + secureGrpcServer: secureGrpcServer, + httpServer: httpServer, + config: config, + chain: chainID.Chain(), + restCollector: accessMetrics, + restHandler: restHandler, + } + backendNotifierActor, backendNotifierWorker := events.NewFinalizationActor(eng.notifyBackendOnBlockFinalized) + eng.backendNotifierActor = backendNotifierActor + + eng.Component = component.NewComponentManagerBuilder(). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + <-secureGrpcServer.Done() + }). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + <-unsecureGrpcServer.Done() + }). + AddWorker(eng.serveGRPCWebProxyWorker). + AddWorker(eng.serveREST). + AddWorker(finalizedCacheWorker). + AddWorker(backendNotifierWorker). + AddWorker(eng.shutdownWorker). + Build() + + builder := NewRPCEngineBuilder(eng, me, finalizedCache) if rpcMetricsEnabled { builder.WithMetrics() } @@ -202,155 +131,91 @@ func NewBuilder(log zerolog.Logger, return builder, nil } -// Ready returns a ready channel that is closed once the engine has fully -// started. The RPC engine is ready when the gRPC server has successfully -// started. -func (e *Engine) Ready() <-chan struct{} { - e.unit.Launch(e.serveUnsecureGRPC) - e.unit.Launch(e.serveSecureGRPC) - e.unit.Launch(e.serveGRPCWebProxy) - if e.config.RESTListenAddr != "" { - e.unit.Launch(e.serveREST) - } - return e.unit.Ready() +// shutdownWorker is a worker routine which shuts down all servers when the context is cancelled. +func (e *Engine) shutdownWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + <-ctx.Done() + e.shutdown() } -// Done returns a done channel that is closed once the engine has fully stopped. -// It sends a signal to stop the gRPC server, then closes the channel. -func (e *Engine) Done() <-chan struct{} { - return e.unit.Done( - e.unsecureGrpcServer.GracefulStop, - e.secureGrpcServer.GracefulStop, - func() { - err := e.httpServer.Shutdown(context.Background()) - if err != nil { - e.log.Error().Err(err).Msg("error stopping http server") - } - }, - func() { - if e.restServer != nil { - err := e.restServer.Shutdown(context.Background()) - if err != nil { - e.log.Error().Err(err).Msg("error stopping http REST server") - } - } - }) -} +// shutdown sequentially shuts down all servers managed by this engine. +// Errors which occur while shutting down a server are logged and otherwise ignored. +func (e *Engine) shutdown() { + // use unbounded context, rely on shutdown logic to have timeout + ctx := context.Background() -// SubmitLocal submits an event originating on the local node. -func (e *Engine) SubmitLocal(event interface{}) { - e.unit.Launch(func() { - err := e.process(event) + err := e.httpServer.Shutdown(ctx) + if err != nil { + e.log.Error().Err(err).Msg("error stopping http server") + } + if e.restServer != nil { + err := e.restServer.Shutdown(ctx) if err != nil { - e.log.Error().Err(err).Msg("could not process submitted event") + e.log.Error().Err(err).Msg("error stopping http REST server") } - }) + } } -func (e *Engine) UnsecureGRPCAddress() net.Addr { - e.addrLock.RLock() - defer e.addrLock.RUnlock() - return e.unsecureGrpcAddress +// OnFinalizedBlock responds to block finalization events. +func (e *Engine) OnFinalizedBlock(block *model.Block) { + e.finalizedHeaderCacheActor.OnFinalizedBlock(block) + e.backendNotifierActor.OnFinalizedBlock(block) } -func (e *Engine) SecureGRPCAddress() net.Addr { - e.addrLock.RLock() - defer e.addrLock.RUnlock() - return e.secureGrpcAddress +// notifyBackendOnBlockFinalized is invoked by the FinalizationActor when a new block is finalized. +// It notifies the backend of the newly finalized block. +func (e *Engine) notifyBackendOnBlockFinalized(_ *model.Block) error { + finalizedHeader := e.finalizedHeaderCache.Get() + e.backend.NotifyFinalizedBlockHeight(finalizedHeader.Height) + return nil } +// RestApiAddress returns the listen address of the REST API server. +// Guaranteed to be non-nil after Engine.Ready is closed. func (e *Engine) RestApiAddress() net.Addr { e.addrLock.RLock() defer e.addrLock.RUnlock() return e.restAPIAddress } -// process processes the given ingestion engine event. Events that are given -// to this function originate within the expulsion engine on the node with the -// given origin ID. -func (e *Engine) process(event interface{}) error { - switch entity := event.(type) { - case *flow.Block: - e.backend.NotifyFinalizedBlockHeight(entity.Header.Height) - return nil - default: - return fmt.Errorf("invalid event type (%T)", event) - } -} - -// serveUnsecureGRPC starts the unsecure gRPC server -// When this function returns, the server is considered ready. -func (e *Engine) serveUnsecureGRPC() { - - e.log.Info().Str("grpc_address", e.config.UnsecureGRPCListenAddr).Msg("starting grpc server on address") - - l, err := net.Listen("tcp", e.config.UnsecureGRPCListenAddr) - if err != nil { - e.log.Err(err).Msg("failed to start the grpc server") - return - } - - // save the actual address on which we are listening (may be different from e.config.UnsecureGRPCListenAddr if not port - // was specified) - e.addrLock.Lock() - e.unsecureGrpcAddress = l.Addr() - e.addrLock.Unlock() - - e.log.Debug().Str("unsecure_grpc_address", e.unsecureGrpcAddress.String()).Msg("listening on port") - - err = e.unsecureGrpcServer.Serve(l) // blocking call - if err != nil { - e.log.Fatal().Err(err).Msg("fatal error in unsecure grpc server") - } -} - -// serveSecureGRPC starts the secure gRPC server -// When this function returns, the server is considered ready. -func (e *Engine) serveSecureGRPC() { - - e.log.Info().Str("secure_grpc_address", e.config.SecureGRPCListenAddr).Msg("starting grpc server on address") +// serveGRPCWebProxyWorker is a worker routine which starts the gRPC web proxy server. +func (e *Engine) serveGRPCWebProxyWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + log := e.log.With().Str("http_proxy_address", e.config.HTTPListenAddr).Logger() + log.Info().Msg("starting http proxy server on address") - l, err := net.Listen("tcp", e.config.SecureGRPCListenAddr) + l, err := net.Listen("tcp", e.config.HTTPListenAddr) if err != nil { - e.log.Err(err).Msg("failed to start the grpc server") + e.log.Err(err).Msg("failed to start the grpc web proxy server") + ctx.Throw(err) return } + ready() - e.addrLock.Lock() - e.secureGrpcAddress = l.Addr() - e.addrLock.Unlock() - - e.log.Debug().Str("secure_grpc_address", e.secureGrpcAddress.String()).Msg("listening on port") - - err = e.secureGrpcServer.Serve(l) // blocking call + err = e.httpServer.Serve(l) // blocking call if err != nil { - e.log.Fatal().Err(err).Msg("fatal error in secure grpc server") + if errors.Is(err, http.ErrServerClosed) { + return + } + log.Err(err).Msg("fatal error in grpc web proxy server") + ctx.Throw(err) } } -// serveGRPCWebProxy starts the gRPC web proxy server -func (e *Engine) serveGRPCWebProxy() { - log := e.log.With().Str("http_proxy_address", e.config.HTTPListenAddr).Logger() - - log.Info().Msg("starting http proxy server on address") - - err := e.httpServer.ListenAndServe() - if errors.Is(err, http.ErrServerClosed) { +// serveREST is a worker routine which starts the HTTP REST server. +// The ready callback is called after the server address is bound and set. +func (e *Engine) serveREST(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + if e.config.RESTListenAddr == "" { + e.log.Debug().Msg("no REST API address specified - not starting the server") + ready() return } - if err != nil { - e.log.Err(err).Msg("failed to start the http proxy server") - } -} - -// serveREST starts the HTTP REST server -func (e *Engine) serveREST() { e.log.Info().Str("rest_api_address", e.config.RESTListenAddr).Msg("starting REST server on address") - r, err := rest.NewServer(e.backend, e.config.RESTListenAddr, e.log, e.chain) + r, err := rest.NewServer(e.restHandler, e.config.RESTListenAddr, e.log, e.chain, e.restCollector) if err != nil { e.log.Err(err).Msg("failed to initialize the REST server") + ctx.Throw(err) return } e.restServer = r @@ -358,6 +223,7 @@ func (e *Engine) serveREST() { l, err := net.Listen("tcp", e.config.RESTListenAddr) if err != nil { e.log.Err(err).Msg("failed to start the REST server") + ctx.Throw(err) return } @@ -366,12 +232,14 @@ func (e *Engine) serveREST() { e.addrLock.Unlock() e.log.Debug().Str("rest_api_address", e.restAPIAddress.String()).Msg("listening on port") + ready() err = e.restServer.Serve(l) // blocking call if err != nil { if errors.Is(err, http.ErrServerClosed) { return } - e.log.Error().Err(err).Msg("fatal error in REST server") + e.log.Err(err).Msg("fatal error in REST server") + ctx.Throw(err) } } diff --git a/engine/access/rpc/engine_builder.go b/engine/access/rpc/engine_builder.go index 97fa875cef9..370f3d0fff4 100644 --- a/engine/access/rpc/engine_builder.go +++ b/engine/access/rpc/engine_builder.go @@ -4,32 +4,38 @@ import ( "fmt" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" - legacyaccessproto "github.com/onflow/flow/protobuf/go/flow/legacy/access" "github.com/onflow/flow-go/access" legacyaccess "github.com/onflow/flow-go/access/legacy" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/module" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" + legacyaccessproto "github.com/onflow/flow/protobuf/go/flow/legacy/access" ) type RPCEngineBuilder struct { *Engine + me module.Local + finalizedHeaderCache module.FinalizedHeaderCache // optional parameters, only one can be set during build phase signerIndicesDecoder hotstuff.BlockSignerDecoder - handler accessproto.AccessAPIServer // Use the parent interface instead of implementation, so that we can assign it to proxy. + rpcHandler accessproto.AccessAPIServer // Use the parent interface instead of implementation, so that we can assign it to proxy. } // NewRPCEngineBuilder helps to build a new RPC engine. -func NewRPCEngineBuilder(engine *Engine) *RPCEngineBuilder { +func NewRPCEngineBuilder(engine *Engine, me module.Local, finalizedHeaderCache module.FinalizedHeaderCache) *RPCEngineBuilder { // the default handler will use the engine.backend implementation return &RPCEngineBuilder{ - Engine: engine, + Engine: engine, + me: me, + finalizedHeaderCache: finalizedHeaderCache, } } -func (builder *RPCEngineBuilder) Handler() accessproto.AccessAPIServer { - return builder.handler +func (builder *RPCEngineBuilder) RpcHandler() accessproto.AccessAPIServer { + return builder.rpcHandler } // WithBlockSignerDecoder specifies that signer indices in block headers should be translated @@ -45,15 +51,15 @@ func (builder *RPCEngineBuilder) WithBlockSignerDecoder(signerIndicesDecoder hot return builder } -// WithNewHandler specifies that the given `AccessAPIServer` should be used for serving API queries. +// WithRpcHandler specifies that the given `AccessAPIServer` should be used for serving API queries. // Caution: // you can inject either a `BlockSignerDecoder` (via method `WithBlockSignerDecoder`) -// or an `AccessAPIServer` (via method `WithNewHandler`); but not both. If both are +// or an `AccessAPIServer` (via method `WithRpcHandler`); but not both. If both are // specified, the builder will error during the build step. // // Returns self-reference for chaining. -func (builder *RPCEngineBuilder) WithNewHandler(handler accessproto.AccessAPIServer) *RPCEngineBuilder { - builder.handler = handler +func (builder *RPCEngineBuilder) WithRpcHandler(handler accessproto.AccessAPIServer) *RPCEngineBuilder { + builder.rpcHandler = handler return builder } @@ -62,11 +68,11 @@ func (builder *RPCEngineBuilder) WithNewHandler(handler accessproto.AccessAPISer func (builder *RPCEngineBuilder) WithLegacy() *RPCEngineBuilder { // Register legacy gRPC handlers for backwards compatibility, to be removed at a later date legacyaccessproto.RegisterAccessAPIServer( - builder.unsecureGrpcServer, + builder.unsecureGrpcServer.Server, legacyaccess.NewHandler(builder.backend, builder.chain), ) legacyaccessproto.RegisterAccessAPIServer( - builder.secureGrpcServer, + builder.secureGrpcServer.Server, legacyaccess.NewHandler(builder.backend, builder.chain), ) return builder @@ -77,24 +83,24 @@ func (builder *RPCEngineBuilder) WithLegacy() *RPCEngineBuilder { func (builder *RPCEngineBuilder) WithMetrics() *RPCEngineBuilder { // Not interested in legacy metrics, so initialize here grpc_prometheus.EnableHandlingTimeHistogram() - grpc_prometheus.Register(builder.unsecureGrpcServer) - grpc_prometheus.Register(builder.secureGrpcServer) + grpc_prometheus.Register(builder.unsecureGrpcServer.Server) + grpc_prometheus.Register(builder.secureGrpcServer.Server) return builder } func (builder *RPCEngineBuilder) Build() (*Engine, error) { - if builder.signerIndicesDecoder != nil && builder.handler != nil { + if builder.signerIndicesDecoder != nil && builder.rpcHandler != nil { return nil, fmt.Errorf("only BlockSignerDecoder (via method `WithBlockSignerDecoder`) or AccessAPIServer (via method `WithNewHandler`) can be specified but not both") } - handler := builder.handler - if handler == nil { + rpcHandler := builder.rpcHandler + if rpcHandler == nil { if builder.signerIndicesDecoder == nil { - handler = access.NewHandler(builder.Engine.backend, builder.Engine.chain) + rpcHandler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me) } else { - handler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, access.WithBlockSignerDecoder(builder.signerIndicesDecoder)) + rpcHandler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me, access.WithBlockSignerDecoder(builder.signerIndicesDecoder)) } } - accessproto.RegisterAccessAPIServer(builder.unsecureGrpcServer, handler) - accessproto.RegisterAccessAPIServer(builder.secureGrpcServer, handler) + accessproto.RegisterAccessAPIServer(builder.unsecureGrpcServer.Server, rpcHandler) + accessproto.RegisterAccessAPIServer(builder.secureGrpcServer.Server, rpcHandler) return builder.Engine, nil } diff --git a/engine/access/rpc/http_server.go b/engine/access/rpc/http_server.go index feca4d2d1eb..036361a9ad4 100644 --- a/engine/access/rpc/http_server.go +++ b/engine/access/rpc/http_server.go @@ -27,23 +27,18 @@ var defaultHTTPHeaders = []HTTPHeader{ }, } -// NewHTTPServer creates and intializes a new HTTP GRPC proxy server -func NewHTTPServer( - grpcServer *grpc.Server, - address string, -) *http.Server { +// newHTTPProxyServer creates a new HTTP GRPC proxy server. +func newHTTPProxyServer(grpcServer *grpc.Server) *http.Server { wrappedServer := grpcweb.WrapServer( grpcServer, grpcweb.WithOriginFunc(func(origin string) bool { return true }), ) - mux := http.NewServeMux() - // register gRPC HTTP proxy + mux := http.NewServeMux() mux.Handle("/", wrappedHandler(wrappedServer, defaultHTTPHeaders)) httpServer := &http.Server{ - Addr: address, Handler: mux, } diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index 59f292cf80c..8ff5695c3c6 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -8,18 +8,22 @@ import ( "testing" "time" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" accessmock "github.com/onflow/flow-go/engine/access/mock" + "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/grpcserver" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network" @@ -27,6 +31,8 @@ import ( storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/grpcutils" "github.com/onflow/flow-go/utils/unittest" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) type RateLimitTestSuite struct { @@ -56,6 +62,13 @@ type RateLimitTestSuite struct { // test rate limit rateLimit int burstLimit int + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer } func (suite *RateLimitTestSuite) SetupTest() { @@ -96,6 +109,14 @@ func (suite *RateLimitTestSuite) SetupTest() { HTTPListenAddr: unittest.DefaultAddress, } + // generate a server certificate that will be served by the GRPC server + networkingKey := unittest.NetworkingPrivKeyFixture() + x509Certificate, err := grpcutils.X509Certificate(networkingKey) + assert.NoError(suite.T(), err) + tlsConfig := grpcutils.DefaultServerTLSConfig(x509Certificate) + // set the transport credentials for the server to use + config.TransportCredentials = credentials.NewTLS(tlsConfig) + // set the rate limit to test with suite.rateLimit = 2 // set the burst limit to test with @@ -109,32 +130,92 @@ func (suite *RateLimitTestSuite) SetupTest() { "Ping": suite.rateLimit, } - rpcEngBuilder, err := NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, nil, - nil, suite.chainID, suite.metrics, suite.metrics, 0, 0, false, false, apiRateLimt, apiBurstLimt) - assert.NoError(suite.T(), err) + suite.secureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.SecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + apiRateLimt, + apiBurstLimt, + grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() + + suite.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.UnsecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + apiRateLimt, + apiBurstLimt).Build() + + block := unittest.BlockHeaderFixture() + suite.snapshot.On("Head").Return(block, nil) + + backend := backend.New( + suite.state, + suite.collClient, + nil, + suite.blocks, + suite.headers, + suite.collections, + suite.transactions, + nil, + nil, + suite.chainID, + suite.metrics, + nil, + false, + 0, + nil, + nil, + suite.log, + 0, + nil, + false) + + rpcEngBuilder, err := NewBuilder( + suite.log, + suite.state, + config, + suite.chainID, + suite.metrics, + false, + suite.me, + backend, + backend, + suite.secureGrpcServer, + suite.unsecureGrpcServer) + + require.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() - assert.NoError(suite.T(), err) - unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second) + require.NoError(suite.T(), err) + suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + + suite.rpcEng.Start(suite.ctx) + + suite.secureGrpcServer.Start(suite.ctx) + suite.unsecureGrpcServer.Start(suite.ctx) + + // wait for the servers to startup + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) - // wait for the server to startup - assert.Eventually(suite.T(), func() bool { - return suite.rpcEng.UnsecureGRPCAddress() != nil - }, 5*time.Second, 10*time.Millisecond) + // wait for the engine to startup + unittest.RequireCloseBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second, "engine not ready at startup") // create the access api client - suite.client, suite.closer, err = accessAPIClient(suite.rpcEng.UnsecureGRPCAddress().String()) - assert.NoError(suite.T(), err) + suite.client, suite.closer, err = accessAPIClient(suite.unsecureGrpcServer.GRPCAddress().String()) + require.NoError(suite.T(), err) } func (suite *RateLimitTestSuite) TearDownTest() { + if suite.cancel != nil { + suite.cancel() + } // close the client if suite.closer != nil { suite.closer.Close() } - // close the server - if suite.rpcEng != nil { - unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) - } + // close servers + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Done(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Done(), 2*time.Second) } func TestRateLimit(t *testing.T) { diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index 66933a15dc7..783ed0d3110 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -7,18 +7,23 @@ import ( "testing" "time" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow-go/crypto" accessmock "github.com/onflow/flow-go/engine/access/mock" "github.com/onflow/flow-go/engine/access/rpc" + "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/grpcserver" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network" @@ -51,6 +56,13 @@ type SecureGRPCTestSuite struct { collections *storagemock.Collections transactions *storagemock.Transactions receipts *storagemock.ExecutionReceipts + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer } func (suite *SecureGRPCTestSuite) SetupTest() { @@ -101,17 +113,82 @@ func (suite *SecureGRPCTestSuite) SetupTest() { // save the public key to use later in tests later suite.publicKey = networkingKey.PublicKey() - rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, nil, - nil, suite.chainID, suite.metrics, suite.metrics, 0, 0, false, false, nil, nil) + suite.secureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.SecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + nil, + nil, + grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() + + suite.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.UnsecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + nil, + nil).Build() + + block := unittest.BlockHeaderFixture() + suite.snapshot.On("Head").Return(block, nil) + + backend := backend.New( + suite.state, + suite.collClient, + nil, + suite.blocks, + suite.headers, + suite.collections, + suite.transactions, + nil, + nil, + suite.chainID, + suite.metrics, + nil, + false, + 0, + nil, + nil, + suite.log, + 0, + nil, + false) + + rpcEngBuilder, err := rpc.NewBuilder( + suite.log, + suite.state, + config, + suite.chainID, + suite.metrics, + false, + suite.me, + backend, + backend, + suite.secureGrpcServer, + suite.unsecureGrpcServer, + ) assert.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() assert.NoError(suite.T(), err) + suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + + suite.rpcEng.Start(suite.ctx) + + suite.secureGrpcServer.Start(suite.ctx) + suite.unsecureGrpcServer.Start(suite.ctx) + + // wait for the servers to startup + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) + + // wait for the engine to startup unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second) +} - // wait for the server to startup - assert.Eventually(suite.T(), func() bool { - return suite.rpcEng.SecureGRPCAddress() != nil - }, 5*time.Second, 10*time.Millisecond) +func (suite *SecureGRPCTestSuite) TearDownTest() { + suite.cancel() + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Done(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Done(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) } func TestSecureGRPC(t *testing.T) { @@ -142,13 +219,19 @@ func (suite *SecureGRPCTestSuite) TestAPICallUsingSecureGRPC() { _, err := client.Ping(ctx, req) assert.Error(suite.T(), err) }) -} -func (suite *SecureGRPCTestSuite) TearDownTest() { - // close the server - if suite.rpcEng != nil { - unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) - } + suite.Run("happy path - connection fails, unsecure client can not get info from secure server connection", func() { + conn, err := grpc.Dial( + suite.secureGrpcServer.GRPCAddress().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + assert.NoError(suite.T(), err) + + client := accessproto.NewAccessAPIClient(conn) + closer := io.Closer(conn) + defer closer.Close() + + _, err = client.Ping(ctx, req) + assert.Error(suite.T(), err) + }) } // secureGRPCClient creates a secure GRPC client using the given public key @@ -157,7 +240,7 @@ func (suite *SecureGRPCTestSuite) secureGRPCClient(publicKey crypto.PublicKey) ( assert.NoError(suite.T(), err) conn, err := grpc.Dial( - suite.rpcEng.SecureGRPCAddress().String(), + suite.secureGrpcServer.GRPCAddress().String(), grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) assert.NoError(suite.T(), err) diff --git a/engine/access/state_stream/backend.go b/engine/access/state_stream/backend.go index 00400728915..33c5e18cb77 100644 --- a/engine/access/state_stream/backend.go +++ b/engine/access/state_stream/backend.go @@ -12,11 +12,11 @@ import ( "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/counters" "github.com/onflow/flow-go/module/executiondatasync/execution_data" - herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" + "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/utils/logging" ) const ( @@ -29,9 +29,13 @@ const ( // DefaultSendTimeout is the default timeout for sending a message to the client. After the timeout // expires, the connection is closed. DefaultSendTimeout = 30 * time.Second + + // DefaultResponseLimit is default max responses per second allowed on a stream. After exceeding + // the limit, the stream is paused until more capacity is available. + DefaultResponseLimit = float64(0) ) -type GetExecutionDataFunc func(context.Context, flow.Identifier) (*execution_data.BlockExecutionDataEntity, error) +type GetExecutionDataFunc func(context.Context, uint64) (*execution_data.BlockExecutionDataEntity, error) type GetStartHeightFunc func(flow.Identifier, uint64) (uint64, error) type API interface { @@ -44,14 +48,20 @@ type StateStreamBackend struct { ExecutionDataBackend EventsBackend - log zerolog.Logger - state protocol.State - headers storage.Headers - seals storage.Seals - results storage.ExecutionResults - execDataStore execution_data.ExecutionDataStore - execDataCache *herocache.Cache - broadcaster *engine.Broadcaster + log zerolog.Logger + state protocol.State + headers storage.Headers + seals storage.Seals + results storage.ExecutionResults + execDataStore execution_data.ExecutionDataStore + execDataCache *cache.ExecutionDataCache + broadcaster *engine.Broadcaster + rootBlockHeight uint64 + rootBlockID flow.Identifier + + // highestHeight contains the highest consecutive block height for which we have received a + // new Execution Data notification. + highestHeight counters.StrictMonotonousCounter } func New( @@ -62,20 +72,31 @@ func New( seals storage.Seals, results storage.ExecutionResults, execDataStore execution_data.ExecutionDataStore, - execDataCache *herocache.Cache, + execDataCache *cache.ExecutionDataCache, broadcaster *engine.Broadcaster, + rootHeight uint64, + highestAvailableHeight uint64, ) (*StateStreamBackend, error) { logger := log.With().Str("module", "state_stream_api").Logger() + // cache the root block height and ID for runtime lookups. + rootBlockID, err := headers.BlockIDByHeight(rootHeight) + if err != nil { + return nil, fmt.Errorf("could not get root block ID: %w", err) + } + b := &StateStreamBackend{ - log: logger, - state: state, - headers: headers, - seals: seals, - results: results, - execDataStore: execDataStore, - execDataCache: execDataCache, - broadcaster: broadcaster, + log: logger, + state: state, + headers: headers, + seals: seals, + results: results, + execDataStore: execDataStore, + execDataCache: execDataCache, + broadcaster: broadcaster, + rootBlockHeight: rootHeight, + rootBlockID: rootBlockID, + highestHeight: counters.NewMonotonousCounter(highestAvailableHeight), } b.ExecutionDataBackend = ExecutionDataBackend{ @@ -83,6 +104,7 @@ func New( headers: headers, broadcaster: broadcaster, sendTimeout: config.ClientSendTimeout, + responseLimit: config.ResponseLimit, sendBufferSize: int(config.ClientSendBufferSize), getExecutionData: b.getExecutionData, getStartHeight: b.getStartHeight, @@ -90,9 +112,9 @@ func New( b.EventsBackend = EventsBackend{ log: logger, - headers: headers, broadcaster: broadcaster, sendTimeout: config.ClientSendTimeout, + responseLimit: config.ResponseLimit, sendBufferSize: int(config.ClientSendBufferSize), getExecutionData: b.getExecutionData, getStartHeight: b.getStartHeight, @@ -101,37 +123,23 @@ func New( return b, nil } -func (b *StateStreamBackend) getExecutionData(ctx context.Context, blockID flow.Identifier) (*execution_data.BlockExecutionDataEntity, error) { - if cached, ok := b.execDataCache.ByID(blockID); ok { - b.log.Trace(). - Hex("block_id", logging.ID(blockID)). - Msg("execution data cache hit") - return cached.(*execution_data.BlockExecutionDataEntity), nil +// getExecutionData returns the execution data for the given block height. +// Expected errors during normal operation: +// - storage.ErrNotFound or execution_data.BlobNotFoundError: execution data for the given block height is not available. +func (b *StateStreamBackend) getExecutionData(ctx context.Context, height uint64) (*execution_data.BlockExecutionDataEntity, error) { + // fail early if no notification has been received for the given block height. + // note: it's possible for the data to exist in the data store before the notification is + // received. this ensures a consistent view is available to all streams. + if height > b.highestHeight.Value() { + return nil, fmt.Errorf("execution data for block %d is not available yet: %w", height, storage.ErrNotFound) } - b.log.Trace(). - Hex("block_id", logging.ID(blockID)). - Msg("execution data cache miss") - seal, err := b.seals.FinalizedSealForBlock(blockID) + execData, err := b.execDataCache.ByHeight(ctx, height) if err != nil { - return nil, fmt.Errorf("could not get finalized seal for block: %w", err) + return nil, fmt.Errorf("could not get execution data for block %d: %w", height, err) } - result, err := b.results.ByID(seal.ResultID) - if err != nil { - return nil, fmt.Errorf("could not get execution result (id: %s): %w", seal.ResultID, err) - } - - execData, err := b.execDataStore.GetExecutionData(ctx, result.ExecutionDataID) - if err != nil { - return nil, fmt.Errorf("could not get execution data (id: %s): %w", result.ExecutionDataID, err) - } - - blockExecData := execution_data.NewBlockExecutionDataEntity(result.ExecutionDataID, execData) - - b.execDataCache.Add(blockID, blockExecData) - - return blockExecData, nil + return execData, nil } // getStartHeight returns the start height to use when searching. @@ -144,7 +152,13 @@ func (b *StateStreamBackend) getStartHeight(startBlockID flow.Identifier, startH return 0, status.Errorf(codes.InvalidArgument, "only one of start block ID and start height may be provided") } - // first, if a start block ID is provided, use that + // if the start block is the root block, there will not be an execution data. skip it and + // begin from the next block. + // Note: we can skip the block lookup since it was already done in the constructor + if startBlockID == b.rootBlockID || startHeight == b.rootBlockHeight { + return b.rootBlockHeight + 1, nil + } + // invalid or missing block IDs will result in an error if startBlockID != flow.ZeroID { header, err := b.headers.ByBlockID(startBlockID) @@ -154,9 +168,12 @@ func (b *StateStreamBackend) getStartHeight(startBlockID flow.Identifier, startH return header.Height, nil } - // next, if the start height is provided, use that - // heights that are in the future or before the root block will result in an error + // heights that have not been indexed yet will result in an error if startHeight > 0 { + if startHeight < b.rootBlockHeight { + return 0, status.Errorf(codes.InvalidArgument, "start height must be greater than or equal to the root height %d", b.rootBlockHeight) + } + header, err := b.headers.ByHeight(startHeight) if err != nil { return 0, rpc.ConvertStorageError(fmt.Errorf("could not get header for height %d: %w", startHeight, err)) @@ -171,3 +188,8 @@ func (b *StateStreamBackend) getStartHeight(startBlockID flow.Identifier, startH } return header.Height, nil } + +// SetHighestHeight sets the highest height for which execution data is available. +func (b *StateStreamBackend) setHighestHeight(height uint64) bool { + return b.highestHeight.Set(height) +} diff --git a/engine/access/state_stream/backend_events.go b/engine/access/state_stream/backend_events.go index 0f6472f59f8..2691ef5e7d0 100644 --- a/engine/access/state_stream/backend_events.go +++ b/engine/access/state_stream/backend_events.go @@ -6,11 +6,9 @@ import ( "time" "github.com/rs/zerolog" - "google.golang.org/grpc/status" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/logging" ) @@ -22,9 +20,9 @@ type EventsResponse struct { type EventsBackend struct { log zerolog.Logger - headers storage.Headers broadcaster *engine.Broadcaster sendTimeout time.Duration + responseLimit float64 sendBufferSize int getExecutionData GetExecutionDataFunc @@ -34,33 +32,21 @@ type EventsBackend struct { func (b EventsBackend) SubscribeEvents(ctx context.Context, startBlockID flow.Identifier, startHeight uint64, filter EventFilter) Subscription { nextHeight, err := b.getStartHeight(startBlockID, startHeight) if err != nil { - sub := NewSubscription(b.sendBufferSize) - if st, ok := status.FromError(err); ok { - sub.Fail(status.Errorf(st.Code(), "could not get start height: %s", st.Message())) - return sub - } - - sub.Fail(fmt.Errorf("could not get start height: %w", err)) - return sub + return NewFailedSubscription(err, "could not get start height") } sub := NewHeightBasedSubscription(b.sendBufferSize, nextHeight, b.getResponseFactory(filter)) - go NewStreamer(b.log, b.broadcaster, b.sendTimeout, sub).Stream(ctx) + go NewStreamer(b.log, b.broadcaster, b.sendTimeout, b.responseLimit, sub).Stream(ctx) return sub } func (b EventsBackend) getResponseFactory(filter EventFilter) GetDataByHeightFunc { return func(ctx context.Context, height uint64) (interface{}, error) { - header, err := b.headers.ByHeight(height) - if err != nil { - return nil, fmt.Errorf("could not get block header for height %d: %w", height, err) - } - - executionData, err := b.getExecutionData(ctx, header.ID()) + executionData, err := b.getExecutionData(ctx, height) if err != nil { - return nil, fmt.Errorf("could not get execution data for block %s: %w", header.ID(), err) + return nil, fmt.Errorf("could not get execution data for block %d: %w", height, err) } events := []flow.Event{} @@ -69,13 +55,13 @@ func (b EventsBackend) getResponseFactory(filter EventFilter) GetDataByHeightFun } b.log.Trace(). - Hex("block_id", logging.ID(header.ID())). - Uint64("height", header.Height). + Hex("block_id", logging.ID(executionData.BlockID)). + Uint64("height", height). Msgf("sending %d events", len(events)) return &EventsResponse{ - BlockID: header.ID(), - Height: header.Height, + BlockID: executionData.BlockID, + Height: height, Events: events, }, nil } diff --git a/engine/access/state_stream/backend_events_test.go b/engine/access/state_stream/backend_events_test.go index 1b3067399c9..68ca0a789cb 100644 --- a/engine/access/state_stream/backend_events_test.go +++ b/engine/access/state_stream/backend_events_test.go @@ -62,6 +62,18 @@ func (s *BackendEventsSuite) TestSubscribeEvents() { startBlockID: s.blocks[0].ID(), startHeight: 0, }, + { + name: "happy path - start from root block by height", + highestBackfill: len(s.blocks) - 1, // backfill all blocks + startBlockID: flow.ZeroID, + startHeight: s.backend.rootBlockHeight, // start from root block + }, + { + name: "happy path - start from root block by id", + highestBackfill: len(s.blocks) - 1, // backfill all blocks + startBlockID: s.backend.rootBlockID, // start from root block + startHeight: 0, + }, } // supports simple address comparisions for testing @@ -96,8 +108,7 @@ func (s *BackendEventsSuite) TestSubscribeEvents() { // this simulates a subscription on a past block for i := 0; i <= test.highestBackfill; i++ { s.T().Logf("backfilling block %d", i) - execData := s.execDataMap[s.blocks[i].ID()] - s.execDataDistributor.OnExecutionDataReceived(execData) + s.backend.setHighestHeight(s.blocks[i].Header.Height) } subCtx, subCancel := context.WithCancel(ctx) @@ -105,13 +116,12 @@ func (s *BackendEventsSuite) TestSubscribeEvents() { // loop over all of the blocks for i, b := range s.blocks { - execData := s.execDataMap[b.ID()] s.T().Logf("checking block %d %v", i, b.ID()) // simulate new exec data received. // exec data for all blocks with index <= highestBackfill were already received if i > test.highestBackfill { - s.execDataDistributor.OnExecutionDataReceived(execData) + s.backend.setHighestHeight(b.Header.Height) s.broadcaster.Publish() } @@ -167,22 +177,30 @@ func (s *BackendExecutionDataSuite) TestSubscribeEventsHandlesErrors() { assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err())) }) + s.Run("returns error for start height before root height", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeEvents(subCtx, flow.ZeroID, s.backend.rootBlockHeight-1, EventFilter{}) + assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err()), "expected InvalidArgument, got %v: %v", status.Code(sub.Err()).String(), sub.Err()) + }) + s.Run("returns error for unindexed start blockID", func() { subCtx, subCancel := context.WithCancel(ctx) defer subCancel() sub := s.backend.SubscribeEvents(subCtx, unittest.IdentifierFixture(), 0, EventFilter{}) - assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "exepected NotFound, got %v: %v", status.Code(sub.Err()).String(), sub.Err()) + assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "expected NotFound, got %v: %v", status.Code(sub.Err()).String(), sub.Err()) }) // make sure we're starting with a fresh cache - s.execDataCache.Clear() + s.execDataHeroCache.Clear() s.Run("returns error for unindexed start height", func() { subCtx, subCancel := context.WithCancel(ctx) defer subCancel() sub := s.backend.SubscribeEvents(subCtx, flow.ZeroID, s.blocks[len(s.blocks)-1].Header.Height+10, EventFilter{}) - assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "exepected NotFound, got %v: %v", status.Code(sub.Err()).String(), sub.Err()) + assert.Equal(s.T(), codes.NotFound, status.Code(sub.Err()), "expected NotFound, got %v: %v", status.Code(sub.Err()).String(), sub.Err()) }) } diff --git a/engine/access/state_stream/backend_executiondata.go b/engine/access/state_stream/backend_executiondata.go index b39df9da610..0443c6ba9ba 100644 --- a/engine/access/state_stream/backend_executiondata.go +++ b/engine/access/state_stream/backend_executiondata.go @@ -27,6 +27,7 @@ type ExecutionDataBackend struct { headers storage.Headers broadcaster *engine.Broadcaster sendTimeout time.Duration + responseLimit float64 sendBufferSize int getExecutionData GetExecutionDataFunc @@ -34,7 +35,12 @@ type ExecutionDataBackend struct { } func (b *ExecutionDataBackend) GetExecutionDataByBlockID(ctx context.Context, blockID flow.Identifier) (*execution_data.BlockExecutionData, error) { - executionData, err := b.getExecutionData(ctx, blockID) + header, err := b.headers.ByBlockID(blockID) + if err != nil { + return nil, fmt.Errorf("could not get block header for %s: %w", blockID, err) + } + + executionData, err := b.getExecutionData(ctx, header.Height) if err != nil { // need custom not found handler due to blob not found error @@ -51,36 +57,24 @@ func (b *ExecutionDataBackend) GetExecutionDataByBlockID(ctx context.Context, bl func (b *ExecutionDataBackend) SubscribeExecutionData(ctx context.Context, startBlockID flow.Identifier, startHeight uint64) Subscription { nextHeight, err := b.getStartHeight(startBlockID, startHeight) if err != nil { - sub := NewSubscription(b.sendBufferSize) - if st, ok := status.FromError(err); ok { - sub.Fail(status.Errorf(st.Code(), "could not get start height: %s", st.Message())) - return sub - } - - sub.Fail(fmt.Errorf("could not get start height: %w", err)) - return sub + return NewFailedSubscription(err, "could not get start height") } sub := NewHeightBasedSubscription(b.sendBufferSize, nextHeight, b.getResponse) - go NewStreamer(b.log, b.broadcaster, b.sendTimeout, sub).Stream(ctx) + go NewStreamer(b.log, b.broadcaster, b.sendTimeout, b.responseLimit, sub).Stream(ctx) return sub } func (b *ExecutionDataBackend) getResponse(ctx context.Context, height uint64) (interface{}, error) { - header, err := b.headers.ByHeight(height) - if err != nil { - return nil, fmt.Errorf("could not get block header for height %d: %w", height, err) - } - - executionData, err := b.getExecutionData(ctx, header.ID()) + executionData, err := b.getExecutionData(ctx, height) if err != nil { - return nil, fmt.Errorf("could not get execution data for block %s: %w", header.ID(), err) + return nil, fmt.Errorf("could not get execution data for block %d: %w", height, err) } return &ExecutionDataResponse{ - Height: header.Height, + Height: height, ExecutionData: executionData.BlockExecutionData, }, nil } diff --git a/engine/access/state_stream/backend_executiondata_test.go b/engine/access/state_stream/backend_executiondata_test.go index 37547043fe1..361cb64aa80 100644 --- a/engine/access/state_stream/backend_executiondata_test.go +++ b/engine/access/state_stream/backend_executiondata_test.go @@ -1,10 +1,8 @@ package state_stream import ( - "bytes" "context" "fmt" - "math/rand" "testing" "time" @@ -18,15 +16,12 @@ import ( "google.golang.org/grpc/status" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/ledger" - "github.com/onflow/flow-go/ledger/common/testutils" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/blobs" "github.com/onflow/flow-go/module/executiondatasync/execution_data" - herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" - "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" + "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" + "github.com/onflow/flow-go/module/mempool/herocache" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/module/state_synchronization/requester" protocolmock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/storage" storagemock "github.com/onflow/flow-go/storage/mock" @@ -43,17 +38,18 @@ type BackendExecutionDataSuite struct { suite.Suite state *protocolmock.State + params *protocolmock.Params snapshot *protocolmock.Snapshot headers *storagemock.Headers seals *storagemock.Seals results *storagemock.ExecutionResults - bs blobs.Blobstore - eds execution_data.ExecutionDataStore - broadcaster *engine.Broadcaster - execDataDistributor *requester.ExecutionDataDistributor - execDataCache *herocache.Cache - backend *StateStreamBackend + bs blobs.Blobstore + eds execution_data.ExecutionDataStore + broadcaster *engine.Broadcaster + execDataCache *cache.ExecutionDataCache + execDataHeroCache *herocache.BlockExecutionData + backend *StateStreamBackend blocks []*flow.Block blockEvents map[flow.Identifier]flow.EventsList @@ -68,12 +64,11 @@ func TestBackendExecutionDataSuite(t *testing.T) { } func (s *BackendExecutionDataSuite) SetupTest() { - rand.Seed(time.Now().UnixNano()) - logger := unittest.Logger() s.state = protocolmock.NewState(s.T()) s.snapshot = protocolmock.NewSnapshot(s.T()) + s.params = protocolmock.NewParams(s.T()) s.headers = storagemock.NewHeaders(s.T()) s.seals = storagemock.NewSeals(s.T()) s.results = storagemock.NewExecutionResults(s.T()) @@ -82,15 +77,9 @@ func (s *BackendExecutionDataSuite) SetupTest() { s.eds = execution_data.NewExecutionDataStore(s.bs, execution_data.DefaultSerializer) s.broadcaster = engine.NewBroadcaster() - s.execDataDistributor = requester.NewExecutionDataDistributor() - s.execDataCache = herocache.NewCache( - DefaultCacheSize, - herocache.DefaultOversizeFactor, - heropool.LRUEjection, - logger, - metrics.NewNoopCollector(), - ) + s.execDataHeroCache = herocache.NewBlockExecutionData(DefaultCacheSize, logger, metrics.NewNoopCollector()) + s.execDataCache = cache.NewExecutionDataCache(s.eds, s.headers, s.seals, s.results, s.execDataHeroCache) conf := Config{ ClientSendTimeout: DefaultSendTimeout, @@ -98,18 +87,6 @@ func (s *BackendExecutionDataSuite) SetupTest() { } var err error - s.backend, err = New( - logger, - conf, - s.state, - s.headers, - s.seals, - s.results, - s.eds, - s.execDataCache, - s.broadcaster, - ) - require.NoError(s.T(), err) blockCount := 5 s.execDataMap = make(map[flow.Identifier]*execution_data.BlockExecutionDataEntity, blockCount) @@ -120,24 +97,41 @@ func (s *BackendExecutionDataSuite) SetupTest() { s.blocks = make([]*flow.Block, 0, blockCount) // generate blockCount consecutive blocks with associated seal, result and execution data - firstBlock := unittest.BlockFixture() - parent := firstBlock.Header + rootBlock := unittest.BlockFixture() + parent := rootBlock.Header + s.blockMap[rootBlock.Header.Height] = &rootBlock + + s.T().Logf("Generating %d blocks, root block: %d %s", blockCount, rootBlock.Header.Height, rootBlock.ID()) + for i := 0; i < blockCount; i++ { - var block *flow.Block - if i == 0 { - block = &firstBlock - } else { - block = unittest.BlockWithParentFixture(parent) - } + block := unittest.BlockWithParentFixture(parent) // update for next iteration parent = block.Header seal := unittest.BlockSealsFixture(1)[0] result := unittest.ExecutionResultFixture() blockEvents := unittest.BlockEventsFixture(block.Header, (i%len(testEventTypes))*3+1, testEventTypes...) - execData := blockExecutionDataFixture(s.T(), block, blockEvents.Events) - result.ExecutionDataID, err = s.eds.AddExecutionData(context.TODO(), execData) + numChunks := 5 + chunkDatas := make([]*execution_data.ChunkExecutionData, 0, numChunks) + for i := 0; i < numChunks; i++ { + var events flow.EventsList + switch { + case i >= len(blockEvents.Events): + events = flow.EventsList{} + case i == numChunks-1: + events = blockEvents.Events[i:] + default: + events = flow.EventsList{blockEvents.Events[i]} + } + chunkDatas = append(chunkDatas, unittest.ChunkExecutionDataFixture(s.T(), execution_data.DefaultMaxBlobSize/5, unittest.WithChunkEvents(events))) + } + execData := unittest.BlockExecutionDataFixture( + unittest.WithBlockExecutionDataBlockID(block.ID()), + unittest.WithChunkExecutionDatas(chunkDatas...), + ) + + result.ExecutionDataID, err = s.eds.Add(context.TODO(), execData) assert.NoError(s.T(), err) s.blocks = append(s.blocks, block) @@ -151,7 +145,7 @@ func (s *BackendExecutionDataSuite) SetupTest() { } s.state.On("Sealed").Return(s.snapshot, nil).Maybe() - s.snapshot.On("Head").Return(firstBlock.Header, nil).Maybe() + s.snapshot.On("Head").Return(s.blocks[0].Header, nil).Maybe() s.seals.On("FinalizedSealForBlock", mock.AnythingOfType("flow.Identifier")).Return( func(blockID flow.Identifier) *flow.Seal { @@ -216,6 +210,36 @@ func (s *BackendExecutionDataSuite) SetupTest() { return storage.ErrNotFound }, ).Maybe() + + s.headers.On("BlockIDByHeight", mock.AnythingOfType("uint64")).Return( + func(height uint64) flow.Identifier { + if block, ok := s.blockMap[height]; ok { + return block.Header.ID() + } + return flow.ZeroID + }, + func(height uint64) error { + if _, ok := s.blockMap[height]; ok { + return nil + } + return storage.ErrNotFound + }, + ).Maybe() + + s.backend, err = New( + logger, + conf, + s.state, + s.headers, + s.seals, + s.results, + s.eds, + s.execDataCache, + s.broadcaster, + rootBlock.Header.Height, + rootBlock.Header.Height, // initialize with no downloaded data + ) + require.NoError(s.T(), err) } func (s *BackendExecutionDataSuite) TestGetExecutionDataByBlockID() { @@ -227,9 +251,12 @@ func (s *BackendExecutionDataSuite) TestGetExecutionDataByBlockID() { result := s.resultMap[seal.ResultID] execData := s.execDataMap[block.ID()] + // notify backend block is available + s.backend.setHighestHeight(block.Header.Height) + var err error s.Run("happy path TestGetExecutionDataByBlockID success", func() { - result.ExecutionDataID, err = s.eds.AddExecutionData(ctx, execData.BlockExecutionData) + result.ExecutionDataID, err = s.eds.Add(ctx, execData.BlockExecutionData) require.NoError(s.T(), err) res, err := s.backend.GetExecutionDataByBlockID(ctx, block.ID()) @@ -237,7 +264,7 @@ func (s *BackendExecutionDataSuite) TestGetExecutionDataByBlockID() { assert.NoError(s.T(), err) }) - s.execDataCache.Clear() + s.execDataHeroCache.Clear() s.Run("missing exec data for TestGetExecutionDataByBlockID failure", func() { result.ExecutionDataID = unittest.IdentifierFixture() @@ -248,58 +275,6 @@ func (s *BackendExecutionDataSuite) TestGetExecutionDataByBlockID() { }) } -func blockExecutionDataFixture(t *testing.T, block *flow.Block, events []flow.Event) *execution_data.BlockExecutionData { - numChunks := 5 - minSerializedSize := 5 * execution_data.DefaultMaxBlobSize - - chunks := make([]*execution_data.ChunkExecutionData, numChunks) - - for i := 0; i < numChunks; i++ { - var e flow.EventsList - switch { - case i >= len(events): - e = flow.EventsList{} - case i == numChunks-1: - e = events[i:] - default: - e = flow.EventsList{events[i]} - } - chunks[i] = chunkExecutionDataFixture(t, uint64(minSerializedSize), e) - } - - return &execution_data.BlockExecutionData{ - BlockID: block.ID(), - ChunkExecutionDatas: chunks, - } -} - -func chunkExecutionDataFixture(t *testing.T, minSerializedSize uint64, events []flow.Event) *execution_data.ChunkExecutionData { - ced := &execution_data.ChunkExecutionData{ - TrieUpdate: testutils.TrieUpdateFixture(1, 1, 8), - Events: events, - } - - size := 1 - - for { - buf := &bytes.Buffer{} - require.NoError(t, execution_data.DefaultSerializer.Serialize(buf, ced)) - if buf.Len() >= int(minSerializedSize) { - return ced - } - - v := make([]byte, size) - _, err := rand.Read(v) - require.NoError(t, err) - - k, err := ced.TrieUpdate.Payloads[0].Key() - require.NoError(t, err) - - ced.TrieUpdate.Payloads[0] = ledger.NewPayload(k, v) - size *= 2 - } -} - func (s *BackendExecutionDataSuite) TestSubscribeExecutionData() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -328,12 +303,24 @@ func (s *BackendExecutionDataSuite) TestSubscribeExecutionData() { startBlockID: s.blocks[0].ID(), startHeight: 0, }, + { + name: "happy path - start from root block by height", + highestBackfill: len(s.blocks) - 1, // backfill all blocks + startBlockID: flow.ZeroID, + startHeight: s.backend.rootBlockHeight, // start from root block + }, + { + name: "happy path - start from root block by id", + highestBackfill: len(s.blocks) - 1, // backfill all blocks + startBlockID: s.backend.rootBlockID, // start from root block + startHeight: 0, + }, } for _, test := range tests { s.Run(test.name, func() { // make sure we're starting with a fresh cache - s.execDataCache.Clear() + s.execDataHeroCache.Clear() s.T().Logf("len(s.execDataMap) %d", len(s.execDataMap)) @@ -341,8 +328,7 @@ func (s *BackendExecutionDataSuite) TestSubscribeExecutionData() { // this simulates a subscription on a past block for i := 0; i <= test.highestBackfill; i++ { s.T().Logf("backfilling block %d", i) - execData := s.execDataMap[s.blocks[i].ID()] - s.execDataDistributor.OnExecutionDataReceived(execData) + s.backend.setHighestHeight(s.blocks[i].Header.Height) } subCtx, subCancel := context.WithCancel(ctx) @@ -356,7 +342,7 @@ func (s *BackendExecutionDataSuite) TestSubscribeExecutionData() { // simulate new exec data received. // exec data for all blocks with index <= highestBackfill were already received if i > test.highestBackfill { - s.execDataDistributor.OnExecutionDataReceived(execData) + s.backend.setHighestHeight(b.Header.Height) s.broadcaster.Publish() } @@ -404,6 +390,14 @@ func (s *BackendExecutionDataSuite) TestSubscribeExecutionDataHandlesErrors() { assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err())) }) + s.Run("returns error for start height before root height", func() { + subCtx, subCancel := context.WithCancel(ctx) + defer subCancel() + + sub := s.backend.SubscribeExecutionData(subCtx, flow.ZeroID, s.backend.rootBlockHeight-1) + assert.Equal(s.T(), codes.InvalidArgument, status.Code(sub.Err())) + }) + s.Run("returns error for unindexed start blockID", func() { subCtx, subCancel := context.WithCancel(ctx) defer subCancel() @@ -413,7 +407,7 @@ func (s *BackendExecutionDataSuite) TestSubscribeExecutionDataHandlesErrors() { }) // make sure we're starting with a fresh cache - s.execDataCache.Clear() + s.execDataHeroCache.Clear() s.Run("returns error for unindexed start height", func() { subCtx, subCancel := context.WithCancel(ctx) diff --git a/engine/access/state_stream/engine.go b/engine/access/state_stream/engine.go index 29d17c7411a..cb3a3e73813 100644 --- a/engine/access/state_stream/engine.go +++ b/engine/access/state_stream/engine.go @@ -2,23 +2,18 @@ package state_stream import ( "fmt" - "net" "time" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" access "github.com/onflow/flow/protobuf/go/flow/executiondata" "github.com/rs/zerolog" - "google.golang.org/grpc" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" + "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/module/irrecoverable" - herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" - "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/logging" @@ -49,6 +44,11 @@ type Config struct { // ClientSendBufferSize is the size of the response buffer for sending messages to the client. ClientSendBufferSize uint + + // ResponseLimit is the max responses per second allowed on a stream. After exceeding the limit, + // the stream is paused until more capacity is available. Searches of past data can be CPU + // intensive, so this helps manage the impact. + ResponseLimit float64 } // Engine exposes the server with the state stream API. @@ -58,15 +58,13 @@ type Engine struct { *component.ComponentManager log zerolog.Logger backend *StateStreamBackend - server *grpc.Server config Config chain flow.Chain handler *Handler execDataBroadcaster *engine.Broadcaster - execDataCache *herocache.Cache - - stateStreamGrpcAddress net.Addr + execDataCache *cache.ExecutionDataCache + headers storage.Headers } // NewEng returns a new ingress server. @@ -74,56 +72,33 @@ func NewEng( log zerolog.Logger, config Config, execDataStore execution_data.ExecutionDataStore, + execDataCache *cache.ExecutionDataCache, state protocol.State, headers storage.Headers, seals storage.Seals, results storage.ExecutionResults, chainID flow.ChainID, - apiRatelimits map[string]int, // the api rate limit (max calls per second) for each of the gRPC API e.g. Ping->100, GetExecutionDataByBlockID->300 - apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the gRPC API e.g. Ping->50, GetExecutionDataByBlockID->10 - heroCacheMetrics module.HeroCacheMetrics, + initialBlockHeight uint64, + highestBlockHeight uint64, + server *grpcserver.GrpcServer, ) (*Engine, error) { logger := log.With().Str("engine", "state_stream_rpc").Logger() - // create a GRPC server to serve GRPC clients - grpcOpts := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(int(config.MaxExecutionDataMsgSize)), - grpc.MaxSendMsgSize(int(config.MaxExecutionDataMsgSize)), - } - - var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors - // if rpc metrics is enabled, add the grpc metrics interceptor as a server option - if config.RpcMetricsEnabled { - interceptors = append(interceptors, grpc_prometheus.UnaryServerInterceptor) - } - - if len(apiRatelimits) > 0 { - // create a rate limit interceptor - rateLimitInterceptor := rpc.NewRateLimiterInterceptor(log, apiRatelimits, apiBurstLimits).UnaryServerInterceptor - // append the rate limit interceptor to the list of interceptors - interceptors = append(interceptors, rateLimitInterceptor) - } - - // add the logging interceptor, ensure it is innermost wrapper - interceptors = append(interceptors, rpc.LoggingInterceptor(log)...) - - // create a chained unary interceptor - chainedInterceptors := grpc.ChainUnaryInterceptor(interceptors...) - grpcOpts = append(grpcOpts, chainedInterceptors) - - server := grpc.NewServer(grpcOpts...) + broadcaster := engine.NewBroadcaster() - execDataCache := herocache.NewCache( - config.ExecutionDataCacheSize, - herocache.DefaultOversizeFactor, - heropool.LRUEjection, + backend, err := New( logger, - heroCacheMetrics, + config, + state, + headers, + seals, + results, + execDataStore, + execDataCache, + broadcaster, + initialBlockHeight, + highestBlockHeight, ) - - broadcaster := engine.NewBroadcaster() - - backend, err := New(logger, config, state, headers, seals, results, execDataStore, execDataCache, broadcaster) if err != nil { return nil, fmt.Errorf("could not create state stream backend: %w", err) } @@ -131,7 +106,7 @@ func NewEng( e := &Engine{ log: logger, backend: backend, - server: server, + headers: headers, chain: chainID.Chain(), config: config, handler: NewHandler(backend, chainID.Chain(), config.EventFilterConfig, config.MaxGlobalStreams), @@ -140,44 +115,40 @@ func NewEng( } e.ComponentManager = component.NewComponentManagerBuilder(). - AddWorker(e.serve). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + <-server.Done() + }). Build() - access.RegisterExecutionDataAPIServer(e.server, e.handler) + access.RegisterExecutionDataAPIServer(server.Server, e.handler) return e, nil } // OnExecutionData is called to notify the engine when a new execution data is received. +// The caller must guarantee that execution data is locally available for all blocks with +// heights between the initialBlockHeight provided during startup and the block height of +// the execution data provided. func (e *Engine) OnExecutionData(executionData *execution_data.BlockExecutionDataEntity) { - e.log.Trace(). - Hex("block_id", logging.ID(executionData.BlockID)). - Msg("received execution data") + lg := e.log.With().Hex("block_id", logging.ID(executionData.BlockID)).Logger() - _ = e.execDataCache.Add(executionData.BlockID, executionData) - e.execDataBroadcaster.Publish() -} + lg.Trace().Msg("received execution data") -// serve starts the gRPC server. -// When this function returns, the server is considered ready. -func (e *Engine) serve(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - e.log.Info().Str("state_stream_address", e.config.ListenAddr).Msg("starting grpc server on address") - l, err := net.Listen("tcp", e.config.ListenAddr) + header, err := e.headers.ByBlockID(executionData.BlockID) if err != nil { - ctx.Throw(fmt.Errorf("error starting grpc server: %w", err)) + // if the execution data is available, the block must be locally finalized + lg.Fatal().Err(err).Msg("failed to get header for execution data") + return } - e.stateStreamGrpcAddress = l.Addr() - e.log.Debug().Str("state_stream_address", e.stateStreamGrpcAddress.String()).Msg("listening on port") - - go func() { - ready() - err = e.server.Serve(l) - if err != nil { - ctx.Throw(fmt.Errorf("error trying to serve grpc server: %w", err)) - } - }() + if ok := e.backend.setHighestHeight(header.Height); !ok { + // this means that the height was lower than the current highest height + // OnExecutionData is guaranteed by the requester to be called in order, but may be called + // multiple times for the same block. + lg.Debug().Msg("execution data for block already received") + return + } - <-ctx.Done() - e.server.GracefulStop() + e.execDataBroadcaster.Publish() } diff --git a/engine/access/state_stream/mock/get_data_by_height_func.go b/engine/access/state_stream/mock/get_data_by_height_func.go new file mode 100644 index 00000000000..6584160b378 --- /dev/null +++ b/engine/access/state_stream/mock/get_data_by_height_func.go @@ -0,0 +1,55 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// GetDataByHeightFunc is an autogenerated mock type for the GetDataByHeightFunc type +type GetDataByHeightFunc struct { + mock.Mock +} + +// Execute provides a mock function with given fields: ctx, height +func (_m *GetDataByHeightFunc) Execute(ctx context.Context, height uint64) (interface{}, error) { + ret := _m.Called(ctx, height) + + var r0 interface{} + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, uint64) (interface{}, error)); ok { + return rf(ctx, height) + } + if rf, ok := ret.Get(0).(func(context.Context, uint64) interface{}); ok { + r0 = rf(ctx, height) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interface{}) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, uint64) error); ok { + r1 = rf(ctx, height) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewGetDataByHeightFunc interface { + mock.TestingT + Cleanup(func()) +} + +// NewGetDataByHeightFunc creates a new instance of GetDataByHeightFunc. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGetDataByHeightFunc(t mockConstructorTestingTNewGetDataByHeightFunc) *GetDataByHeightFunc { + mock := &GetDataByHeightFunc{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/access/state_stream/mock/get_execution_data_func.go b/engine/access/state_stream/mock/get_execution_data_func.go new file mode 100644 index 00000000000..50fe8087e21 --- /dev/null +++ b/engine/access/state_stream/mock/get_execution_data_func.go @@ -0,0 +1,56 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + context "context" + + execution_data "github.com/onflow/flow-go/module/executiondatasync/execution_data" + mock "github.com/stretchr/testify/mock" +) + +// GetExecutionDataFunc is an autogenerated mock type for the GetExecutionDataFunc type +type GetExecutionDataFunc struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0, _a1 +func (_m *GetExecutionDataFunc) Execute(_a0 context.Context, _a1 uint64) (*execution_data.BlockExecutionDataEntity, error) { + ret := _m.Called(_a0, _a1) + + var r0 *execution_data.BlockExecutionDataEntity + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, uint64) (*execution_data.BlockExecutionDataEntity, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, uint64) *execution_data.BlockExecutionDataEntity); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*execution_data.BlockExecutionDataEntity) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, uint64) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewGetExecutionDataFunc interface { + mock.TestingT + Cleanup(func()) +} + +// NewGetExecutionDataFunc creates a new instance of GetExecutionDataFunc. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGetExecutionDataFunc(t mockConstructorTestingTNewGetExecutionDataFunc) *GetExecutionDataFunc { + mock := &GetExecutionDataFunc{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/access/state_stream/mock/get_start_height_func.go b/engine/access/state_stream/mock/get_start_height_func.go new file mode 100644 index 00000000000..b97a77e1d39 --- /dev/null +++ b/engine/access/state_stream/mock/get_start_height_func.go @@ -0,0 +1,52 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// GetStartHeightFunc is an autogenerated mock type for the GetStartHeightFunc type +type GetStartHeightFunc struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0, _a1 +func (_m *GetStartHeightFunc) Execute(_a0 flow.Identifier, _a1 uint64) (uint64, error) { + ret := _m.Called(_a0, _a1) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier, uint64) (uint64, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(flow.Identifier, uint64) uint64); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(flow.Identifier, uint64) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewGetStartHeightFunc interface { + mock.TestingT + Cleanup(func()) +} + +// NewGetStartHeightFunc creates a new instance of GetStartHeightFunc. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGetStartHeightFunc(t mockConstructorTestingTNewGetStartHeightFunc) *GetStartHeightFunc { + mock := &GetStartHeightFunc{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/access/state_stream/mock/streamable.go b/engine/access/state_stream/mock/streamable.go new file mode 100644 index 00000000000..d1ec4de5f7d --- /dev/null +++ b/engine/access/state_stream/mock/streamable.go @@ -0,0 +1,95 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + time "time" +) + +// Streamable is an autogenerated mock type for the Streamable type +type Streamable struct { + mock.Mock +} + +// Close provides a mock function with given fields: +func (_m *Streamable) Close() { + _m.Called() +} + +// Fail provides a mock function with given fields: _a0 +func (_m *Streamable) Fail(_a0 error) { + _m.Called(_a0) +} + +// ID provides a mock function with given fields: +func (_m *Streamable) ID() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// Next provides a mock function with given fields: _a0 +func (_m *Streamable) Next(_a0 context.Context) (interface{}, error) { + ret := _m.Called(_a0) + + var r0 interface{} + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (interface{}, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(context.Context) interface{}); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interface{}) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Send provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Streamable) Send(_a0 context.Context, _a1 interface{}, _a2 time.Duration) error { + ret := _m.Called(_a0, _a1, _a2) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, interface{}, time.Duration) error); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewStreamable interface { + mock.TestingT + Cleanup(func()) +} + +// NewStreamable creates a new instance of Streamable. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewStreamable(t mockConstructorTestingTNewStreamable) *Streamable { + mock := &Streamable{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/access/state_stream/mock/subscription.go b/engine/access/state_stream/mock/subscription.go new file mode 100644 index 00000000000..066506ff57c --- /dev/null +++ b/engine/access/state_stream/mock/subscription.go @@ -0,0 +1,69 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import mock "github.com/stretchr/testify/mock" + +// Subscription is an autogenerated mock type for the Subscription type +type Subscription struct { + mock.Mock +} + +// Channel provides a mock function with given fields: +func (_m *Subscription) Channel() <-chan interface{} { + ret := _m.Called() + + var r0 <-chan interface{} + if rf, ok := ret.Get(0).(func() <-chan interface{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan interface{}) + } + } + + return r0 +} + +// Err provides a mock function with given fields: +func (_m *Subscription) Err() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ID provides a mock function with given fields: +func (_m *Subscription) ID() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +type mockConstructorTestingTNewSubscription interface { + mock.TestingT + Cleanup(func()) +} + +// NewSubscription creates a new instance of Subscription. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewSubscription(t mockConstructorTestingTNewSubscription) *Subscription { + mock := &Subscription{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/access/state_stream/streamer.go b/engine/access/state_stream/streamer.go index d2313f7d693..22d01394525 100644 --- a/engine/access/state_stream/streamer.go +++ b/engine/access/state_stream/streamer.go @@ -7,6 +7,7 @@ import ( "time" "github.com/rs/zerolog" + "golang.org/x/time/rate" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/module/executiondatasync/execution_data" @@ -25,21 +26,30 @@ type Streamable interface { // Streamer type Streamer struct { log zerolog.Logger + sub Streamable broadcaster *engine.Broadcaster sendTimeout time.Duration - sub Streamable + limiter *rate.Limiter } func NewStreamer( log zerolog.Logger, broadcaster *engine.Broadcaster, sendTimeout time.Duration, + limit float64, sub Streamable, ) *Streamer { + var limiter *rate.Limiter + if limit > 0 { + // allows for 1 response per call, averaging `limit` responses per second over longer time frames + limiter = rate.NewLimiter(rate.Limit(limit), 1) + } + return &Streamer{ log: log.With().Str("sub_id", sub.ID()).Logger(), broadcaster: broadcaster, sendTimeout: sendTimeout, + limiter: limiter, sub: sub, } } @@ -79,6 +89,11 @@ func (s *Streamer) Stream(ctx context.Context) { // sendAllAvailable reads data from the streamable and sends it to the client until no more data is available. func (s *Streamer) sendAllAvailable(ctx context.Context) error { for { + // blocking wait for the streamer's rate limit to have available capacity + if err := s.checkRateLimit(ctx); err != nil { + return fmt.Errorf("error waiting for response capacity: %w", err) + } + response, err := s.sub.Next(ctx) if err != nil { @@ -102,3 +117,14 @@ func (s *Streamer) sendAllAvailable(ctx context.Context) error { } } } + +// checkRateLimit checks the stream's rate limit and blocks until there is room to send a response. +// An error is returned if the context is canceled or the expected wait time exceeds the context's +// deadline. +func (s *Streamer) checkRateLimit(ctx context.Context) error { + if s.limiter == nil { + return nil + } + + return s.limiter.WaitN(ctx, 1) +} diff --git a/engine/access/state_stream/streamer_test.go b/engine/access/state_stream/streamer_test.go new file mode 100644 index 00000000000..6c80feec7ed --- /dev/null +++ b/engine/access/state_stream/streamer_test.go @@ -0,0 +1,153 @@ +package state_stream_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/access/state_stream" + streammock "github.com/onflow/flow-go/engine/access/state_stream/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +type testData struct { + data string + err error +} + +var testErr = fmt.Errorf("test error") + +func TestStream(t *testing.T) { + t.Parallel() + + ctx := context.Background() + timeout := state_stream.DefaultSendTimeout + + sub := streammock.NewStreamable(t) + sub.On("ID").Return(uuid.NewString()) + + tests := []testData{} + for i := 0; i < 4; i++ { + tests = append(tests, testData{fmt.Sprintf("test%d", i), nil}) + } + tests = append(tests, testData{"", testErr}) + + broadcaster := engine.NewBroadcaster() + streamer := state_stream.NewStreamer(unittest.Logger(), broadcaster, timeout, state_stream.DefaultResponseLimit, sub) + + for _, d := range tests { + sub.On("Next", mock.Anything).Return(d.data, d.err).Once() + if d.err == nil { + sub.On("Send", mock.Anything, d.data, timeout).Return(nil).Once() + } else { + mocked := sub.On("Fail", mock.Anything).Return().Once() + mocked.RunFn = func(args mock.Arguments) { + assert.ErrorIs(t, args.Get(0).(error), d.err) + } + } + } + + broadcaster.Publish() + + unittest.RequireReturnsBefore(t, func() { + streamer.Stream(ctx) + }, 100*time.Millisecond, "streamer.Stream() should return quickly") +} + +func TestStreamRatelimited(t *testing.T) { + t.Parallel() + + ctx := context.Background() + timeout := state_stream.DefaultSendTimeout + duration := 100 * time.Millisecond + + for _, limit := range []float64{0.2, 3, 20, 500} { + t.Run(fmt.Sprintf("responses are limited - %.1f rps", limit), func(t *testing.T) { + sub := streammock.NewStreamable(t) + sub.On("ID").Return(uuid.NewString()) + + broadcaster := engine.NewBroadcaster() + streamer := state_stream.NewStreamer(unittest.Logger(), broadcaster, timeout, limit, sub) + + var nextCalls, sendCalls int + sub.On("Next", mock.Anything).Return("data", nil).Run(func(args mock.Arguments) { + nextCalls++ + }) + sub.On("Send", mock.Anything, "data", timeout).Return(nil).Run(func(args mock.Arguments) { + sendCalls++ + }) + + broadcaster.Publish() + + unittest.RequireNeverReturnBefore(t, func() { + streamer.Stream(ctx) + }, duration, "streamer.Stream() should never stop") + + // check the number of calls and make sure they are sane. + // ratelimit uses a token bucket algorithm which adds 1 token every 1/r seconds. This + // comes to roughly 10% of r within 100ms. + // + // Add a large buffer since the algorithm only guarantees the rate over longer time + // ranges. Since this test covers various orders of magnitude, we can still validate it + // is working as expected. + target := int(limit * float64(duration) / float64(time.Second)) + if target == 0 { + target = 1 + } + + assert.LessOrEqual(t, nextCalls, target*3) + assert.LessOrEqual(t, sendCalls, target*3) + }) + } +} + +// TestLongStreamRatelimited tests that the streamer is uses the correct rate limit over a longer +// period of time +func TestLongStreamRatelimited(t *testing.T) { + t.Parallel() + + unittest.SkipUnless(t, unittest.TEST_LONG_RUNNING, "skipping long stream rate limit test") + + ctx := context.Background() + timeout := state_stream.DefaultSendTimeout + + limit := 5.0 + duration := 30 * time.Second + + sub := streammock.NewStreamable(t) + sub.On("ID").Return(uuid.NewString()) + + broadcaster := engine.NewBroadcaster() + streamer := state_stream.NewStreamer(unittest.Logger(), broadcaster, timeout, limit, sub) + + var nextCalls, sendCalls int + sub.On("Next", mock.Anything).Return("data", nil).Run(func(args mock.Arguments) { + nextCalls++ + }) + sub.On("Send", mock.Anything, "data", timeout).Return(nil).Run(func(args mock.Arguments) { + sendCalls++ + }) + + broadcaster.Publish() + + unittest.RequireNeverReturnBefore(t, func() { + streamer.Stream(ctx) + }, duration, "streamer.Stream() should never stop") + + // check the number of calls and make sure they are sane. + // over a longer time, the rate limit should be more accurate + target := int(limit) * int(duration/time.Second) + diff := 5 // 5 ~= 3% of 150 expected + + assert.LessOrEqual(t, nextCalls, target+diff) + assert.GreaterOrEqual(t, nextCalls, target-diff) + + assert.LessOrEqual(t, sendCalls, target+diff) + assert.GreaterOrEqual(t, sendCalls, target-diff) +} diff --git a/engine/access/state_stream/subscription.go b/engine/access/state_stream/subscription.go index 83f9775a005..df354d42af1 100644 --- a/engine/access/state_stream/subscription.go +++ b/engine/access/state_stream/subscription.go @@ -7,6 +7,7 @@ import ( "time" "github.com/google/uuid" + "google.golang.org/grpc/status" ) // DefaultSendBufferSize is the default buffer size for the subscription's send channel. @@ -27,13 +28,15 @@ type Subscription interface { // ID returns the unique identifier for this subscription used for logging ID() string - // Channel returns the channel from which subscriptino data can be read + // Channel returns the channel from which subscription data can be read Channel() <-chan interface{} // Err returns the error that caused the subscription to fail Err() error } +var _ Subscription = (*SubscriptionImpl)(nil) + type SubscriptionImpl struct { id string @@ -63,7 +66,7 @@ func (sub *SubscriptionImpl) ID() string { return sub.id } -// Channel returns the channel from which subscriptino data can be read +// Channel returns the channel from which subscription data can be read func (sub *SubscriptionImpl) Channel() <-chan interface{} { return sub.ch } @@ -107,6 +110,22 @@ func (sub *SubscriptionImpl) Send(ctx context.Context, v interface{}, timeout ti } } +// NewFailedSubscription returns a new subscription that has already failed with the given error and +// message. This is useful to return an error that occurred during subscription setup. +func NewFailedSubscription(err error, msg string) *SubscriptionImpl { + sub := NewSubscription(0) + + // if error is a grpc error, wrap it to preserve the error code + if st, ok := status.FromError(err); ok { + sub.Fail(status.Errorf(st.Code(), "%s: %s", msg, st.Message())) + return sub + } + + // otherwise, return wrap the message normally + sub.Fail(fmt.Errorf("%s: %w", msg, err)) + return sub +} + var _ Subscription = (*HeightBasedSubscription)(nil) var _ Streamable = (*HeightBasedSubscription)(nil) diff --git a/engine/collection/compliance/core.go b/engine/collection/compliance/core.go index 568ab3fce17..2f5d4eab6b3 100644 --- a/engine/collection/compliance/core.go +++ b/engine/collection/compliance/core.go @@ -12,12 +12,13 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" "github.com/onflow/flow-go/model/cluster" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/compliance" + "github.com/onflow/flow-go/module/counters" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/state" @@ -34,14 +35,15 @@ import ( // - The only exception is calls to `ProcessFinalizedView`, which is the only concurrency-safe // method of compliance.Core type Core struct { - log zerolog.Logger // used to log relevant actions with context - config compliance.Config - engineMetrics module.EngineMetrics - mempoolMetrics module.MempoolMetrics - hotstuffMetrics module.HotstuffMetrics - collectionMetrics module.CollectionMetrics - headers storage.Headers - state clusterkv.MutableState + log zerolog.Logger // used to log relevant actions with context + config compliance.Config + engineMetrics module.EngineMetrics + mempoolMetrics module.MempoolMetrics + hotstuffMetrics module.HotstuffMetrics + collectionMetrics module.CollectionMetrics + proposalViolationNotifier hotstuff.ProposalViolationConsumer + headers storage.Headers + state clusterkv.MutableState // track latest finalized view/height - used to efficiently drop outdated or too-far-ahead blocks finalizedView counters.StrictMonotonousCounter finalizedHeight counters.StrictMonotonousCounter @@ -60,6 +62,7 @@ func NewCore( mempool module.MempoolMetrics, hotstuffMetrics module.HotstuffMetrics, collectionMetrics module.CollectionMetrics, + proposalViolationNotifier hotstuff.ProposalViolationConsumer, headers storage.Headers, state clusterkv.MutableState, pending module.PendingClusterBlockBuffer, @@ -68,29 +71,24 @@ func NewCore( hotstuff module.HotStuff, voteAggregator hotstuff.VoteAggregator, timeoutAggregator hotstuff.TimeoutAggregator, - opts ...compliance.Opt, + config compliance.Config, ) (*Core, error) { - - config := compliance.DefaultConfig() - for _, apply := range opts { - apply(&config) - } - c := &Core{ - log: log.With().Str("cluster_compliance", "core").Logger(), - config: config, - engineMetrics: collector, - mempoolMetrics: mempool, - hotstuffMetrics: hotstuffMetrics, - collectionMetrics: collectionMetrics, - headers: headers, - state: state, - pending: pending, - sync: sync, - hotstuff: hotstuff, - validator: validator, - voteAggregator: voteAggregator, - timeoutAggregator: timeoutAggregator, + log: log.With().Str("cluster_compliance", "core").Logger(), + config: config, + engineMetrics: collector, + mempoolMetrics: mempool, + hotstuffMetrics: hotstuffMetrics, + collectionMetrics: collectionMetrics, + proposalViolationNotifier: proposalViolationNotifier, + headers: headers, + state: state, + pending: pending, + sync: sync, + hotstuff: hotstuff, + validator: validator, + voteAggregator: voteAggregator, + timeoutAggregator: timeoutAggregator, } // initialize finalized boundary cache @@ -108,28 +106,31 @@ func NewCore( // OnBlockProposal handles incoming block proposals. // No errors are expected during normal operation. -func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.ClusterBlockProposal) error { +func (c *Core) OnBlockProposal(proposal flow.Slashable[*messages.ClusterBlockProposal]) error { startTime := time.Now() defer func() { c.hotstuffMetrics.BlockProcessingDuration(time.Since(startTime)) }() - block := proposal.Block.ToInternal() - header := block.Header + block := flow.Slashable[*cluster.Block]{ + OriginID: proposal.OriginID, + Message: proposal.Message.Block.ToInternal(), + } + header := block.Message.Header blockID := header.ID() finalHeight := c.finalizedHeight.Value() finalView := c.finalizedView.Value() log := c.log.With(). - Hex("origin_id", originID[:]). + Hex("origin_id", proposal.OriginID[:]). Str("chain_id", header.ChainID.String()). Uint64("block_height", header.Height). Uint64("block_view", header.View). Hex("block_id", blockID[:]). Hex("parent_id", header.ParentID[:]). - Hex("ref_block_id", block.Payload.ReferenceBlockID[:]). - Hex("collection_id", logging.Entity(block.Payload.Collection)). - Int("tx_count", block.Payload.Collection.Len()). + Hex("ref_block_id", block.Message.Payload.ReferenceBlockID[:]). + Hex("collection_id", logging.Entity(block.Message.Payload.Collection)). + Int("tx_count", block.Message.Payload.Collection.Len()). Time("timestamp", header.Timestamp). Hex("proposer", header.ProposerID[:]). Hex("parent_signer_indices", header.ParentVoterIndices). @@ -137,7 +138,8 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Clus Uint64("finalized_view", finalView). Logger() if log.Debug().Enabled() { - log = log.With().Strs("tx_ids", flow.IdentifierList(block.Payload.Collection.Light().Transactions).Strings()).Logger() + log = log.With().Strs("tx_ids", + flow.IdentifierList(block.Message.Payload.Collection.Light().Transactions).Strings()).Logger() } log.Info().Msg("block proposal received") @@ -147,12 +149,13 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Clus return nil } + skipNewProposalsThreshold := c.config.GetSkipNewProposalsThreshold() // ignore proposals which are too far ahead of our local finalized state // instead, rely on sync engine to catch up finalization more effectively, and avoid // large subtree of blocks to be cached. - if header.View > finalView+c.config.SkipNewProposalsThreshold { + if header.View > finalView+skipNewProposalsThreshold { log.Debug(). - Uint64("skip_new_proposals_threshold", c.config.SkipNewProposalsThreshold). + Uint64("skip_new_proposals_threshold", skipNewProposalsThreshold). Msg("dropping block too far ahead of locally finalized view") return nil } @@ -193,7 +196,7 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Clus _, found := c.pending.ByID(header.ParentID) if found { // add the block to the cache - _ = c.pending.Add(originID, block) + _ = c.pending.Add(block) c.mempoolMetrics.MempoolEntries(metrics.ResourceClusterProposal, c.pending.Size()) return nil @@ -202,27 +205,25 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Clus // if the proposal is connected to a block that is neither in the cache, nor // in persistent storage, its direct parent is missing; cache the proposal // and request the parent - parent, err := c.headers.ByBlockID(header.ParentID) - if errors.Is(err, storage.ErrNotFound) { - _ = c.pending.Add(originID, block) - + exists, err := c.headers.Exists(header.ParentID) + if err != nil { + return fmt.Errorf("could not check parent exists: %w", err) + } + if !exists { + _ = c.pending.Add(block) c.mempoolMetrics.MempoolEntries(metrics.ResourceClusterProposal, c.pending.Size()) c.sync.RequestBlock(header.ParentID, header.Height-1) log.Debug().Msg("requesting missing parent for proposal") return nil } - if err != nil { - return fmt.Errorf("could not check parent: %w", err) - } - // At this point, we should be able to connect the proposal to the finalized // state and should process it to see whether to forward to hotstuff or not. // processBlockAndDescendants is a recursive function. Here we trace the // execution of the entire recursion, which might include processing the // proposal's pending children. There is another span within // processBlockProposal that measures the time spent for a single proposal. - err = c.processBlockAndDescendants(block, parent) + err = c.processBlockAndDescendants(block) c.mempoolMetrics.MempoolEntries(metrics.ResourceClusterProposal, c.pending.Size()) if err != nil { return fmt.Errorf("could not process block proposal: %w", err) @@ -235,24 +236,33 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Clus // its pending descendants. By induction, any child block of a // valid proposal is itself connected to the finalized state and can be // processed as well. -func (c *Core) processBlockAndDescendants(proposal *cluster.Block, parent *flow.Header) error { - blockID := proposal.ID() +func (c *Core) processBlockAndDescendants(proposal flow.Slashable[*cluster.Block]) error { + header := proposal.Message.Header + blockID := header.ID() log := c.log.With(). Str("block_id", blockID.String()). - Uint64("block_height", proposal.Header.Height). - Uint64("block_view", proposal.Header.View). - Uint64("parent_view", parent.View). + Uint64("block_height", header.Height). + Uint64("block_view", header.View). + Uint64("parent_view", header.ParentView). Logger() // process block itself - err := c.processBlockProposal(proposal, parent) + err := c.processBlockProposal(proposal.Message) if err != nil { if checkForAndLogOutdatedInputError(err, log) || checkForAndLogUnverifiableInputError(err, log) { return nil } - if checkForAndLogInvalidInputError(err, log) { + if invalidBlockErr, ok := model.AsInvalidProposalError(err); ok { + log.Err(err).Msg("received invalid block from other node (potential slashing evidence?)") + + // notify consumers about invalid block + c.proposalViolationNotifier.OnInvalidBlockDetected(flow.Slashable[model.InvalidProposalError]{ + OriginID: proposal.OriginID, + Message: *invalidBlockErr, + }) + // notify VoteAggregator about the invalid block - err = c.voteAggregator.InvalidBlock(model.ProposalFromFlow(proposal.Header)) + err = c.voteAggregator.InvalidBlock(model.ProposalFromFlow(header)) if err != nil { if mempool.IsBelowPrunedThresholdError(err) { log.Warn().Msg("received invalid block, but is below pruned threshold") @@ -274,7 +284,7 @@ func (c *Core) processBlockAndDescendants(proposal *cluster.Block, parent *flow. return nil } for _, child := range children { - cpr := c.processBlockAndDescendants(child.Message, proposal.Header) + cpr := c.processBlockAndDescendants(child) if cpr != nil { // unexpected error: potentially corrupted internal state => abort processing and escalate error return cpr @@ -291,9 +301,9 @@ func (c *Core) processBlockAndDescendants(proposal *cluster.Block, parent *flow. // the finalized state. // Expected errors during normal operations: // - engine.OutdatedInputError if the block proposal is outdated (e.g. orphaned) -// - engine.InvalidInputError if the block proposal is invalid +// - model.InvalidProposalError if the block proposal is invalid // - engine.UnverifiableInputError if the proposal cannot be validated -func (c *Core) processBlockProposal(proposal *cluster.Block, parent *flow.Header) error { +func (c *Core) processBlockProposal(proposal *cluster.Block) error { header := proposal.Header blockID := header.ID() log := c.log.With(). @@ -312,13 +322,13 @@ func (c *Core) processBlockProposal(proposal *cluster.Block, parent *flow.Header hotstuffProposal := model.ProposalFromFlow(header) err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { - if model.IsInvalidBlockError(err) { - return engine.NewInvalidInputErrorf("invalid block proposal: %w", err) + if model.IsInvalidProposalError(err) { + return err } if errors.Is(err, model.ErrViewForUnknownEpoch) { // The cluster committee never returns ErrViewForUnknownEpoch, therefore this case // is an unexpected error in cluster consensus. - return fmt.Errorf("unexpected error: cluster committee reported unknown epoch : %w", err) + return fmt.Errorf("unexpected error: cluster committee reported unknown epoch : %w", irrecoverable.NewException(err)) } return fmt.Errorf("unexpected error validating proposal: %w", err) } @@ -328,8 +338,7 @@ func (c *Core) processBlockProposal(proposal *cluster.Block, parent *flow.Header if err != nil { if state.IsInvalidExtensionError(err) { // if the block proposes an invalid extension of the cluster state, then the block is invalid - // TODO: we should slash the block proposer - return engine.NewInvalidInputErrorf("invalid extension of cluster state (block: %x, height: %d): %w", blockID, header.Height, err) + return model.NewInvalidProposalErrorf(hotstuffProposal, "invalid extension of cluster state (block: %x, height: %d): %w", blockID, header.Height, err) } else if state.IsOutdatedExtensionError(err) { // cluster state aborted processing of block as it is on an abandoned fork: block is outdated return engine.NewOutdatedInputErrorf("outdated extension of cluster state: %w", err) @@ -380,19 +389,6 @@ func checkForAndLogOutdatedInputError(err error, log zerolog.Logger) bool { return false } -// checkForAndLogInvalidInputError checks whether error is an `engine.InvalidInputError`. -// If this is the case, we emit a log message and return true. -// For any error other than `engine.InvalidInputError`, this function is a no-op -// and returns false. -func checkForAndLogInvalidInputError(err error, log zerolog.Logger) bool { - if engine.IsInvalidInputError(err) { - // the block is invalid; log as error as we desire honest participation - log.Err(err).Msg("received invalid block from other node (potential slashing evidence?)") - return true - } - return false -} - // checkForAndLogUnverifiableInputError checks whether error is an `engine.UnverifiableInputError`. // If this is the case, we emit a log message and return true. // For any error other than `engine.UnverifiableInputError`, this function is a no-op @@ -400,7 +396,8 @@ func checkForAndLogInvalidInputError(err error, log zerolog.Logger) bool { func checkForAndLogUnverifiableInputError(err error, log zerolog.Logger) bool { if engine.IsUnverifiableInputError(err) { // the block cannot be validated - log.Err(err).Msg("received unverifiable block proposal; this is an indicator of a proposal that cannot be verified under current state") + log.Warn().Err(err).Msg("received collection proposal with unknown reference block; " + + "this might be an indicator that the node is slightly behind or the proposer published an invalid collection") return true } return false diff --git a/engine/collection/compliance/core_test.go b/engine/collection/compliance/core_test.go index ffa490fb31e..2ed1292ee32 100644 --- a/engine/collection/compliance/core_test.go +++ b/engine/collection/compliance/core_test.go @@ -2,9 +2,7 @@ package compliance import ( "errors" - "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -49,25 +47,23 @@ type CommonSuite struct { childrenDB map[flow.Identifier][]flow.Slashable[*cluster.Block] // mocked dependencies - state *clusterstate.MutableState - snapshot *clusterstate.Snapshot - metrics *metrics.NoopCollector - headers *storage.Headers - pending *module.PendingClusterBlockBuffer - hotstuff *module.HotStuff - sync *module.BlockRequester - validator *hotstuff.Validator - voteAggregator *hotstuff.VoteAggregator - timeoutAggregator *hotstuff.TimeoutAggregator + state *clusterstate.MutableState + snapshot *clusterstate.Snapshot + metrics *metrics.NoopCollector + proposalViolationNotifier *hotstuff.ProposalViolationConsumer + headers *storage.Headers + pending *module.PendingClusterBlockBuffer + hotstuff *module.HotStuff + sync *module.BlockRequester + validator *hotstuff.Validator + voteAggregator *hotstuff.VoteAggregator + timeoutAggregator *hotstuff.TimeoutAggregator // engine under test core *Core } func (cs *CommonSuite) SetupTest() { - // seed the RNG - rand.Seed(time.Now().UnixNano()) - block := unittest.ClusterBlockFixture() cs.head = &block @@ -96,6 +92,13 @@ func (cs *CommonSuite) SetupTest() { return nil }, ) + cs.headers.On("Exists", mock.Anything).Return( + func(blockID flow.Identifier) bool { + _, exists := cs.headerDB[blockID] + return exists + }, func(blockID flow.Identifier) error { + return nil + }) // set up protocol state mock cs.state = &clusterstate.MutableState{} @@ -166,6 +169,9 @@ func (cs *CommonSuite) SetupTest() { // set up no-op metrics mock cs.metrics = metrics.NewNoopCollector() + // set up notifier for reporting protocol violations + cs.proposalViolationNotifier = hotstuff.NewProposalViolationConsumer(cs.T()) + // initialize the engine core, err := NewCore( unittest.Logger(), @@ -173,6 +179,7 @@ func (cs *CommonSuite) SetupTest() { cs.metrics, cs.metrics, cs.metrics, + cs.proposalViolationNotifier, cs.headers, cs.state, cs.pending, @@ -181,6 +188,7 @@ func (cs *CommonSuite) SetupTest() { cs.hotstuff, cs.voteAggregator, cs.timeoutAggregator, + compliance.DefaultConfig(), ) require.NoError(cs.T(), err, "engine initialization should pass") @@ -204,7 +212,10 @@ func (cs *CoreSuite) TestOnBlockProposalValidParent() { cs.hotstuff.On("SubmitProposal", hotstuffProposal) // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "valid block proposal should pass") } @@ -227,7 +238,10 @@ func (cs *CoreSuite) TestOnBlockProposalValidAncestor() { cs.hotstuff.On("SubmitProposal", hotstuffProposal).Once() // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "valid block proposal should pass") // we should extend the state with the header @@ -242,7 +256,10 @@ func (cs *CoreSuite) TestOnBlockProposalSkipProposalThreshold() { block.Header.Height = cs.head.Header.Height + compliance.DefaultConfig().SkipNewProposalsThreshold + 1 proposal := unittest.ClusterProposalFromBlock(&block) - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err) // block should be dropped - not added to state or cache @@ -272,12 +289,20 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.Run("invalid block error", func() { // the block fails HotStuff validation *cs.validator = *hotstuff.NewValidator(cs.T()) - cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.InvalidBlockError{}) + sentinelError := model.NewInvalidProposalErrorf(hotstuffProposal, "") + cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError) + cs.proposalViolationNotifier.On("OnInvalidBlockDetected", flow.Slashable[model.InvalidProposalError]{ + OriginID: originID, + Message: sentinelError.(model.InvalidProposalError), + }).Return().Once() // we should notify VoteAggregator about the invalid block cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) // the expected error should be handled within the Core - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "proposal with invalid extension should fail") // we should not extend the state with the header @@ -292,9 +317,12 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.ErrViewForUnknownEpoch) // this error is not expected should raise an exception - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.Error(cs.T(), err, "proposal with invalid extension should fail") - require.ErrorIs(cs.T(), err, model.ErrViewForUnknownEpoch) + require.NotErrorIs(cs.T(), err, model.ErrViewForUnknownEpoch) // we should not extend the state with the header cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything) @@ -309,7 +337,10 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.validator.On("ValidateProposal", hotstuffProposal).Return(unexpectedErr) // the error should be propagated - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.ErrorIs(cs.T(), err, unexpectedErr) // we should not extend the state with the header @@ -345,12 +376,22 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { // make sure we fail to extend the state *cs.state = clusterstate.MutableState{} cs.state.On("Final").Return(func() clusterint.Snapshot { return cs.snapshot }) - cs.state.On("Extend", mock.Anything).Return(state.NewInvalidExtensionError("")) + sentinelErr := state.NewInvalidExtensionError("") + cs.state.On("Extend", mock.Anything).Return(sentinelErr) + cs.proposalViolationNotifier.On("OnInvalidBlockDetected", mock.Anything).Run(func(args mock.Arguments) { + err := args.Get(0).(flow.Slashable[model.InvalidProposalError]) + require.ErrorIs(cs.T(), err.Message, sentinelErr) + require.Equal(cs.T(), err.Message.InvalidProposal, hotstuffProposal) + require.Equal(cs.T(), err.OriginID, originID) + }).Return().Once() // we should notify VoteAggregator about the invalid block cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) // the expected error should be handled within the Core - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "proposal with invalid extension should fail") // we should extend the state with the header @@ -368,7 +409,10 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { cs.state.On("Extend", mock.Anything).Return(state.NewOutdatedExtensionError("")) // the expected error should be handled within the Core - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "proposal with invalid extension should fail") // we should extend the state with the header @@ -387,7 +431,10 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { cs.state.On("Extend", mock.Anything).Return(unexpectedErr) // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.ErrorIs(cs.T(), err, unexpectedErr) // we should extend the state with the header @@ -436,7 +483,10 @@ func (cs *CoreSuite) TestProcessBlockAndDescendants() { } // execute the connected children handling - err := cs.core.processBlockAndDescendants(&parent, cs.head.Header) + err := cs.core.processBlockAndDescendants(flow.Slashable[*cluster.Block]{ + OriginID: unittest.IdentifierFixture(), + Message: &parent, + }) require.NoError(cs.T(), err, "should pass handling children") // check that we submitted each child to hotstuff @@ -481,7 +531,10 @@ func (cs *CoreSuite) TestProposalBufferingOrder() { proposal := messages.NewClusterBlockProposal(block) // process and make sure no error occurs (as they are unverifiable) - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "proposal buffering should pass") // make sure no block is forwarded to hotstuff @@ -513,7 +566,10 @@ func (cs *CoreSuite) TestProposalBufferingOrder() { proposalsLookup[missing.ID()] = missing // process the root proposal - err := cs.core.OnBlockProposal(originID, missingProposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.ClusterBlockProposal]{ + OriginID: originID, + Message: missingProposal, + }) require.NoError(cs.T(), err, "root proposal should pass") // make sure we submitted all four proposals diff --git a/engine/collection/compliance/engine.go b/engine/collection/compliance/engine.go index e49c2dfc35c..c890630982f 100644 --- a/engine/collection/compliance/engine.go +++ b/engine/collection/compliance/engine.go @@ -5,8 +5,8 @@ import ( "github.com/rs/zerolog" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/consensus/hotstuff/tracker" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/collection" "github.com/onflow/flow-go/engine/common/fifoqueue" @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/events" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/state/protocol" @@ -27,18 +28,18 @@ const defaultBlockQueueCapacity = 10_000 // Engine is responsible for handling incoming messages, queueing for processing, broadcasting proposals. // Implements collection.Compliance interface. type Engine struct { - *component.ComponentManager - log zerolog.Logger - metrics module.EngineMetrics - me module.Local - headers storage.Headers - payloads storage.ClusterPayloads - state protocol.State - core *Core - pendingBlocks *fifoqueue.FifoQueue // queue for processing inbound blocks - pendingBlocksNotifier engine.Notifier - finalizedBlockTracker *tracker.NewestBlockTracker - finalizedBlockNotifier engine.Notifier + component.Component + hotstuff.FinalizationConsumer + + log zerolog.Logger + metrics module.EngineMetrics + me module.Local + headers storage.Headers + payloads storage.ClusterPayloads + state protocol.State + core *Core + pendingBlocks *fifoqueue.FifoQueue // queue for processing inbound blocks + pendingBlocksNotifier engine.Notifier } var _ collection.Compliance = (*Engine)(nil) @@ -64,23 +65,23 @@ func NewEngine( } eng := &Engine{ - log: engineLog, - metrics: core.engineMetrics, - me: me, - headers: core.headers, - payloads: payloads, - state: state, - core: core, - pendingBlocks: blocksQueue, - pendingBlocksNotifier: engine.NewNotifier(), - finalizedBlockTracker: tracker.NewNewestBlockTracker(), - finalizedBlockNotifier: engine.NewNotifier(), + log: engineLog, + metrics: core.engineMetrics, + me: me, + headers: core.headers, + payloads: payloads, + state: state, + core: core, + pendingBlocks: blocksQueue, + pendingBlocksNotifier: engine.NewNotifier(), } + finalizationActor, finalizationWorker := events.NewFinalizationActor(eng.processOnFinalizedBlock) + eng.FinalizationConsumer = finalizationActor // create the component manager and worker threads - eng.ComponentManager = component.NewComponentManagerBuilder(). + eng.Component = component.NewComponentManagerBuilder(). AddWorker(eng.processBlocksLoop). - AddWorker(eng.finalizationProcessingLoop). + AddWorker(finalizationWorker). Build() return eng, nil @@ -119,7 +120,7 @@ func (e *Engine) processQueuedBlocks(doneSignal <-chan struct{}) error { msg, ok := e.pendingBlocks.Pop() if ok { inBlock := msg.(flow.Slashable[*messages.ClusterBlockProposal]) - err := e.core.OnBlockProposal(inBlock.OriginID, inBlock.Message) + err := e.core.OnBlockProposal(inBlock) e.core.engineMetrics.MessageHandled(metrics.EngineClusterCompliance, metrics.MessageBlockProposal) if err != nil { return fmt.Errorf("could not handle block proposal: %w", err) @@ -133,17 +134,6 @@ func (e *Engine) processQueuedBlocks(doneSignal <-chan struct{}) error { } } -// OnFinalizedBlock implements the `OnFinalizedBlock` callback from the `hotstuff.FinalizationConsumer` -// It informs compliance.Core about finalization of the respective block. -// -// CAUTION: the input to this callback is treated as trusted; precautions should be taken that messages -// from external nodes cannot be considered as inputs to this function -func (e *Engine) OnFinalizedBlock(block *model.Block) { - if e.finalizedBlockTracker.Track(block) { - e.finalizedBlockNotifier.Notify() - } -} - // OnClusterBlockProposal feeds a new block proposal into the processing pipeline. // Incoming proposals are queued and eventually dispatched by worker. func (e *Engine) OnClusterBlockProposal(proposal flow.Slashable[*messages.ClusterBlockProposal]) { @@ -166,23 +156,16 @@ func (e *Engine) OnSyncedClusterBlock(syncedBlock flow.Slashable[*messages.Clust } } -// finalizationProcessingLoop is a separate goroutine that performs processing of finalization events -func (e *Engine) finalizationProcessingLoop(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - ready() - - doneSignal := ctx.Done() - blockFinalizedSignal := e.finalizedBlockNotifier.Channel() - for { - select { - case <-doneSignal: - return - case <-blockFinalizedSignal: - // retrieve the latest finalized header, so we know the height - finalHeader, err := e.headers.ByBlockID(e.finalizedBlockTracker.NewestBlock().BlockID) - if err != nil { // no expected errors - ctx.Throw(err) - } - e.core.ProcessFinalizedBlock(finalHeader) - } +// processOnFinalizedBlock informs compliance.Core about finalization of the respective block. +// The input to this callback is treated as trusted. This method should be executed on +// `OnFinalizedBlock` notifications from the node-internal consensus instance. +// No errors expected during normal operations. +func (e *Engine) processOnFinalizedBlock(block *model.Block) error { + // retrieve the latest finalized header, so we know the height + finalHeader, err := e.headers.ByBlockID(block.BlockID) + if err != nil { // no expected errors + return fmt.Errorf("could not get finalized header: %w", err) } + e.core.ProcessFinalizedBlock(finalHeader) + return nil } diff --git a/engine/collection/compliance/engine_test.go b/engine/collection/compliance/engine_test.go index cdc470b9dbb..3c760ed05c3 100644 --- a/engine/collection/compliance/engine_test.go +++ b/engine/collection/compliance/engine_test.go @@ -165,8 +165,6 @@ func (cs *EngineSuite) TestSubmittingMultipleEntries() { for i := 0; i < blockCount; i++ { block := unittest.ClusterBlockWithParent(cs.head) proposal := messages.NewClusterBlockProposal(&block) - // store the data for retrieval - cs.headerDB[block.Header.ParentID] = cs.head hotstuffProposal := model.ProposalFromFlow(block.Header) cs.hotstuff.On("SubmitProposal", hotstuffProposal).Return().Once() cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() @@ -185,8 +183,6 @@ func (cs *EngineSuite) TestSubmittingMultipleEntries() { block := unittest.ClusterBlockWithParent(cs.head) proposal := messages.NewClusterBlockProposal(&block) - // store the data for retrieval - cs.headerDB[block.Header.ParentID] = cs.head hotstuffProposal := model.ProposalFromFlow(block.Header) cs.hotstuff.On("SubmitProposal", hotstuffProposal).Once() cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() @@ -224,6 +220,7 @@ func (cs *EngineSuite) TestOnFinalizedBlock() { Run(func(_ mock.Arguments) { wg.Done() }). Return(uint(0)).Once() - cs.engine.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock.Header)) + err := cs.engine.processOnFinalizedBlock(model.BlockFromFlow(finalizedBlock.Header)) + require.NoError(cs.T(), err) unittest.AssertReturnsBefore(cs.T(), wg.Wait, time.Second, "an expected call to block buffer wasn't made") } diff --git a/engine/collection/epochmgr/engine.go b/engine/collection/epochmgr/engine.go index eee3891dc1a..5ce5184e7b1 100644 --- a/engine/collection/epochmgr/engine.go +++ b/engine/collection/epochmgr/engine.go @@ -8,6 +8,7 @@ import ( "github.com/rs/zerolog" + "github.com/onflow/flow-go/engine/collection" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" @@ -56,11 +57,11 @@ type Engine struct { epochs map[uint64]*RunningEpochComponents // epoch-scoped components per epoch // internal event notifications - epochTransitionEvents chan *flow.Header // sends first block of new epoch - epochSetupPhaseStartedEvents chan *flow.Header // sends first block of EpochSetup phase - epochStopEvents chan uint64 // sends counter of epoch to stop - - cm *component.ComponentManager + epochTransitionEvents chan *flow.Header // sends first block of new epoch + epochSetupPhaseStartedEvents chan *flow.Header // sends first block of EpochSetup phase + epochStopEvents chan uint64 // sends counter of epoch to stop + clusterIDUpdateDistributor collection.ClusterEvents // sends cluster ID updates to consumers + cm *component.ComponentManager component.Component } @@ -75,6 +76,7 @@ func New( voter module.ClusterRootQCVoter, factory EpochComponentsFactory, heightEvents events.Heights, + clusterIDUpdateDistributor collection.ClusterEvents, ) (*Engine, error) { e := &Engine{ log: log.With().Str("engine", "epochmgr").Logger(), @@ -89,6 +91,7 @@ func New( epochTransitionEvents: make(chan *flow.Header, 1), epochSetupPhaseStartedEvents: make(chan *flow.Header, 1), epochStopEvents: make(chan uint64, 1), + clusterIDUpdateDistributor: clusterIDUpdateDistributor, } e.cm = component.NewComponentManagerBuilder(). @@ -448,7 +451,6 @@ func (e *Engine) onEpochSetupPhaseStarted(ctx irrecoverable.SignalerContext, nex // No errors are expected during normal operation. func (e *Engine) startEpochComponents(engineCtx irrecoverable.SignalerContext, counter uint64, components *EpochComponents) error { epochCtx, cancel, errCh := irrecoverable.WithSignallerAndCancel(engineCtx) - // start component using its own context components.Start(epochCtx) go e.handleEpochErrors(engineCtx, errCh) @@ -456,6 +458,11 @@ func (e *Engine) startEpochComponents(engineCtx irrecoverable.SignalerContext, c select { case <-components.Ready(): e.storeEpochComponents(counter, NewRunningEpochComponents(components, cancel)) + activeClusterIDS, err := e.activeClusterIDs() + if err != nil { + return fmt.Errorf("failed to get active cluster IDs: %w", err) + } + e.clusterIDUpdateDistributor.ActiveClustersChanged(activeClusterIDS) return nil case <-time.After(e.startupTimeout): cancel() // cancel current context if we didn't start in time @@ -481,6 +488,11 @@ func (e *Engine) stopEpochComponents(counter uint64) error { case <-components.Done(): e.removeEpoch(counter) e.pools.ForEpoch(counter).Clear() + activeClusterIDS, err := e.activeClusterIDs() + if err != nil { + return fmt.Errorf("failed to get active cluster IDs: %w", err) + } + e.clusterIDUpdateDistributor.ActiveClustersChanged(activeClusterIDS) return nil case <-time.After(e.startupTimeout): return fmt.Errorf("could not stop epoch %d components after %s", counter, e.startupTimeout) @@ -512,3 +524,19 @@ func (e *Engine) removeEpoch(counter uint64) { delete(e.epochs, counter) e.mu.Unlock() } + +// activeClusterIDs returns the active canonical cluster ID's for the assigned collection clusters. +// No errors are expected during normal operation. +func (e *Engine) activeClusterIDs() (flow.ChainIDList, error) { + e.mu.RLock() + defer e.mu.RUnlock() + clusterIDs := make(flow.ChainIDList, 0) + for _, epoch := range e.epochs { + chainID, err := epoch.state.Params().ChainID() // cached, does not hit database + if err != nil { + return nil, fmt.Errorf("failed to get active cluster ids: %w", err) + } + clusterIDs = append(clusterIDs, chainID) + } + return clusterIDs, nil +} diff --git a/engine/collection/epochmgr/engine_test.go b/engine/collection/epochmgr/engine_test.go index e477c9a9256..b7882031ba6 100644 --- a/engine/collection/epochmgr/engine_test.go +++ b/engine/collection/epochmgr/engine_test.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" mockhotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" epochmgr "github.com/onflow/flow-go/engine/collection/epochmgr/mock" + mockcollection "github.com/onflow/flow-go/engine/collection/mock" "github.com/onflow/flow-go/model/flow" realmodule "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" @@ -45,7 +46,6 @@ type mockComponents struct { } func newMockComponents(t *testing.T) *mockComponents { - components := &mockComponents{ state: cluster.NewState(t), prop: mockcomponent.NewComponent(t), @@ -67,7 +67,9 @@ func newMockComponents(t *testing.T) *mockComponents { components.voteAggregator.On("Start", mock.Anything) components.timeoutAggregator.On("Start", mock.Anything) components.messageHub.On("Start", mock.Anything) - + params := cluster.NewParams(t) + params.On("ChainID").Return(flow.ChainID("chain-id"), nil).Maybe() + components.state.On("Params").Return(params).Maybe() return components } @@ -100,6 +102,8 @@ type Suite struct { errs <-chan error engine *Engine + + engineEventsDistributor *mockcollection.EngineEvents } // MockFactoryCreate mocks the epoch factory to create epoch components for the given epoch. @@ -149,6 +153,7 @@ func (suite *Suite) SetupTest() { suite.phase = flow.EpochPhaseSetup suite.header = unittest.BlockHeaderFixture() suite.epochQuery = mocks.NewEpochQuery(suite.T(), suite.counter) + suite.state.On("Final").Return(suite.snap) suite.state.On("AtBlockID", suite.header.ID()).Return(suite.snap).Maybe() suite.snap.On("Epochs").Return(suite.epochQuery) @@ -167,9 +172,12 @@ func (suite *Suite) SetupTest() { return herocache.NewTransactions(1000, suite.log, metrics.NewNoopCollector()) }) + suite.engineEventsDistributor = mockcollection.NewEngineEvents(suite.T()) + var err error - suite.engine, err = New(suite.log, suite.me, suite.state, suite.pools, suite.voter, suite.factory, suite.heights) + suite.engine, err = New(suite.log, suite.me, suite.state, suite.pools, suite.voter, suite.factory, suite.heights, suite.engineEventsDistributor) suite.Require().Nil(err) + } // StartEngine starts the engine under test, and spawns a routine to check for irrecoverable errors. @@ -258,13 +266,16 @@ func (suite *Suite) MockAsUnauthorizedNode(forEpoch uint64) { suite.MockFactoryCreate(mock.MatchedBy(authorizedMatcher)) var err error - suite.engine, err = New(suite.log, suite.me, suite.state, suite.pools, suite.voter, suite.factory, suite.heights) + suite.engine, err = New(suite.log, suite.me, suite.state, suite.pools, suite.voter, suite.factory, suite.heights, suite.engineEventsDistributor) suite.Require().Nil(err) } // TestRestartInSetupPhase tests that, if we start up during the setup phase, // we should kick off the root QC voter func (suite *Suite) TestRestartInSetupPhase() { + // we expect 1 ActiveClustersChanged events when the engine first starts and the first set of epoch components are started + suite.engineEventsDistributor.On("ActiveClustersChanged", mock.AnythingOfType("flow.ChainIDList")).Once() + defer suite.engineEventsDistributor.AssertExpectations(suite.T()) // we are in setup phase suite.phase = flow.EpochPhaseSetup // should call voter with next epoch @@ -285,6 +296,9 @@ func (suite *Suite) TestRestartInSetupPhase() { // When the finalized height is within the first tx_expiry blocks of the new epoch // the engine should restart the previous epoch cluster consensus. func (suite *Suite) TestStartAfterEpochBoundary_WithinTxExpiry() { + // we expect 2 ActiveClustersChanged events once when the engine first starts and the first set of epoch components are started and on restart + suite.engineEventsDistributor.On("ActiveClustersChanged", mock.AnythingOfType("flow.ChainIDList")).Twice() + defer suite.engineEventsDistributor.AssertExpectations(suite.T()) suite.phase = flow.EpochPhaseStaking // transition epochs, so that a Previous epoch is queryable suite.TransitionEpoch() @@ -305,6 +319,9 @@ func (suite *Suite) TestStartAfterEpochBoundary_WithinTxExpiry() { // When the finalized height is beyond the first tx_expiry blocks of the new epoch // the engine should NOT restart the previous epoch cluster consensus. func (suite *Suite) TestStartAfterEpochBoundary_BeyondTxExpiry() { + // we expect 1 ActiveClustersChanged events when the engine first starts and the first set of epoch components are started + suite.engineEventsDistributor.On("ActiveClustersChanged", mock.AnythingOfType("flow.ChainIDList")).Once() + defer suite.engineEventsDistributor.AssertExpectations(suite.T()) suite.phase = flow.EpochPhaseStaking // transition epochs, so that a Previous epoch is queryable suite.TransitionEpoch() @@ -325,6 +342,9 @@ func (suite *Suite) TestStartAfterEpochBoundary_BeyondTxExpiry() { // boundary that we could start the previous epoch cluster consensus - however, // since we are not approved for the epoch, we should only start current epoch components. func (suite *Suite) TestStartAfterEpochBoundary_NotApprovedForPreviousEpoch() { + // we expect 1 ActiveClustersChanged events when the current epoch components are started + suite.engineEventsDistributor.On("ActiveClustersChanged", mock.AnythingOfType("flow.ChainIDList")).Once() + defer suite.engineEventsDistributor.AssertExpectations(suite.T()) suite.phase = flow.EpochPhaseStaking // transition epochs, so that a Previous epoch is queryable suite.TransitionEpoch() @@ -346,6 +366,9 @@ func (suite *Suite) TestStartAfterEpochBoundary_NotApprovedForPreviousEpoch() { // boundary that we should start the previous epoch cluster consensus. However, we are // not approved for the current epoch -> we should only start *current* epoch components. func (suite *Suite) TestStartAfterEpochBoundary_NotApprovedForCurrentEpoch() { + // we expect 1 ActiveClustersChanged events when the current epoch components are started + suite.engineEventsDistributor.On("ActiveClustersChanged", mock.AnythingOfType("flow.ChainIDList")).Once() + defer suite.engineEventsDistributor.AssertExpectations(suite.T()) suite.phase = flow.EpochPhaseStaking // transition epochs, so that a Previous epoch is queryable suite.TransitionEpoch() @@ -393,6 +416,10 @@ func (suite *Suite) TestStartAsUnauthorizedNode() { // TestRespondToPhaseChange should kick off root QC voter when we receive an event // indicating the EpochSetup phase has started. func (suite *Suite) TestRespondToPhaseChange() { + // we expect 1 ActiveClustersChanged events when the engine first starts and the first set of epoch components are started + suite.engineEventsDistributor.On("ActiveClustersChanged", mock.AnythingOfType("flow.ChainIDList")).Once() + defer suite.engineEventsDistributor.AssertExpectations(suite.T()) + // start in staking phase suite.phase = flow.EpochPhaseStaking // should call voter with next epoch @@ -418,6 +445,13 @@ func (suite *Suite) TestRespondToPhaseChange() { // - register callback to stop the previous epoch's cluster consensus // - stop the previous epoch's cluster consensus when the callback is invoked func (suite *Suite) TestRespondToEpochTransition() { + // we expect 3 ActiveClustersChanged events + // - once when the engine first starts and the first set of epoch components are started + // - once when the epoch transitions and the new set of epoch components are started + // - once when the epoch transitions and the old set of epoch components are stopped + expectedNumOfEvents := 3 + suite.engineEventsDistributor.On("ActiveClustersChanged", mock.AnythingOfType("flow.ChainIDList")).Times(expectedNumOfEvents) + defer suite.engineEventsDistributor.AssertExpectations(suite.T()) // we are in committed phase suite.phase = flow.EpochPhaseCommitted diff --git a/engine/collection/epochmgr/factories/builder.go b/engine/collection/epochmgr/factories/builder.go index 53eb96f31f2..a00a73ac97e 100644 --- a/engine/collection/epochmgr/factories/builder.go +++ b/engine/collection/epochmgr/factories/builder.go @@ -11,11 +11,14 @@ import ( finalizer "github.com/onflow/flow-go/module/finalizer/collection" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/network" + clusterstate "github.com/onflow/flow-go/state/cluster" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) type BuilderFactory struct { db *badger.DB + protoState protocol.State mainChainHeaders storage.Headers trace module.Tracer opts []builder.Opt @@ -26,6 +29,7 @@ type BuilderFactory struct { func NewBuilderFactory( db *badger.DB, + protoState protocol.State, mainChainHeaders storage.Headers, trace module.Tracer, metrics module.CollectionMetrics, @@ -36,6 +40,7 @@ func NewBuilderFactory( factory := &BuilderFactory{ db: db, + protoState: protoState, mainChainHeaders: mainChainHeaders, trace: trace, metrics: metrics, @@ -47,19 +52,24 @@ func NewBuilderFactory( } func (f *BuilderFactory) Create( + clusterState clusterstate.State, clusterHeaders storage.Headers, clusterPayloads storage.ClusterPayloads, pool mempool.Transactions, + epoch uint64, ) (module.Builder, *finalizer.Finalizer, error) { build, err := builder.NewBuilder( f.db, f.trace, + f.protoState, + clusterState, f.mainChainHeaders, clusterHeaders, clusterPayloads, pool, f.log, + epoch, f.opts..., ) if err != nil { diff --git a/engine/collection/epochmgr/factories/cluster_state.go b/engine/collection/epochmgr/factories/cluster_state.go index 52e6f8f19f7..7f786f4ff36 100644 --- a/engine/collection/epochmgr/factories/cluster_state.go +++ b/engine/collection/epochmgr/factories/cluster_state.go @@ -47,7 +47,7 @@ func (f *ClusterStateFactory) Create(stateRoot *clusterkv.StateRoot) ( } var clusterState *clusterkv.State if isBootStrapped { - clusterState, err = clusterkv.OpenState(f.db, f.tracer, headers, payloads, stateRoot.ClusterID()) + clusterState, err = clusterkv.OpenState(f.db, f.tracer, headers, payloads, stateRoot.ClusterID(), stateRoot.EpochCounter()) if err != nil { return nil, nil, nil, nil, fmt.Errorf("could not open cluster state: %w", err) } diff --git a/engine/collection/epochmgr/factories/compliance.go b/engine/collection/epochmgr/factories/compliance.go index 777a5db03b6..8988c8d3615 100644 --- a/engine/collection/epochmgr/factories/compliance.go +++ b/engine/collection/epochmgr/factories/compliance.go @@ -26,7 +26,7 @@ type ComplianceEngineFactory struct { mempoolMetrics module.MempoolMetrics protoState protocol.State transactions storage.Transactions - complianceOpts []modulecompliance.Opt + config modulecompliance.Config } // NewComplianceEngineFactory returns a new collection compliance engine factory. @@ -39,7 +39,7 @@ func NewComplianceEngineFactory( mempoolMetrics module.MempoolMetrics, protoState protocol.State, transactions storage.Transactions, - complianceOpts ...modulecompliance.Opt, + config modulecompliance.Config, ) (*ComplianceEngineFactory, error) { factory := &ComplianceEngineFactory{ @@ -51,13 +51,14 @@ func NewComplianceEngineFactory( mempoolMetrics: mempoolMetrics, protoState: protoState, transactions: transactions, - complianceOpts: complianceOpts, + config: config, } return factory, nil } func (f *ComplianceEngineFactory) Create( hotstuffMetrics module.HotstuffMetrics, + notifier hotstuff.ProposalViolationConsumer, clusterState cluster.MutableState, headers storage.Headers, payloads storage.ClusterPayloads, @@ -75,6 +76,7 @@ func (f *ComplianceEngineFactory) Create( f.mempoolMetrics, hotstuffMetrics, f.colMetrics, + notifier, headers, clusterState, cache, @@ -83,7 +85,7 @@ func (f *ComplianceEngineFactory) Create( hot, voteAggregator, timeoutAggregator, - f.complianceOpts..., + f.config, ) if err != nil { return nil, fmt.Errorf("could create cluster compliance core: %w", err) diff --git a/engine/collection/epochmgr/factories/epoch.go b/engine/collection/epochmgr/factories/epoch.go index ca5bb9b03e4..25f6c42ab89 100644 --- a/engine/collection/epochmgr/factories/epoch.go +++ b/engine/collection/epochmgr/factories/epoch.go @@ -67,7 +67,7 @@ func (factory *EpochComponentsFactory) Create( err error, ) { - counter, err := epoch.Counter() + epochCounter, err := epoch.Counter() if err != nil { err = fmt.Errorf("could not get epoch counter: %w", err) return @@ -81,7 +81,7 @@ func (factory *EpochComponentsFactory) Create( } _, exists := identities.ByNodeID(factory.me.NodeID()) if !exists { - err = fmt.Errorf("%w (node_id=%x, epoch=%d)", epochmgr.ErrNotAuthorizedForEpoch, factory.me.NodeID(), counter) + err = fmt.Errorf("%w (node_id=%x, epoch=%d)", epochmgr.ErrNotAuthorizedForEpoch, factory.me.NodeID(), epochCounter) return } @@ -109,7 +109,7 @@ func (factory *EpochComponentsFactory) Create( blocks storage.ClusterBlocks ) - stateRoot, err := badger.NewStateRoot(cluster.RootBlock(), cluster.RootQC()) + stateRoot, err := badger.NewStateRoot(cluster.RootBlock(), cluster.RootQC(), cluster.EpochCounter()) if err != nil { err = fmt.Errorf("could not create valid state root: %w", err) return @@ -123,9 +123,9 @@ func (factory *EpochComponentsFactory) Create( } // get the transaction pool for the epoch - pool := factory.pools.ForEpoch(counter) + pool := factory.pools.ForEpoch(epochCounter) - builder, finalizer, err := factory.builder.Create(headers, payloads, pool) + builder, finalizer, err := factory.builder.Create(state, headers, payloads, pool, epochCounter) if err != nil { err = fmt.Errorf("could not create builder/finalizer: %w", err) return @@ -161,6 +161,7 @@ func (factory *EpochComponentsFactory) Create( complianceEng, err := factory.compliance.Create( metrics, + hotstuffModules.Notifier, mutableState, headers, payloads, @@ -175,7 +176,7 @@ func (factory *EpochComponentsFactory) Create( return } compliance = complianceEng - hotstuffModules.FinalizationDistributor.AddOnBlockFinalizedConsumer(complianceEng.OnFinalizedBlock) + hotstuffModules.Notifier.AddOnBlockFinalizedConsumer(complianceEng.OnFinalizedBlock) sync, err = factory.sync.Create(cluster.Members(), state, blocks, syncCore, complianceEng) if err != nil { diff --git a/engine/collection/epochmgr/factories/hotstuff.go b/engine/collection/epochmgr/factories/hotstuff.go index 9b27bfc7201..05bc6c0ebfa 100644 --- a/engine/collection/epochmgr/factories/hotstuff.go +++ b/engine/collection/epochmgr/factories/hotstuff.go @@ -75,13 +75,13 @@ func (f *HotStuffFactory) CreateModules( // setup metrics/logging with the new chain ID log := f.createLogger(cluster) metrics := f.createMetrics(cluster.ChainID()) + telemetryConsumer := notifications.NewTelemetryConsumer(log) + slashingConsumer := notifications.NewSlashingViolationsConsumer(log) notifier := pubsub.NewDistributor() - finalizationDistributor := pubsub.NewFinalizationDistributor() - notifier.AddConsumer(finalizationDistributor) notifier.AddConsumer(notifications.NewLogConsumer(log)) notifier.AddConsumer(hotmetrics.NewMetricsConsumer(metrics)) - notifier.AddConsumer(notifications.NewTelemetryConsumer(log)) - notifier.AddConsumer(notifications.NewSlashingViolationsConsumer(log)) + notifier.AddParticipantConsumer(telemetryConsumer) + notifier.AddProposalViolationConsumer(slashingConsumer) var ( err error @@ -114,11 +114,13 @@ func (f *HotStuffFactory) CreateModules( return nil, nil, err } - qcDistributor := pubsub.NewQCCreatedDistributor() + voteAggregationDistributor := pubsub.NewVoteAggregationDistributor() + voteAggregationDistributor.AddVoteCollectorConsumer(telemetryConsumer) + voteAggregationDistributor.AddVoteAggregationViolationConsumer(slashingConsumer) verifier := verification.NewStakingVerifier() validator := validatorImpl.NewMetricsWrapper(validatorImpl.New(committee, verifier), metrics) - voteProcessorFactory := votecollector.NewStakingVoteProcessorFactory(committee, qcDistributor.OnQcConstructedFromVotes) + voteProcessorFactory := votecollector.NewStakingVoteProcessorFactory(committee, voteAggregationDistributor.OnQcConstructedFromVotes) voteAggregator, err := consensus.NewVoteAggregator( log, metrics, @@ -127,17 +129,19 @@ func (f *HotStuffFactory) CreateModules( // since we don't want to aggregate votes for finalized view, // the lowest retained view starts with the next view of the last finalized view. finalizedBlock.View+1, - notifier, + voteAggregationDistributor, voteProcessorFactory, - finalizationDistributor, + notifier.FollowerDistributor, ) if err != nil { return nil, nil, err } - timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() - timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory(log, timeoutCollectorDistributor, committee, validator, msig.CollectorTimeoutTag) + timeoutCollectorDistributor := pubsub.NewTimeoutAggregationDistributor() + timeoutCollectorDistributor.AddTimeoutCollectorConsumer(telemetryConsumer) + timeoutCollectorDistributor.AddTimeoutAggregationViolationConsumer(slashingConsumer) + timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory(log, timeoutCollectorDistributor, committee, validator, msig.CollectorTimeoutTag) timeoutAggregator, err := consensus.NewTimeoutAggregator( log, metrics, @@ -161,9 +165,8 @@ func (f *HotStuffFactory) CreateModules( Persist: persister.New(f.db, cluster.ChainID()), VoteAggregator: voteAggregator, TimeoutAggregator: timeoutAggregator, - QCCreatedDistributor: qcDistributor, - TimeoutCollectorDistributor: timeoutCollectorDistributor, - FinalizationDistributor: finalizationDistributor, + VoteCollectorDistributor: voteAggregationDistributor.VoteCollectorDistributor, + TimeoutCollectorDistributor: timeoutCollectorDistributor.TimeoutCollectorDistributor, }, metrics, nil } @@ -188,6 +191,7 @@ func (f *HotStuffFactory) Create( participant, err := consensus.NewParticipant( log, metrics, + f.mempoolMetrics, builder, finalizedBlock, pendingBlocks, diff --git a/engine/collection/events.go b/engine/collection/events.go new file mode 100644 index 00000000000..1c5809806b4 --- /dev/null +++ b/engine/collection/events.go @@ -0,0 +1,19 @@ +package collection + +import "github.com/onflow/flow-go/model/flow" + +// EngineEvents set of methods used to distribute and consume events related to collection node engine components. +type EngineEvents interface { + ClusterEvents +} + +// ClusterEvents defines methods used to disseminate cluster ID update events. +// Cluster IDs are updated when a new set of epoch components start and the old set of epoch components stops. +// A new list of cluster IDs will be assigned when the new set of epoch components are started, and the old set of cluster +// IDs are removed when the current set of epoch components are stopped. The implementation must be concurrency safe and non-blocking. +type ClusterEvents interface { + // ActiveClustersChanged is called when a new cluster ID update event is distributed. + // Any error encountered on consuming event must handle internally by the implementation. + // The implementation must be concurrency safe, but can be blocking. + ActiveClustersChanged(flow.ChainIDList) +} diff --git a/engine/collection/events/cluster_events_distributor.go b/engine/collection/events/cluster_events_distributor.go new file mode 100644 index 00000000000..caff0ebd26a --- /dev/null +++ b/engine/collection/events/cluster_events_distributor.go @@ -0,0 +1,36 @@ +package events + +import ( + "sync" + + "github.com/onflow/flow-go/engine/collection" + "github.com/onflow/flow-go/model/flow" +) + +// ClusterEventsDistributor distributes cluster events to a list of subscribers. +type ClusterEventsDistributor struct { + subscribers []collection.ClusterEvents + mu sync.RWMutex +} + +var _ collection.ClusterEvents = (*ClusterEventsDistributor)(nil) + +// NewClusterEventsDistributor returns a new events *ClusterEventsDistributor. +func NewClusterEventsDistributor() *ClusterEventsDistributor { + return &ClusterEventsDistributor{} +} + +func (d *ClusterEventsDistributor) AddConsumer(consumer collection.ClusterEvents) { + d.mu.Lock() + defer d.mu.Unlock() + d.subscribers = append(d.subscribers, consumer) +} + +// ActiveClustersChanged distributes events to all subscribers. +func (d *ClusterEventsDistributor) ActiveClustersChanged(list flow.ChainIDList) { + d.mu.RLock() + defer d.mu.RUnlock() + for _, sub := range d.subscribers { + sub.ActiveClustersChanged(list) + } +} diff --git a/engine/collection/events/distributor.go b/engine/collection/events/distributor.go new file mode 100644 index 00000000000..39e723f30db --- /dev/null +++ b/engine/collection/events/distributor.go @@ -0,0 +1,23 @@ +package events + +import ( + "github.com/onflow/flow-go/engine/collection" +) + +// CollectionEngineEventsDistributor set of structs that implement all collection engine event interfaces. +type CollectionEngineEventsDistributor struct { + *ClusterEventsDistributor +} + +var _ collection.EngineEvents = (*CollectionEngineEventsDistributor)(nil) + +// NewDistributor returns a new *CollectionEngineEventsDistributor. +func NewDistributor() *CollectionEngineEventsDistributor { + return &CollectionEngineEventsDistributor{ + ClusterEventsDistributor: NewClusterEventsDistributor(), + } +} + +func (d *CollectionEngineEventsDistributor) AddConsumer(consumer collection.EngineEvents) { + d.ClusterEventsDistributor.AddConsumer(consumer) +} diff --git a/engine/collection/message_hub/message_hub.go b/engine/collection/message_hub/message_hub.go index ee1dc26ff05..6c73ec2ab22 100644 --- a/engine/collection/message_hub/message_hub.go +++ b/engine/collection/message_hub/message_hub.go @@ -440,7 +440,12 @@ func (h *MessageHub) Process(channel channels.Channel, originID flow.Identifier, } h.forwardToOwnTimeoutAggregator(t) default: - h.log.Warn().Msgf("%v delivered unsupported message %T through %v", originID, message, channel) + h.log.Warn(). + Bool(logging.KeySuspicious, true). + Hex("origin_id", logging.ID(originID)). + Str("message_type", fmt.Sprintf("%T", message)). + Str("channel", channel.String()). + Msgf("delivered unsupported message type") } return nil } diff --git a/engine/collection/message_hub/message_hub_test.go b/engine/collection/message_hub/message_hub_test.go index 9d574082475..7e60e4d7877 100644 --- a/engine/collection/message_hub/message_hub_test.go +++ b/engine/collection/message_hub/message_hub_test.go @@ -2,7 +2,6 @@ package message_hub import ( "context" - "math/rand" "sync" "testing" "time" @@ -68,9 +67,6 @@ type MessageHubSuite struct { } func (s *MessageHubSuite) SetupTest() { - // seed the RNG - rand.Seed(time.Now().UnixNano()) - // initialize the paramaters s.cluster = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleCollection), diff --git a/engine/collection/mock/cluster_events.go b/engine/collection/mock/cluster_events.go new file mode 100644 index 00000000000..a17e4db4a9a --- /dev/null +++ b/engine/collection/mock/cluster_events.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// ClusterEvents is an autogenerated mock type for the ClusterEvents type +type ClusterEvents struct { + mock.Mock +} + +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *ClusterEvents) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewClusterEvents interface { + mock.TestingT + Cleanup(func()) +} + +// NewClusterEvents creates a new instance of ClusterEvents. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewClusterEvents(t mockConstructorTestingTNewClusterEvents) *ClusterEvents { + mock := &ClusterEvents{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/collection/mock/engine_events.go b/engine/collection/mock/engine_events.go new file mode 100644 index 00000000000..4eb5ce20268 --- /dev/null +++ b/engine/collection/mock/engine_events.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// EngineEvents is an autogenerated mock type for the EngineEvents type +type EngineEvents struct { + mock.Mock +} + +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *EngineEvents) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewEngineEvents interface { + mock.TestingT + Cleanup(func()) +} + +// NewEngineEvents creates a new instance of EngineEvents. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewEngineEvents(t mockConstructorTestingTNewEngineEvents) *EngineEvents { + mock := &EngineEvents{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/collection/synchronization/engine.go b/engine/collection/synchronization/engine.go index 77ebdbd7792..8c6fbafa806 100644 --- a/engine/collection/synchronization/engine.go +++ b/engine/collection/synchronization/engine.go @@ -5,7 +5,6 @@ package synchronization import ( "errors" "fmt" - "math/rand" "time" "github.com/hashicorp/go-multierror" @@ -27,6 +26,7 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/rand" ) // defaultSyncResponseQueueCapacity maximum capacity of sync responses queue @@ -361,9 +361,19 @@ func (e *Engine) pollHeight() { return } + nonce, err := rand.Uint64() + if err != nil { + // TODO: this error should be returned by pollHeight() + // it is logged for now since the only error possible is related to a failure + // of the system entropy generation. Such error is going to cause failures in other + // components where it's handled properly and will lead to crashing the module. + e.log.Error().Err(err).Msg("nonce generation failed during pollHeight") + return + } + // send the request for synchronization req := &messages.SyncRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, Height: head.Height, } err = e.con.Multicast(req, synccore.DefaultPollNodes, e.participants.NodeIDs()...) @@ -379,12 +389,21 @@ func (e *Engine) sendRequests(ranges []chainsync.Range, batches []chainsync.Batc var errs *multierror.Error for _, ran := range ranges { + nonce, err := rand.Uint64() + if err != nil { + // TODO: this error should be returned by sendRequests + // it is logged for now since the only error possible is related to a failure + // of the system entropy generation. Such error is going to cause failures in other + // components where it's handled properly and will lead to crashing the module. + e.log.Error().Err(err).Msg("nonce generation failed during range request") + return + } req := &messages.RangeRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, FromHeight: ran.From, ToHeight: ran.To, } - err := e.con.Multicast(req, synccore.DefaultBlockRequestNodes, e.participants.NodeIDs()...) + err = e.con.Multicast(req, synccore.DefaultBlockRequestNodes, e.participants.NodeIDs()...) if err != nil { errs = multierror.Append(errs, fmt.Errorf("could not submit range request: %w", err)) continue @@ -399,11 +418,20 @@ func (e *Engine) sendRequests(ranges []chainsync.Range, batches []chainsync.Batc } for _, batch := range batches { + nonce, err := rand.Uint64() + if err != nil { + // TODO: this error should be returned by sendRequests + // it is logged for now since the only error possible is related to a failure + // of the system entropy generation. Such error is going to cause failures in other + // components where it's handled properly and will lead to crashing the module. + e.log.Error().Err(err).Msg("nonce generation failed during batch request") + return + } req := &messages.BatchRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, BlockIDs: batch.BlockIDs, } - err := e.con.Multicast(req, synccore.DefaultBlockRequestNodes, e.participants.NodeIDs()...) + err = e.con.Multicast(req, synccore.DefaultBlockRequestNodes, e.participants.NodeIDs()...) if err != nil { errs = multierror.Append(errs, fmt.Errorf("could not submit batch request: %w", err)) continue diff --git a/engine/collection/synchronization/engine_test.go b/engine/collection/synchronization/engine_test.go index cd79ffe1931..a637a9eedec 100644 --- a/engine/collection/synchronization/engine_test.go +++ b/engine/collection/synchronization/engine_test.go @@ -57,9 +57,6 @@ type SyncSuite struct { } func (ss *SyncSuite) SetupTest() { - // seed the RNG - rand.Seed(time.Now().UnixNano()) - // generate own ID ss.participants = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleCollection)) ss.myID = ss.participants[0].NodeID diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index c83830e7b56..a8f04173099 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -1,7 +1,6 @@ package test import ( - "context" "sync" "testing" "time" @@ -17,7 +16,6 @@ import ( "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/util" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/mocknetwork" @@ -101,14 +99,9 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) tc.root, err = inmem.SnapshotFromBootstrapState(root, result, seal, qc) require.NoError(t, err) - cancelCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - ctx := irrecoverable.NewMockSignalerContext(t, cancelCtx) - defer cancel() - // create a mock node for each collector identity for _, collector := range nodeInfos { - node := testutil.CollectionNode(tc.T(), ctx, tc.hub, collector, tc.root) + node := testutil.CollectionNode(tc.T(), tc.hub, collector, tc.root) tc.nodes = append(tc.nodes, node) } @@ -274,8 +267,8 @@ func (tc *ClusterSwitchoverTestCase) ExpectTransaction(epochCounter uint64, clus } // ClusterState opens and returns a read-only cluster state for the given node and cluster ID. -func (tc *ClusterSwitchoverTestCase) ClusterState(node testmock.CollectionNode, clusterID flow.ChainID) cluster.State { - state, err := bcluster.OpenState(node.PublicDB, node.Tracer, node.Headers, node.ClusterPayloads, clusterID) +func (tc *ClusterSwitchoverTestCase) ClusterState(node testmock.CollectionNode, clusterID flow.ChainID, epoch uint64) cluster.State { + state, err := bcluster.OpenState(node.PublicDB, node.Tracer, node.Headers, node.ClusterPayloads, clusterID, epoch) require.NoError(tc.T(), err) return state } @@ -371,7 +364,7 @@ func (tc *ClusterSwitchoverTestCase) CheckClusterState( clusterInfo protocol.Cluster, ) { node := tc.Collector(identity.NodeID) - state := tc.ClusterState(node, clusterInfo.ChainID()) + state := tc.ClusterState(node, clusterInfo.ChainID(), clusterInfo.EpochCounter()) expected := tc.sentTransactions[clusterInfo.EpochCounter()][clusterInfo.Index()] unittest.NewClusterStateChecker(state). ExpectTxCount(len(expected)). diff --git a/engine/common/fifoqueue/fifoqueue.go b/engine/common/fifoqueue/fifoqueue.go index ed4ad58d8b1..cc921251c38 100644 --- a/engine/common/fifoqueue/fifoqueue.go +++ b/engine/common/fifoqueue/fifoqueue.go @@ -58,6 +58,14 @@ func WithLengthObserver(callback QueueLengthObserver) ConstructorOption { } } +// WithLengthMetricObserver attaches a length observer which calls the given observe function. +// It can be used to concisely bind a metrics observer implementing module.MempoolMetrics to the queue. +func WithLengthMetricObserver(resource string, observe func(resource string, length uint)) ConstructorOption { + return WithLengthObserver(func(l int) { + observe(resource, uint(l)) + }) +} + // NewFifoQueue is the Constructor for FifoQueue func NewFifoQueue(maxCapacity int, options ...ConstructorOption) (*FifoQueue, error) { if maxCapacity < 1 { diff --git a/engine/common/follower/cache/cache.go b/engine/common/follower/cache/cache.go index 6be4cf13cd6..9d60ec379f6 100644 --- a/engine/common/follower/cache/cache.go +++ b/engine/common/follower/cache/cache.go @@ -6,9 +6,11 @@ import ( "github.com/rs/zerolog" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/counters" herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" ) @@ -17,8 +19,6 @@ var ( ErrDisconnectedBatch = errors.New("batch must be a sequence of connected blocks") ) -// OnEquivocation is a callback to report observing two different blocks with the same view. -type OnEquivocation func(first *flow.Block, other *flow.Block) type BlocksByID map[flow.Identifier]*flow.Block // batchContext contains contextual data for batch of blocks. Per convention, a batch is @@ -30,6 +30,10 @@ type batchContext struct { // equivocatingBlocks holds the list of equivocations that the batch contained, when comparing to the // cached blocks. An equivocation are two blocks for the same view that have different block IDs. equivocatingBlocks [][2]*flow.Block + + // redundant marks if ALL blocks in batch are already stored in cache, meaning that + // such input is identical to what was previously processed. + redundant bool } // Cache stores pending blocks received from other replicas, caches blocks by blockID, and maintains @@ -45,8 +49,8 @@ type Cache struct { byView map[uint64]BlocksByID // lookup of blocks by their respective view; used to detect equivocation byParent map[flow.Identifier]BlocksByID // lookup of blocks by their parentID, for finding a block's known children - onEquivocation OnEquivocation // when message equivocation has been detected report it using this callback - lowestView counters.StrictMonotonousCounter // lowest view that the cache accepts blocks for + notifier hotstuff.ProposalViolationConsumer // equivocations will be reported using this notifier + lowestView counters.StrictMonotonousCounter // lowest view that the cache accepts blocks for } // Peek performs lookup of cached block by blockID. @@ -62,7 +66,7 @@ func (c *Cache) Peek(blockID flow.Identifier) *flow.Block { } // NewCache creates new instance of Cache -func NewCache(log zerolog.Logger, limit uint32, collector module.HeroCacheMetrics, onEquivocation OnEquivocation) *Cache { +func NewCache(log zerolog.Logger, limit uint32, collector module.HeroCacheMetrics, notifier hotstuff.ProposalViolationConsumer) *Cache { // We consume ejection event from HeroCache to here to drop ejected blocks from our secondary indices. distributor := NewDistributor() cache := &Cache{ @@ -74,9 +78,9 @@ func NewCache(log zerolog.Logger, limit uint32, collector module.HeroCacheMetric collector, herocache.WithTracer(distributor), ), - byView: make(map[uint64]BlocksByID), - byParent: make(map[flow.Identifier]BlocksByID), - onEquivocation: onEquivocation, + byView: make(map[uint64]BlocksByID), + byParent: make(map[flow.Identifier]BlocksByID), + notifier: notifier, } distributor.AddConsumer(cache.handleEjectedEntity) return cache @@ -151,7 +155,12 @@ func (c *Cache) AddBlocks(batch []*flow.Block) (certifiedBatch []*flow.Block, ce // (result stored in `batchContext.batchParent`) // * check whether last block in batch has a child already in the cache // (result stored in `batchContext.batchChild`) + // * check if input is redundant (indicated by `batchContext.redundant`), i.e. ALL blocks + // are already known: then skip further processing bc := c.unsafeAtomicAdd(blockIDs, batch) + if bc.redundant { + return nil, nil, nil + } // If there exists a child of the last block in the batch, then the entire batch is certified. // Otherwise, all blocks in the batch _except_ for the last one are certified @@ -174,7 +183,7 @@ func (c *Cache) AddBlocks(batch []*flow.Block) (certifiedBatch []*flow.Block, ce // report equivocations for _, pair := range bc.equivocatingBlocks { - c.onEquivocation(pair[0], pair[1]) + c.notifier.OnDoubleProposeDetected(model.BlockFromFlow(pair[0].Header), model.BlockFromFlow(pair[1].Header)) } if len(certifiedBatch) < 1 { @@ -241,6 +250,7 @@ func (c *Cache) removeByView(view uint64, blocks BlocksByID) { // - check for equivocating blocks // - check whether first block in batch (index 0) has a parent already in the cache // - check whether last block in batch has a child already in the cache +// - check whether all blocks were previously stored in the cache // // Concurrency SAFE. // @@ -272,12 +282,17 @@ func (c *Cache) unsafeAtomicAdd(blockIDs []flow.Identifier, fullBlocks []*flow.B } // add blocks to underlying cache, check for equivocation and report if detected + storedBlocks := uint64(0) for i, block := range fullBlocks { - equivocation := c.cache(blockIDs[i], block) + equivocation, cached := c.cache(blockIDs[i], block) if equivocation != nil { bc.equivocatingBlocks = append(bc.equivocatingBlocks, [2]*flow.Block{equivocation, block}) } + if cached { + storedBlocks++ + } } + bc.redundant = storedBlocks < 1 return bc } @@ -286,23 +301,26 @@ func (c *Cache) unsafeAtomicAdd(blockIDs []flow.Identifier, fullBlocks []*flow.B // equivocation. The first return value contains the already-cached equivocating block or `nil` otherwise. // Repeated calls with the same block are no-ops. // CAUTION: not concurrency safe: execute within Cache's lock. -func (c *Cache) cache(blockID flow.Identifier, block *flow.Block) (equivocation *flow.Block) { +func (c *Cache) cache(blockID flow.Identifier, block *flow.Block) (equivocation *flow.Block, stored bool) { cachedBlocksAtView, haveCachedBlocksAtView := c.byView[block.Header.View] // Check whether there is a block with the same view already in the cache. - // During happy-path operations `cachedBlocksAtView` contains usually zero blocks or exactly one block - // which is `fullBlock` (duplicate). Larger sets of blocks can only be caused by slashable byzantine actions. + // During happy-path operations `cachedBlocksAtView` contains usually zero blocks or exactly one block, which + // is our input `block` (duplicate). Larger sets of blocks can only be caused by slashable byzantine actions. for otherBlockID, otherBlock := range cachedBlocksAtView { if otherBlockID == blockID { - return nil // already stored + return nil, false // already stored } // have two blocks for the same view but with different IDs => equivocation! equivocation = otherBlock - break // we care whether the + break // we care whether we find an equivocation, but don't need to enumerate all equivocations } + // Note: Even if this node detects an equivocation, we still have to process the block. This is because + // the node might be the only one seeing the equivocation, and other nodes might certify the block, + // in which case also this node needs to process the block to continue following consensus. // block is not a duplicate: store in the underlying HeroCache and add it to secondary indices - added := c.backend.Add(blockID, block) - if !added { // future proofing code: we allow an overflowing HeroCache to potentially eject the newly added element. + stored = c.backend.Add(blockID, block) + if !stored { // future proofing code: we allow an overflowing HeroCache to potentially eject the newly added element. return } diff --git a/engine/common/follower/cache/cache_test.go b/engine/common/follower/cache/cache_test.go index c8f9af688ad..d5c42f5ea80 100644 --- a/engine/common/follower/cache/cache_test.go +++ b/engine/common/follower/cache/cache_test.go @@ -11,7 +11,8 @@ import ( "go.uber.org/atomic" "golang.org/x/exp/slices" - "github.com/onflow/flow-go/engine/common/follower/cache/mock" + "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/utils/unittest" @@ -27,14 +28,14 @@ const defaultHeroCacheLimit = 1000 type CacheSuite struct { suite.Suite - onEquivocation *mock.OnEquivocation - cache *Cache + consumer *mocks.ProposalViolationConsumer + cache *Cache } func (s *CacheSuite) SetupTest() { collector := metrics.NewNoopCollector() - s.onEquivocation = mock.NewOnEquivocation(s.T()) - s.cache = NewCache(unittest.Logger(), defaultHeroCacheLimit, collector, s.onEquivocation.Execute) + s.consumer = mocks.NewProposalViolationConsumer(s.T()) + s.cache = NewCache(unittest.Logger(), defaultHeroCacheLimit, collector, s.consumer) } // TestPeek tests if previously added blocks can be queried by block ID. @@ -67,7 +68,8 @@ func (s *CacheSuite) TestBlocksEquivocation() { block.Header.View = blocks[i].Header.View // update parentID so blocks are still connected block.Header.ParentID = equivocatedBlocks[i-1].ID() - s.onEquivocation.On("Execute", blocks[i], block).Once() + s.consumer.On("OnDoubleProposeDetected", + model.BlockFromFlow(blocks[i].Header), model.BlockFromFlow(block.Header)).Return().Once() } _, _, err = s.cache.AddBlocks(equivocatedBlocks) require.NoError(s.T(), err) @@ -166,6 +168,30 @@ func (s *CacheSuite) TestAddBatch() { require.Equal(s.T(), blocks[len(blocks)-1].Header.QuorumCertificate(), certifyingQC) } +// TestDuplicatedBatch checks that processing redundant inputs rejects batches where all blocks +// already reside in the cache. Batches that have at least one new block should be accepted. +func (s *CacheSuite) TestDuplicatedBatch() { + blocks := unittest.ChainFixtureFrom(10, unittest.BlockHeaderFixture()) + + certifiedBatch, certifyingQC, err := s.cache.AddBlocks(blocks[1:]) + require.NoError(s.T(), err) + require.Equal(s.T(), blocks[1:len(blocks)-1], certifiedBatch) + require.Equal(s.T(), blocks[len(blocks)-1].Header.QuorumCertificate(), certifyingQC) + + // add same batch again, this has to be rejected as redundant input + certifiedBatch, certifyingQC, err = s.cache.AddBlocks(blocks[1:]) + require.NoError(s.T(), err) + require.Empty(s.T(), certifiedBatch) + require.Nil(s.T(), certifyingQC) + + // add batch with one extra leading block, this has to accepted even though 9 out of 10 blocks + // were already processed + certifiedBatch, certifyingQC, err = s.cache.AddBlocks(blocks) + require.NoError(s.T(), err) + require.Equal(s.T(), blocks[:len(blocks)-1], certifiedBatch) + require.Equal(s.T(), blocks[len(blocks)-1].Header.QuorumCertificate(), certifyingQC) +} + // TestPruneUpToView tests that blocks lower than pruned height will be properly filtered out from incoming batch. func (s *CacheSuite) TestPruneUpToView() { blocks := unittest.ChainFixtureFrom(3, unittest.BlockHeaderFixture()) @@ -291,7 +317,7 @@ func (s *CacheSuite) TestAddOverCacheLimit() { // create blocks more than limit workers := 10 blocksPerWorker := 10 - s.cache = NewCache(unittest.Logger(), uint32(blocksPerWorker), metrics.NewNoopCollector(), s.onEquivocation.Execute) + s.cache = NewCache(unittest.Logger(), uint32(blocksPerWorker), metrics.NewNoopCollector(), s.consumer) blocks := unittest.ChainFixtureFrom(blocksPerWorker*workers, unittest.BlockHeaderFixture()) diff --git a/engine/common/follower/cache/mock/on_equivocation.go b/engine/common/follower/cache/mock/on_equivocation.go deleted file mode 100644 index 7f0119be8f5..00000000000 --- a/engine/common/follower/cache/mock/on_equivocation.go +++ /dev/null @@ -1,33 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mock - -import ( - flow "github.com/onflow/flow-go/model/flow" - mock "github.com/stretchr/testify/mock" -) - -// OnEquivocation is an autogenerated mock type for the OnEquivocation type -type OnEquivocation struct { - mock.Mock -} - -// Execute provides a mock function with given fields: first, other -func (_m *OnEquivocation) Execute(first *flow.Block, other *flow.Block) { - _m.Called(first, other) -} - -type mockConstructorTestingTNewOnEquivocation interface { - mock.TestingT - Cleanup(func()) -} - -// NewOnEquivocation creates a new instance of OnEquivocation. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewOnEquivocation(t mockConstructorTestingTNewOnEquivocation) *OnEquivocation { - mock := &OnEquivocation{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/engine/common/follower/compliance_core.go b/engine/common/follower/compliance_core.go index b80b9da8334..c8ebf1b7a82 100644 --- a/engine/common/follower/compliance_core.go +++ b/engine/common/follower/compliance_core.go @@ -38,17 +38,18 @@ const defaultPendingBlocksCacheCapacity = 1000 // Generally is NOT concurrency safe but some functions can be used in concurrent setup. type ComplianceCore struct { *component.ComponentManager - log zerolog.Logger - mempoolMetrics module.MempoolMetrics - tracer module.Tracer - pendingCache *cache.Cache - pendingTree *pending_tree.PendingTree - state protocol.FollowerState - follower module.HotStuffFollower - validator hotstuff.Validator - sync module.BlockRequester - certifiedRangesChan chan CertifiedBlocks // delivers ranges of certified blocks to main core worker - finalizedBlocksChan chan *flow.Header // delivers finalized blocks to main core worker. + log zerolog.Logger + mempoolMetrics module.MempoolMetrics + tracer module.Tracer + proposalViolationNotifier hotstuff.ProposalViolationConsumer + pendingCache *cache.Cache + pendingTree *pending_tree.PendingTree + state protocol.FollowerState + follower module.HotStuffFollower + validator hotstuff.Validator + sync module.BlockRequester + certifiedRangesChan chan CertifiedBlocks // delivers ranges of certified blocks to main core worker + finalizedBlocksChan chan *flow.Header // delivers finalized blocks to main core worker. } var _ complianceCore = (*ComplianceCore)(nil) @@ -58,34 +59,31 @@ var _ complianceCore = (*ComplianceCore)(nil) func NewComplianceCore(log zerolog.Logger, mempoolMetrics module.MempoolMetrics, heroCacheCollector module.HeroCacheMetrics, - finalizationConsumer hotstuff.FinalizationConsumer, + followerConsumer hotstuff.FollowerConsumer, state protocol.FollowerState, follower module.HotStuffFollower, validator hotstuff.Validator, sync module.BlockRequester, tracer module.Tracer, ) (*ComplianceCore, error) { - onEquivocation := func(block, otherBlock *flow.Block) { - finalizationConsumer.OnDoubleProposeDetected(model.BlockFromFlow(block.Header), model.BlockFromFlow(otherBlock.Header)) - } - finalizedBlock, err := state.Final().Head() if err != nil { return nil, fmt.Errorf("could not query finalized block: %w", err) } c := &ComplianceCore{ - log: log.With().Str("engine", "follower_core").Logger(), - mempoolMetrics: mempoolMetrics, - state: state, - pendingCache: cache.NewCache(log, defaultPendingBlocksCacheCapacity, heroCacheCollector, onEquivocation), - pendingTree: pending_tree.NewPendingTree(finalizedBlock), - follower: follower, - validator: validator, - sync: sync, - tracer: tracer, - certifiedRangesChan: make(chan CertifiedBlocks, defaultCertifiedRangeChannelCapacity), - finalizedBlocksChan: make(chan *flow.Header, defaultFinalizedBlocksChannelCapacity), + log: log.With().Str("engine", "follower_core").Logger(), + mempoolMetrics: mempoolMetrics, + state: state, + proposalViolationNotifier: followerConsumer, + pendingCache: cache.NewCache(log, defaultPendingBlocksCacheCapacity, heroCacheCollector, followerConsumer), + pendingTree: pending_tree.NewPendingTree(finalizedBlock), + follower: follower, + validator: validator, + sync: sync, + tracer: tracer, + certifiedRangesChan: make(chan CertifiedBlocks, defaultCertifiedRangeChannelCapacity), + finalizedBlocksChan: make(chan *flow.Header, defaultFinalizedBlocksChannelCapacity), } // prune cache to latest finalized view @@ -134,16 +132,18 @@ func (c *ComplianceCore) OnBlockRange(originID flow.Identifier, batch []*flow.Bl // 1. The block has been signed by the legitimate primary for the view. This is important in case // there are multiple blocks for the view. We need to differentiate the following byzantine cases: // (i) Some other consensus node that is _not_ primary is trying to publish a block. - // This would result in the validation below failing with an `InvalidBlockError`. + // This would result in the validation below failing with an `InvalidProposalError`. // (ii) The legitimate primary for the view is equivocating. In this case, the validity check // below would pass. Though, the `PendingTree` would eventually notice this, when we connect // the equivocating blocks to the latest finalized block. // 2. The QC within the block is valid. A valid QC proves validity of all ancestors. err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { - if model.IsInvalidBlockError(err) { - // TODO potential slashing - log.Err(err).Msgf("received invalid block proposal (potential slashing evidence)") + if invalidBlockError, ok := model.AsInvalidProposalError(err); ok { + c.proposalViolationNotifier.OnInvalidBlockDetected(flow.Slashable[model.InvalidProposalError]{ + OriginID: originID, + Message: *invalidBlockError, + }) return nil } if errors.Is(err, model.ErrViewForUnknownEpoch) { @@ -266,8 +266,11 @@ func (c *ComplianceCore) processCertifiedBlocks(ctx context.Context, blocks Cert return fmt.Errorf("could not extend protocol state with certified block: %w", err) } - hotstuffProposal := model.ProposalFromFlow(certifiedBlock.Block.Header) - c.follower.SubmitProposal(hotstuffProposal) // submit the model to follower for async processing + b, err := model.NewCertifiedBlock(model.BlockFromFlow(certifiedBlock.Block.Header), certifiedBlock.CertifyingQC) + if err != nil { + return fmt.Errorf("failed to convert certified block %v to HotStuff type: %w", certifiedBlock.Block.ID(), err) + } + c.follower.AddCertifiedBlock(&b) // submit the model to follower for async processing } return nil } diff --git a/engine/common/follower/compliance_core_test.go b/engine/common/follower/compliance_core_test.go index 38c857d8974..522fc26160e 100644 --- a/engine/common/follower/compliance_core_test.go +++ b/engine/common/follower/compliance_core_test.go @@ -3,7 +3,6 @@ package follower import ( "context" "errors" - "fmt" "sync" "testing" "time" @@ -34,13 +33,13 @@ func TestFollowerCore(t *testing.T) { type CoreSuite struct { suite.Suite - originID flow.Identifier - finalizedBlock *flow.Header - state *protocol.FollowerState - follower *module.HotStuffFollower - sync *module.BlockRequester - validator *hotstuff.Validator - finalizationConsumer *hotstuff.FinalizationConsumer + originID flow.Identifier + finalizedBlock *flow.Header + state *protocol.FollowerState + follower *module.HotStuffFollower + sync *module.BlockRequester + validator *hotstuff.Validator + followerConsumer *hotstuff.FollowerConsumer ctx irrecoverable.SignalerContext cancel context.CancelFunc @@ -53,7 +52,7 @@ func (s *CoreSuite) SetupTest() { s.follower = module.NewHotStuffFollower(s.T()) s.validator = hotstuff.NewValidator(s.T()) s.sync = module.NewBlockRequester(s.T()) - s.finalizationConsumer = hotstuff.NewFinalizationConsumer(s.T()) + s.followerConsumer = hotstuff.NewFollowerConsumer(s.T()) s.originID = unittest.IdentifierFixture() s.finalizedBlock = unittest.BlockHeaderFixture() @@ -67,7 +66,7 @@ func (s *CoreSuite) SetupTest() { unittest.Logger(), metrics, metrics, - s.finalizationConsumer, + s.followerConsumer, s.state, s.follower, s.validator, @@ -137,7 +136,7 @@ func (s *CoreSuite) TestProcessingRangeHappyPath() { wg.Add(len(blocks) - 1) for i := 1; i < len(blocks); i++ { s.state.On("ExtendCertified", mock.Anything, blocks[i-1], blocks[i].Header.QuorumCertificate()).Return(nil).Once() - s.follower.On("SubmitProposal", model.ProposalFromFlow(blocks[i-1].Header)).Run(func(args mock.Arguments) { + s.follower.On("AddCertifiedBlock", blockWithID(blocks[i-1].ID())).Run(func(args mock.Arguments) { wg.Done() }).Return().Once() } @@ -165,12 +164,18 @@ func (s *CoreSuite) TestProcessingNotOrderedBatch() { func (s *CoreSuite) TestProcessingInvalidBlock() { blocks := unittest.ChainFixtureFrom(10, s.finalizedBlock) - s.validator.On("ValidateProposal", model.ProposalFromFlow(blocks[len(blocks)-1].Header)).Return(model.InvalidBlockError{Err: fmt.Errorf("")}).Once() + invalidProposal := model.ProposalFromFlow(blocks[len(blocks)-1].Header) + sentinelError := model.NewInvalidProposalErrorf(invalidProposal, "") + s.validator.On("ValidateProposal", invalidProposal).Return(sentinelError).Once() + s.followerConsumer.On("OnInvalidBlockDetected", flow.Slashable[model.InvalidProposalError]{ + OriginID: s.originID, + Message: sentinelError.(model.InvalidProposalError), + }).Return().Once() err := s.core.OnBlockRange(s.originID, blocks) require.NoError(s.T(), err, "sentinel error has to be handled internally") exception := errors.New("validate-proposal-exception") - s.validator.On("ValidateProposal", model.ProposalFromFlow(blocks[len(blocks)-1].Header)).Return(exception).Once() + s.validator.On("ValidateProposal", invalidProposal).Return(exception).Once() err = s.core.OnBlockRange(s.originID, blocks) require.ErrorIs(s.T(), err, exception, "exception has to be propagated") } @@ -204,7 +209,7 @@ func (s *CoreSuite) TestProcessingConnectedRangesOutOfOrder() { var wg sync.WaitGroup wg.Add(len(blocks) - 1) for _, block := range blocks[:len(blocks)-1] { - s.follower.On("SubmitProposal", model.ProposalFromFlow(block.Header)).Return().Run(func(args mock.Arguments) { + s.follower.On("AddCertifiedBlock", blockWithID(block.ID())).Return().Run(func(args mock.Arguments) { wg.Done() }).Once() } @@ -234,7 +239,7 @@ func (s *CoreSuite) TestDetectingProposalEquivocation() { otherBlock.Header.View = block.Header.View s.validator.On("ValidateProposal", mock.Anything).Return(nil).Times(2) - s.finalizationConsumer.On("OnDoubleProposeDetected", mock.Anything, mock.Anything).Return().Once() + s.followerConsumer.On("OnDoubleProposeDetected", mock.Anything, mock.Anything).Return().Once() err := s.core.OnBlockRange(s.originID, []*flow.Block{block}) require.NoError(s.T(), err) @@ -266,10 +271,10 @@ func (s *CoreSuite) TestConcurrentAdd() { s.validator.On("ValidateProposal", mock.Anything).Return(nil) // any proposal is valid done := make(chan struct{}) - s.follower.On("SubmitProposal", mock.Anything).Return(nil).Run(func(args mock.Arguments) { + s.follower.On("AddCertifiedBlock", mock.Anything).Return(nil).Run(func(args mock.Arguments) { // ensure that proposals are submitted in-order - proposal := args.Get(0).(*model.Proposal) - if proposal.Block.BlockID == targetSubmittedBlockID { + block := args.Get(0).(*model.CertifiedBlock) + if block.ID() == targetSubmittedBlockID { close(done) } }).Return().Times(len(blocks) - 1) // all proposals have to be submitted @@ -301,3 +306,8 @@ func (s *CoreSuite) TestConcurrentAdd() { unittest.RequireReturnsBefore(s.T(), wg.Wait, time.Millisecond*500, "should submit blocks before timeout") unittest.AssertClosesBefore(s.T(), done, time.Millisecond*500, "should process all blocks before timeout") } + +// blockWithID returns a testify `argumentMatcher` that only accepts blocks with the given ID +func blockWithID(expectedBlockID flow.Identifier) interface{} { + return mock.MatchedBy(func(block *model.CertifiedBlock) bool { return expectedBlockID == block.ID() }) +} diff --git a/engine/common/follower/compliance_engine.go b/engine/common/follower/compliance_engine.go index a0b28e34d17..d7d8c2cb95c 100644 --- a/engine/common/follower/compliance_engine.go +++ b/engine/common/follower/compliance_engine.go @@ -98,6 +98,7 @@ func NewComplianceLayer( headers storage.Headers, finalized *flow.Header, core complianceCore, + config compliance.Config, opts ...EngineOption, ) (*ComplianceEngine, error) { // FIFO queue for inbound block proposals @@ -115,7 +116,7 @@ func NewComplianceLayer( log: log.With().Str("engine", "follower").Logger(), me: me, engMetrics: engMetrics, - config: compliance.DefaultConfig(), + config: config, channel: channels.ReceiveBlocks, pendingProposals: pendingBlocks, syncedBlocks: syncedBlocks, @@ -337,9 +338,10 @@ func (e *ComplianceEngine) submitConnectedBatch(log zerolog.Logger, latestFinali log.Debug().Msgf("dropping range [%d, %d] below finalized view %d", blocks[0].Header.View, lastBlock.View, latestFinalizedView) return } - if lastBlock.View > latestFinalizedView+e.config.SkipNewProposalsThreshold { + skipNewProposalsThreshold := e.config.GetSkipNewProposalsThreshold() + if lastBlock.View > latestFinalizedView+skipNewProposalsThreshold { log.Debug(). - Uint64("skip_new_proposals_threshold", e.config.SkipNewProposalsThreshold). + Uint64("skip_new_proposals_threshold", skipNewProposalsThreshold). Msgf("dropping range [%d, %d] too far ahead of locally finalized view %d", blocks[0].Header.View, lastBlock.View, latestFinalizedView) return diff --git a/engine/common/follower/compliance_engine_test.go b/engine/common/follower/compliance_engine_test.go index 4abceba662a..b1ab1a3ba0e 100644 --- a/engine/common/follower/compliance_engine_test.go +++ b/engine/common/follower/compliance_engine_test.go @@ -15,6 +15,7 @@ import ( followermock "github.com/onflow/flow-go/engine/common/follower/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module/compliance" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" @@ -70,7 +71,8 @@ func (s *EngineSuite) SetupTest() { metrics, s.headers, s.finalized, - s.core) + s.core, + compliance.DefaultConfig()) require.Nil(s.T(), err) s.engine = eng diff --git a/engine/common/follower/integration_test.go b/engine/common/follower/integration_test.go index 17b7171f4e7..663e195462e 100644 --- a/engine/common/follower/integration_test.go +++ b/engine/common/follower/integration_test.go @@ -13,11 +13,11 @@ import ( "github.com/onflow/flow-go/consensus" "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/follower" "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module/compliance" moduleconsensus "github.com/onflow/flow-go/module/finalizer/consensus" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" @@ -53,7 +53,20 @@ func TestFollowerHappyPath(t *testing.T) { all := storageutil.StorageLayer(t, db) // bootstrap root snapshot - state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := pbadger.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) mockTimer := util.MockBlockTimer() @@ -83,7 +96,7 @@ func TestFollowerHappyPath(t *testing.T) { }) require.NoError(t, err) - consensusConsumer := pubsub.NewFinalizationDistributor() + consensusConsumer := pubsub.NewFollowerDistributor() // use real consensus modules forks, err := consensus.NewForks(rootHeader, all.Headers, finalizer, consensusConsumer, rootHeader, rootQC) require.NoError(t, err) @@ -92,12 +105,8 @@ func TestFollowerHappyPath(t *testing.T) { validator := mocks.NewValidator(t) validator.On("ValidateProposal", mock.Anything).Return(nil) - // initialize the follower followerHotstuffLogic - followerHotstuffLogic, err := follower.New(unittest.Logger(), validator, forks) - require.NoError(t, err) - // initialize the follower loop - followerLoop, err := hotstuff.NewFollowerLoop(unittest.Logger(), followerHotstuffLogic) + followerLoop, err := hotstuff.NewFollowerLoop(unittest.Logger(), metrics, forks) require.NoError(t, err) syncCore := module.NewBlockRequester(t) @@ -123,7 +132,16 @@ func TestFollowerHappyPath(t *testing.T) { net.On("Register", mock.Anything, mock.Anything).Return(con, nil) // use real engine - engine, err := NewComplianceLayer(unittest.Logger(), net, me, metrics, all.Headers, rootHeader, followerCore) + engine, err := NewComplianceLayer( + unittest.Logger(), + net, + me, + metrics, + all.Headers, + rootHeader, + followerCore, + compliance.DefaultConfig(), + ) require.NoError(t, err) // don't forget to subscribe for finalization notifications consensusConsumer.AddOnBlockFinalizedConsumer(engine.OnFinalizedBlock) @@ -152,8 +170,15 @@ func TestFollowerHappyPath(t *testing.T) { } pendingBlocks := flowBlocksToBlockProposals(flowBlocks...) - // this block should be finalized based on 2-chain finalization rule - targetBlockHeight := pendingBlocks[len(pendingBlocks)-4].Block.Header.Height + // Regarding the block that we expect to be finalized based on 2-chain finalization rule, we consider the last few blocks in `pendingBlocks` + // ... <-- X <-- Y <-- Z + // ╰─────────╯ + // 2-chain on top of X + // Hence, we expect X to be finalized, which has the index `len(pendingBlocks)-3` + // Note: the HotStuff Follower does not see block Z (as there is no QC for X proving its validity). Instead, it sees the certified block + // [◄(X) Y] ◄(Y) + // where ◄(B) denotes a QC for block B + targetBlockHeight := pendingBlocks[len(pendingBlocks)-3].Block.Header.Height // emulate syncing logic, where we push same blocks over and over. originID := unittest.IdentifierFixture() diff --git a/engine/common/follower/pending_tree/pending_tree_test.go b/engine/common/follower/pending_tree/pending_tree_test.go index 14f45d23ca5..ac482871aa4 100644 --- a/engine/common/follower/pending_tree/pending_tree_test.go +++ b/engine/common/follower/pending_tree/pending_tree_test.go @@ -4,7 +4,6 @@ import ( "fmt" "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -28,7 +27,6 @@ type PendingTreeSuite struct { } func (s *PendingTreeSuite) SetupTest() { - rand.Seed(time.Now().UnixNano()) s.finalized = unittest.BlockHeaderFixture() s.pendingTree = NewPendingTree(s.finalized) } diff --git a/engine/common/grpc/forwarder/forwarder.go b/engine/common/grpc/forwarder/forwarder.go new file mode 100644 index 00000000000..e87fd4749d4 --- /dev/null +++ b/engine/common/grpc/forwarder/forwarder.go @@ -0,0 +1,145 @@ +package forwarder + +import ( + "fmt" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + rpcConnection "github.com/onflow/flow-go/engine/access/rpc/connection" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/grpcutils" + + "github.com/onflow/flow/protobuf/go/flow/access" +) + +// Forwarder forwards all requests to a set of upstream access nodes or observers +type Forwarder struct { + lock sync.Mutex + roundRobin int + ids flow.IdentityList + upstream []access.AccessAPIClient + connections []*grpc.ClientConn + timeout time.Duration + maxMsgSize uint +} + +func NewForwarder(identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*Forwarder, error) { + forwarder := &Forwarder{maxMsgSize: maxMsgSize} + err := forwarder.setFlowAccessAPI(identities, timeout) + return forwarder, err +} + +// setFlowAccessAPI sets a backend access API that forwards some requests to an upstream node. +// It is used by Observer services, Blockchain Data Service, etc. +// Make sure that this is just for observation and not a staked participant in the flow network. +// This means that observers see a copy of the data but there is no interaction to ensure integrity from the root block. +func (f *Forwarder) setFlowAccessAPI(accessNodeAddressAndPort flow.IdentityList, timeout time.Duration) error { + f.timeout = timeout + f.ids = accessNodeAddressAndPort + f.upstream = make([]access.AccessAPIClient, accessNodeAddressAndPort.Count()) + f.connections = make([]*grpc.ClientConn, accessNodeAddressAndPort.Count()) + for i, identity := range accessNodeAddressAndPort { + // Store the faultTolerantClient setup parameters such as address, public, key and timeout, so that + // we can refresh the API on connection loss + f.ids[i] = identity + + // We fail on any single error on startup, so that + // we identify bootstrapping errors early + err := f.reconnectingClient(i) + if err != nil { + return err + } + } + + f.roundRobin = 0 + return nil +} + +// reconnectingClient returns an active client, or +// creates one, if the last one is not ready anymore. +func (f *Forwarder) reconnectingClient(i int) error { + timeout := f.timeout + + if f.connections[i] == nil || f.connections[i].GetState() != connectivity.Ready { + identity := f.ids[i] + var connection *grpc.ClientConn + var err error + if identity.NetworkPubKey == nil { + connection, err = grpc.Dial( + identity.Address, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(f.maxMsgSize))), + grpc.WithTransportCredentials(insecure.NewCredentials()), + rpcConnection.WithClientTimeoutOption(timeout)) + if err != nil { + return err + } + } else { + tlsConfig, err := grpcutils.DefaultClientTLSConfig(identity.NetworkPubKey) + if err != nil { + return fmt.Errorf("failed to get default TLS client config using public flow networking key %s %w", identity.NetworkPubKey.String(), err) + } + + connection, err = grpc.Dial( + identity.Address, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(f.maxMsgSize))), + grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), + rpcConnection.WithClientTimeoutOption(timeout)) + if err != nil { + return fmt.Errorf("cannot connect to %s %w", identity.Address, err) + } + } + connection.Connect() + time.Sleep(1 * time.Second) + state := connection.GetState() + if state != connectivity.Ready && state != connectivity.Connecting { + return fmt.Errorf("%v", state) + } + f.connections[i] = connection + f.upstream[i] = access.NewAccessAPIClient(connection) + } + + return nil +} + +// FaultTolerantClient implements an upstream connection that reconnects on errors +// a reasonable amount of time. +func (f *Forwarder) FaultTolerantClient() (access.AccessAPIClient, error) { + if f.upstream == nil || len(f.upstream) == 0 { + return nil, status.Errorf(codes.Unimplemented, "method not implemented") + } + + // Reasoning: A retry count of three gives an acceptable 5% failure ratio from a 37% failure ratio. + // A bigger number is problematic due to the DNS resolve and connection times, + // plus the need to log and debug each individual connection failure. + // + // This reasoning eliminates the need of making this parameter configurable. + // The logic works rolling over a single connection as well making clean code. + const retryMax = 3 + + f.lock.Lock() + defer f.lock.Unlock() + + var err error + for i := 0; i < retryMax; i++ { + f.roundRobin++ + f.roundRobin = f.roundRobin % len(f.upstream) + err = f.reconnectingClient(f.roundRobin) + if err != nil { + continue + } + state := f.connections[f.roundRobin].GetState() + if state != connectivity.Ready && state != connectivity.Connecting { + continue + } + return f.upstream[f.roundRobin], nil + } + + return nil, status.Errorf(codes.Unavailable, err.Error()) +} diff --git a/engine/common/requester/engine.go b/engine/common/requester/engine.go index f83a2d03780..bc54d3dd45b 100644 --- a/engine/common/requester/engine.go +++ b/engine/common/requester/engine.go @@ -3,7 +3,6 @@ package requester import ( "fmt" "math" - "math/rand" "time" "github.com/rs/zerolog" @@ -20,6 +19,7 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/utils/logging" + "github.com/onflow/flow-go/utils/rand" ) // HandleFunc is a function provided to the requester engine to handle an entity @@ -51,7 +51,6 @@ type Engine struct { items map[flow.Identifier]*Item requests map[uint64]*messages.EntityRequest forcedDispatchOngoing *atomic.Bool // to ensure only trigger dispatching logic once at any time - rng *rand.Rand } // New creates a new requester engine, operating on the provided network channel, and requesting entities from a node @@ -117,7 +116,6 @@ func New(log zerolog.Logger, metrics module.EngineMetrics, net network.Network, items: make(map[flow.Identifier]*Item), // holds all pending items requests: make(map[uint64]*messages.EntityRequest), // holds all sent requests forcedDispatchOngoing: atomic.NewBool(false), - rng: rand.New(rand.NewSource(time.Now().UnixNano())), } // register the engine with the network layer and store the conduit @@ -319,7 +317,12 @@ func (e *Engine) dispatchRequest() (bool, error) { for k := range e.items { rndItems = append(rndItems, e.items[k].EntityID) } - e.rng.Shuffle(len(rndItems), func(i, j int) { rndItems[i], rndItems[j] = rndItems[j], rndItems[i] }) + err = rand.Shuffle(uint(len(rndItems)), func(i, j uint) { + rndItems[i], rndItems[j] = rndItems[j], rndItems[i] + }) + if err != nil { + return false, fmt.Errorf("shuffle failed: %w", err) + } // go through each item and decide if it should be requested again now := time.Now().UTC() @@ -364,7 +367,11 @@ func (e *Engine) dispatchRequest() (bool, error) { if len(providers) == 0 { return false, fmt.Errorf("no valid providers available") } - providerID = providers.Sample(1)[0].NodeID + id, err := providers.Sample(1) + if err != nil { + return false, fmt.Errorf("sampling failed: %w", err) + } + providerID = id[0].NodeID } // add item to list and set retry parameters @@ -396,9 +403,14 @@ func (e *Engine) dispatchRequest() (bool, error) { return false, nil } + nonce, err := rand.Uint64() + if err != nil { + return false, fmt.Errorf("nonce generation failed: %w", err) + } + // create a batch request, send it and store it for reference req := &messages.EntityRequest{ - Nonce: e.rng.Uint64(), + Nonce: nonce, EntityIDs: entityIDs, } diff --git a/engine/common/requester/engine_test.go b/engine/common/requester/engine_test.go index a2a259d44dc..553386c85d6 100644 --- a/engine/common/requester/engine_test.go +++ b/engine/common/requester/engine_test.go @@ -29,7 +29,6 @@ func TestEntityByID(t *testing.T) { request := Engine{ unit: engine.NewUnit(), items: make(map[flow.Identifier]*Item), - rng: rand.New(rand.NewSource(0)), } now := time.Now().UTC() @@ -136,7 +135,6 @@ func TestDispatchRequestVarious(t *testing.T) { items: items, requests: make(map[uint64]*messages.EntityRequest), selector: filter.HasNodeID(targetID), - rng: rand.New(rand.NewSource(0)), } dispatched, err := request.dispatchRequest() require.NoError(t, err) @@ -213,7 +211,6 @@ func TestDispatchRequestBatchSize(t *testing.T) { items: items, requests: make(map[uint64]*messages.EntityRequest), selector: filter.Any, - rng: rand.New(rand.NewSource(0)), } dispatched, err := request.dispatchRequest() require.NoError(t, err) @@ -293,7 +290,6 @@ func TestOnEntityResponseValid(t *testing.T) { close(done) } }, - rng: rand.New(rand.NewSource(0)), } request.items[iwanted1.EntityID] = iwanted1 @@ -377,7 +373,6 @@ func TestOnEntityIntegrityCheck(t *testing.T) { selector: filter.HasNodeID(targetID), create: func() flow.Entity { return &flow.Collection{} }, handle: func(flow.Identifier, flow.Entity) { close(called) }, - rng: rand.New(rand.NewSource(0)), } request.items[iwanted.EntityID] = iwanted diff --git a/engine/common/rpc/convert/accounts.go b/engine/common/rpc/convert/accounts.go new file mode 100644 index 00000000000..0440d3c0685 --- /dev/null +++ b/engine/common/rpc/convert/accounts.go @@ -0,0 +1,92 @@ +package convert + +import ( + "github.com/onflow/flow/protobuf/go/flow/entities" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/crypto/hash" + "github.com/onflow/flow-go/model/flow" +) + +// AccountToMessage converts a flow.Account to a protobuf message +func AccountToMessage(a *flow.Account) (*entities.Account, error) { + keys := make([]*entities.AccountKey, len(a.Keys)) + for i, k := range a.Keys { + messageKey, err := AccountKeyToMessage(k) + if err != nil { + return nil, err + } + keys[i] = messageKey + } + + return &entities.Account{ + Address: a.Address.Bytes(), + Balance: a.Balance, + Code: nil, + Keys: keys, + Contracts: a.Contracts, + }, nil +} + +// MessageToAccount converts a protobuf message to a flow.Account +func MessageToAccount(m *entities.Account) (*flow.Account, error) { + if m == nil { + return nil, ErrEmptyMessage + } + + accountKeys := make([]flow.AccountPublicKey, len(m.GetKeys())) + for i, key := range m.GetKeys() { + accountKey, err := MessageToAccountKey(key) + if err != nil { + return nil, err + } + + accountKeys[i] = *accountKey + } + + return &flow.Account{ + Address: flow.BytesToAddress(m.GetAddress()), + Balance: m.GetBalance(), + Keys: accountKeys, + Contracts: m.Contracts, + }, nil +} + +// AccountKeyToMessage converts a flow.AccountPublicKey to a protobuf message +func AccountKeyToMessage(a flow.AccountPublicKey) (*entities.AccountKey, error) { + publicKey := a.PublicKey.Encode() + return &entities.AccountKey{ + Index: uint32(a.Index), + PublicKey: publicKey, + SignAlgo: uint32(a.SignAlgo), + HashAlgo: uint32(a.HashAlgo), + Weight: uint32(a.Weight), + SequenceNumber: uint32(a.SeqNumber), + Revoked: a.Revoked, + }, nil +} + +// MessageToAccountKey converts a protobuf message to a flow.AccountPublicKey +func MessageToAccountKey(m *entities.AccountKey) (*flow.AccountPublicKey, error) { + if m == nil { + return nil, ErrEmptyMessage + } + + sigAlgo := crypto.SigningAlgorithm(m.GetSignAlgo()) + hashAlgo := hash.HashingAlgorithm(m.GetHashAlgo()) + + publicKey, err := crypto.DecodePublicKey(sigAlgo, m.GetPublicKey()) + if err != nil { + return nil, err + } + + return &flow.AccountPublicKey{ + Index: int(m.GetIndex()), + PublicKey: publicKey, + SignAlgo: sigAlgo, + HashAlgo: hashAlgo, + Weight: int(m.GetWeight()), + SeqNumber: uint64(m.GetSequenceNumber()), + Revoked: m.GetRevoked(), + }, nil +} diff --git a/engine/common/rpc/convert/accounts_test.go b/engine/common/rpc/convert/accounts_test.go new file mode 100644 index 00000000000..a1b3d80e5c7 --- /dev/null +++ b/engine/common/rpc/convert/accounts_test.go @@ -0,0 +1,58 @@ +package convert_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/crypto/hash" + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/fvm" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestConvertAccount tests that converting an account to and from a protobuf message results in +// the same account +func TestConvertAccount(t *testing.T) { + t.Parallel() + + a, err := unittest.AccountFixture() + require.NoError(t, err) + + key2, err := unittest.AccountKeyFixture(128, crypto.ECDSAP256, hash.SHA3_256) + require.NoError(t, err) + + a.Keys = append(a.Keys, key2.PublicKey(500)) + + msg, err := convert.AccountToMessage(a) + require.NoError(t, err) + + converted, err := convert.MessageToAccount(msg) + require.NoError(t, err) + + assert.Equal(t, a, converted) +} + +// TestConvertAccountKey tests that converting an account key to and from a protobuf message results +// in the same account key +func TestConvertAccountKey(t *testing.T) { + t.Parallel() + + privateKey, _ := unittest.AccountKeyDefaultFixture() + accountKey := privateKey.PublicKey(fvm.AccountKeyWeightThreshold) + + // Explicitly test if Revoked is properly converted + accountKey.Revoked = true + + msg, err := convert.AccountKeyToMessage(accountKey) + assert.Nil(t, err) + + converted, err := convert.MessageToAccountKey(msg) + assert.Nil(t, err) + + assert.Equal(t, accountKey, *converted) + assert.Equal(t, accountKey.PublicKey, converted.PublicKey) + assert.Equal(t, accountKey.Revoked, converted.Revoked) +} diff --git a/engine/common/rpc/convert/blocks.go b/engine/common/rpc/convert/blocks.go new file mode 100644 index 00000000000..2e7f5689515 --- /dev/null +++ b/engine/common/rpc/convert/blocks.go @@ -0,0 +1,155 @@ +package convert + +import ( + "fmt" + + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/onflow/flow-go/model/flow" + + "github.com/onflow/flow/protobuf/go/flow/entities" +) + +// BlockToMessage converts a flow.Block to a protobuf Block message. +// signerIDs is a precomputed list of signer IDs for the block based on the block's signer indicies. +func BlockToMessage(h *flow.Block, signerIDs flow.IdentifierList) ( + *entities.Block, + error, +) { + id := h.ID() + + parentID := h.Header.ParentID + t := timestamppb.New(h.Header.Timestamp) + cg := CollectionGuaranteesToMessages(h.Payload.Guarantees) + + seals := BlockSealsToMessages(h.Payload.Seals) + + execResults, err := ExecutionResultsToMessages(h.Payload.Results) + if err != nil { + return nil, err + } + + blockHeader, err := BlockHeaderToMessage(h.Header, signerIDs) + if err != nil { + return nil, err + } + + bh := entities.Block{ + Id: id[:], + Height: h.Header.Height, + ParentId: parentID[:], + Timestamp: t, + CollectionGuarantees: cg, + BlockSeals: seals, + Signatures: [][]byte{h.Header.ParentVoterSigData}, + ExecutionReceiptMetaList: ExecutionResultMetaListToMessages(h.Payload.Receipts), + ExecutionResultList: execResults, + BlockHeader: blockHeader, + } + + return &bh, nil +} + +// BlockToMessageLight converts a flow.Block to the light form of a protobuf Block message. +func BlockToMessageLight(h *flow.Block) *entities.Block { + id := h.ID() + + parentID := h.Header.ParentID + t := timestamppb.New(h.Header.Timestamp) + cg := CollectionGuaranteesToMessages(h.Payload.Guarantees) + + return &entities.Block{ + Id: id[:], + Height: h.Header.Height, + ParentId: parentID[:], + Timestamp: t, + CollectionGuarantees: cg, + Signatures: [][]byte{h.Header.ParentVoterSigData}, + } +} + +// MessageToBlock converts a protobuf Block message to a flow.Block. +func MessageToBlock(m *entities.Block) (*flow.Block, error) { + payload, err := PayloadFromMessage(m) + if err != nil { + return nil, fmt.Errorf("failed to extract payload data from message: %w", err) + } + header, err := MessageToBlockHeader(m.BlockHeader) + if err != nil { + return nil, fmt.Errorf("failed to convert block header: %w", err) + } + return &flow.Block{ + Header: header, + Payload: payload, + }, nil +} + +// BlockSealToMessage converts a flow.Seal to a protobuf BlockSeal message. +func BlockSealToMessage(s *flow.Seal) *entities.BlockSeal { + id := s.BlockID + result := s.ResultID + return &entities.BlockSeal{ + BlockId: id[:], + ExecutionReceiptId: result[:], + ExecutionReceiptSignatures: [][]byte{}, // filling seals signature with zero + FinalState: StateCommitmentToMessage(s.FinalState), + AggregatedApprovalSigs: AggregatedSignaturesToMessages(s.AggregatedApprovalSigs), + ResultId: IdentifierToMessage(s.ResultID), + } +} + +// MessageToBlockSeal converts a protobuf BlockSeal message to a flow.Seal. +func MessageToBlockSeal(m *entities.BlockSeal) (*flow.Seal, error) { + finalState, err := MessageToStateCommitment(m.FinalState) + if err != nil { + return nil, fmt.Errorf("failed to convert message to block seal: %w", err) + } + return &flow.Seal{ + BlockID: MessageToIdentifier(m.BlockId), + ResultID: MessageToIdentifier(m.ResultId), + FinalState: finalState, + AggregatedApprovalSigs: MessagesToAggregatedSignatures(m.AggregatedApprovalSigs), + }, nil +} + +// BlockSealsToMessages converts a slice of flow.Seal to a slice of protobuf BlockSeal messages. +func BlockSealsToMessages(b []*flow.Seal) []*entities.BlockSeal { + seals := make([]*entities.BlockSeal, len(b)) + for i, s := range b { + seals[i] = BlockSealToMessage(s) + } + return seals +} + +// MessagesToBlockSeals converts a slice of protobuf BlockSeal messages to a slice of flow.Seal. +func MessagesToBlockSeals(m []*entities.BlockSeal) ([]*flow.Seal, error) { + seals := make([]*flow.Seal, len(m)) + for i, s := range m { + msg, err := MessageToBlockSeal(s) + if err != nil { + return nil, err + } + seals[i] = msg + } + return seals, nil +} + +// PayloadFromMessage converts a protobuf Block message to a flow.Payload. +func PayloadFromMessage(m *entities.Block) (*flow.Payload, error) { + cgs := MessagesToCollectionGuarantees(m.CollectionGuarantees) + seals, err := MessagesToBlockSeals(m.BlockSeals) + if err != nil { + return nil, err + } + receipts := MessagesToExecutionResultMetaList(m.ExecutionReceiptMetaList) + results, err := MessagesToExecutionResults(m.ExecutionResultList) + if err != nil { + return nil, err + } + return &flow.Payload{ + Guarantees: cgs, + Seals: seals, + Receipts: receipts, + Results: results, + }, nil +} diff --git a/engine/common/rpc/convert/blocks_test.go b/engine/common/rpc/convert/blocks_test.go new file mode 100644 index 00000000000..87af2eabd85 --- /dev/null +++ b/engine/common/rpc/convert/blocks_test.go @@ -0,0 +1,65 @@ +package convert_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestConvertBlock tests that converting a block to and from a protobuf message results in the same +// block +func TestConvertBlock(t *testing.T) { + t.Parallel() + + block := unittest.FullBlockFixture() + block.SetPayload(unittest.PayloadFixture(unittest.WithAllTheFixins)) + + signerIDs := unittest.IdentifierListFixture(5) + + msg, err := convert.BlockToMessage(&block, signerIDs) + require.NoError(t, err) + + converted, err := convert.MessageToBlock(msg) + require.NoError(t, err) + + assert.Equal(t, block, *converted) +} + +// TestConvertBlockLight tests that converting a block to its light form results in only the correct +// fields being set +func TestConvertBlockLight(t *testing.T) { + t.Parallel() + + block := unittest.FullBlockFixture() + block.SetPayload(unittest.PayloadFixture(unittest.WithAllTheFixins)) + + msg := convert.BlockToMessageLight(&block) + + // required fields are set + blockID := block.ID() + assert.Equal(t, 0, bytes.Compare(blockID[:], msg.Id)) + assert.Equal(t, block.Header.Height, msg.Height) + assert.Equal(t, 0, bytes.Compare(block.Header.ParentID[:], msg.ParentId)) + assert.Equal(t, block.Header.Timestamp, msg.Timestamp.AsTime()) + assert.Equal(t, 0, bytes.Compare(block.Header.ParentVoterSigData, msg.Signatures[0])) + + guarantees := []*flow.CollectionGuarantee{} + for _, g := range msg.CollectionGuarantees { + guarantee := convert.MessageToCollectionGuarantee(g) + guarantees = append(guarantees, guarantee) + } + + assert.Equal(t, block.Payload.Guarantees, guarantees) + + // all other fields are not + assert.Nil(t, msg.BlockHeader) + assert.Len(t, msg.BlockSeals, 0) + assert.Len(t, msg.ExecutionReceiptMetaList, 0) + assert.Len(t, msg.ExecutionResultList, 0) +} diff --git a/engine/common/rpc/convert/collections.go b/engine/common/rpc/convert/collections.go new file mode 100644 index 00000000000..69725636449 --- /dev/null +++ b/engine/common/rpc/convert/collections.go @@ -0,0 +1,98 @@ +package convert + +import ( + "fmt" + + "github.com/onflow/flow/protobuf/go/flow/entities" + + "github.com/onflow/flow-go/model/flow" +) + +// CollectionToMessage converts a collection to a protobuf message +func CollectionToMessage(c *flow.Collection) (*entities.Collection, error) { + if c == nil || c.Transactions == nil { + return nil, fmt.Errorf("invalid collection") + } + + transactionsIDs := make([][]byte, len(c.Transactions)) + for i, t := range c.Transactions { + id := t.ID() + transactionsIDs[i] = id[:] + } + + collectionID := c.ID() + + ce := &entities.Collection{ + Id: collectionID[:], + TransactionIds: transactionsIDs, + } + + return ce, nil +} + +// LightCollectionToMessage converts a light collection to a protobuf message +func LightCollectionToMessage(c *flow.LightCollection) (*entities.Collection, error) { + if c == nil || c.Transactions == nil { + return nil, fmt.Errorf("invalid collection") + } + + collectionID := c.ID() + + return &entities.Collection{ + Id: collectionID[:], + TransactionIds: IdentifiersToMessages(c.Transactions), + }, nil +} + +// MessageToLightCollection converts a protobuf message to a light collection +func MessageToLightCollection(m *entities.Collection) (*flow.LightCollection, error) { + transactions := make([]flow.Identifier, 0, len(m.TransactionIds)) + for _, txId := range m.TransactionIds { + transactions = append(transactions, MessageToIdentifier(txId)) + } + + return &flow.LightCollection{ + Transactions: transactions, + }, nil +} + +// CollectionGuaranteeToMessage converts a collection guarantee to a protobuf message +func CollectionGuaranteeToMessage(g *flow.CollectionGuarantee) *entities.CollectionGuarantee { + id := g.ID() + + return &entities.CollectionGuarantee{ + CollectionId: id[:], + Signatures: [][]byte{g.Signature}, + ReferenceBlockId: IdentifierToMessage(g.ReferenceBlockID), + Signature: g.Signature, + SignerIndices: g.SignerIndices, + } +} + +// MessageToCollectionGuarantee converts a protobuf message to a collection guarantee +func MessageToCollectionGuarantee(m *entities.CollectionGuarantee) *flow.CollectionGuarantee { + return &flow.CollectionGuarantee{ + CollectionID: MessageToIdentifier(m.CollectionId), + ReferenceBlockID: MessageToIdentifier(m.ReferenceBlockId), + SignerIndices: m.SignerIndices, + Signature: MessageToSignature(m.Signature), + } +} + +// CollectionGuaranteesToMessages converts a slice of collection guarantees to a slice of protobuf messages +func CollectionGuaranteesToMessages(c []*flow.CollectionGuarantee) []*entities.CollectionGuarantee { + cg := make([]*entities.CollectionGuarantee, len(c)) + for i, g := range c { + cg[i] = CollectionGuaranteeToMessage(g) + } + return cg +} + +// MessagesToCollectionGuarantees converts a slice of protobuf messages to a slice of collection guarantees +func MessagesToCollectionGuarantees(m []*entities.CollectionGuarantee) []*flow.CollectionGuarantee { + cg := make([]*flow.CollectionGuarantee, len(m)) + for i, g := range m { + cg[i] = MessageToCollectionGuarantee(g) + } + return cg +} diff --git a/engine/common/rpc/convert/collections_test.go b/engine/common/rpc/convert/collections_test.go new file mode 100644 index 00000000000..2e14a6dc225 --- /dev/null +++ b/engine/common/rpc/convert/collections_test.go @@ -0,0 +1,86 @@ +package convert_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" + + "github.com/onflow/flow/protobuf/go/flow/entities" +) + +// TestConvertCollection tests that converting a collection to a protobuf message results in the correct +// set of transaction IDs +func TestConvertCollection(t *testing.T) { + t.Parallel() + + collection := unittest.CollectionFixture(5) + txIDs := make([]flow.Identifier, 0, len(collection.Transactions)) + for _, tx := range collection.Transactions { + txIDs = append(txIDs, tx.ID()) + } + + t.Run("convert collection to message", func(t *testing.T) { + msg, err := convert.CollectionToMessage(&collection) + require.NoError(t, err) + + assert.Len(t, msg.TransactionIds, len(txIDs)) + for i, txID := range txIDs { + assert.Equal(t, txID[:], msg.TransactionIds[i]) + } + }) + + var msg *entities.Collection + lightCollection := flow.LightCollection{Transactions: txIDs} + + t.Run("convert light collection to message", func(t *testing.T) { + var err error + msg, err = convert.LightCollectionToMessage(&lightCollection) + require.NoError(t, err) + + assert.Len(t, msg.TransactionIds, len(txIDs)) + for i, txID := range txIDs { + assert.Equal(t, txID[:], msg.TransactionIds[i]) + } + }) + + t.Run("convert message to light collection", func(t *testing.T) { + lightColl, err := convert.MessageToLightCollection(msg) + require.NoError(t, err) + + assert.Equal(t, len(txIDs), len(lightColl.Transactions)) + for _, txID := range lightColl.Transactions { + assert.Equal(t, txID[:], txID[:]) + } + }) +} + +// TestConvertCollectionGuarantee tests that converting a collection guarantee to and from a protobuf +// message results in the same collection guarantee +func TestConvertCollectionGuarantee(t *testing.T) { + t.Parallel() + + guarantee := unittest.CollectionGuaranteeFixture(unittest.WithCollRef(unittest.IdentifierFixture())) + + msg := convert.CollectionGuaranteeToMessage(guarantee) + converted := convert.MessageToCollectionGuarantee(msg) + + assert.Equal(t, guarantee, converted) +} + +// TestConvertCollectionGuarantees tests that converting a collection guarantee to and from a protobuf +// message results in the same collection guarantee +func TestConvertCollectionGuarantees(t *testing.T) { + t.Parallel() + + guarantees := unittest.CollectionGuaranteesFixture(5, unittest.WithCollRef(unittest.IdentifierFixture())) + + msg := convert.CollectionGuaranteesToMessages(guarantees) + converted := convert.MessagesToCollectionGuarantees(msg) + + assert.Equal(t, guarantees, converted) +} diff --git a/engine/common/rpc/convert/convert.go b/engine/common/rpc/convert/convert.go index f1b698e6b11..3419e997def 100644 --- a/engine/common/rpc/convert/convert.go +++ b/engine/common/rpc/convert/convert.go @@ -1,22 +1,13 @@ package convert import ( - "encoding/json" "errors" "fmt" "github.com/onflow/flow/protobuf/go/flow/entities" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/timestamppb" "github.com/onflow/flow-go/crypto" - "github.com/onflow/flow-go/crypto/hash" - "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/executiondatasync/execution_data" - "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/state/protocol/inmem" ) var ErrEmptyMessage = errors.New("protobuf message is empty") @@ -31,198 +22,8 @@ var ValidChainIds = map[string]bool{ flow.MonotonicEmulator.String(): true, } -func MessageToTransaction(m *entities.Transaction, chain flow.Chain) (flow.TransactionBody, error) { - if m == nil { - return flow.TransactionBody{}, ErrEmptyMessage - } - - t := flow.NewTransactionBody() - - proposalKey := m.GetProposalKey() - if proposalKey != nil { - proposalAddress, err := Address(proposalKey.GetAddress(), chain) - if err != nil { - return *t, err - } - t.SetProposalKey(proposalAddress, uint64(proposalKey.GetKeyId()), proposalKey.GetSequenceNumber()) - } - - payer := m.GetPayer() - if payer != nil { - payerAddress, err := Address(payer, chain) - if err != nil { - return *t, err - } - t.SetPayer(payerAddress) - } - - for _, authorizer := range m.GetAuthorizers() { - authorizerAddress, err := Address(authorizer, chain) - if err != nil { - return *t, err - } - t.AddAuthorizer(authorizerAddress) - } - - for _, sig := range m.GetPayloadSignatures() { - addr, err := Address(sig.GetAddress(), chain) - if err != nil { - return *t, err - } - t.AddPayloadSignature(addr, uint64(sig.GetKeyId()), sig.GetSignature()) - } - - for _, sig := range m.GetEnvelopeSignatures() { - addr, err := Address(sig.GetAddress(), chain) - if err != nil { - return *t, err - } - t.AddEnvelopeSignature(addr, uint64(sig.GetKeyId()), sig.GetSignature()) - } - - t.SetScript(m.GetScript()) - t.SetArguments(m.GetArguments()) - t.SetReferenceBlockID(flow.HashToID(m.GetReferenceBlockId())) - t.SetGasLimit(m.GetGasLimit()) - - return *t, nil -} - -func TransactionsToMessages(transactions []*flow.TransactionBody) []*entities.Transaction { - transactionMessages := make([]*entities.Transaction, len(transactions)) - for i, t := range transactions { - transactionMessages[i] = TransactionToMessage(*t) - } - return transactionMessages -} - -func TransactionToMessage(tb flow.TransactionBody) *entities.Transaction { - proposalKeyMessage := &entities.Transaction_ProposalKey{ - Address: tb.ProposalKey.Address.Bytes(), - KeyId: uint32(tb.ProposalKey.KeyIndex), - SequenceNumber: tb.ProposalKey.SequenceNumber, - } - - authMessages := make([][]byte, len(tb.Authorizers)) - for i, auth := range tb.Authorizers { - authMessages[i] = auth.Bytes() - } - - payloadSigMessages := make([]*entities.Transaction_Signature, len(tb.PayloadSignatures)) - - for i, sig := range tb.PayloadSignatures { - payloadSigMessages[i] = &entities.Transaction_Signature{ - Address: sig.Address.Bytes(), - KeyId: uint32(sig.KeyIndex), - Signature: sig.Signature, - } - } - - envelopeSigMessages := make([]*entities.Transaction_Signature, len(tb.EnvelopeSignatures)) - - for i, sig := range tb.EnvelopeSignatures { - envelopeSigMessages[i] = &entities.Transaction_Signature{ - Address: sig.Address.Bytes(), - KeyId: uint32(sig.KeyIndex), - Signature: sig.Signature, - } - } - - return &entities.Transaction{ - Script: tb.Script, - Arguments: tb.Arguments, - ReferenceBlockId: tb.ReferenceBlockID[:], - GasLimit: tb.GasLimit, - ProposalKey: proposalKeyMessage, - Payer: tb.Payer.Bytes(), - Authorizers: authMessages, - PayloadSignatures: payloadSigMessages, - EnvelopeSignatures: envelopeSigMessages, - } -} - -func BlockHeaderToMessage(h *flow.Header, signerIDs flow.IdentifierList) (*entities.BlockHeader, error) { - id := h.ID() - - t := timestamppb.New(h.Timestamp) - var lastViewTC *entities.TimeoutCertificate - if h.LastViewTC != nil { - newestQC := h.LastViewTC.NewestQC - lastViewTC = &entities.TimeoutCertificate{ - View: h.LastViewTC.View, - HighQcViews: h.LastViewTC.NewestQCViews, - SignerIndices: h.LastViewTC.SignerIndices, - SigData: h.LastViewTC.SigData, - HighestQc: &entities.QuorumCertificate{ - View: newestQC.View, - BlockId: newestQC.BlockID[:], - SignerIndices: newestQC.SignerIndices, - SigData: newestQC.SigData, - }, - } - } - parentVoterIds := IdentifiersToMessages(signerIDs) - - return &entities.BlockHeader{ - Id: id[:], - ParentId: h.ParentID[:], - Height: h.Height, - PayloadHash: h.PayloadHash[:], - Timestamp: t, - View: h.View, - ParentView: h.ParentView, - ParentVoterIndices: h.ParentVoterIndices, - ParentVoterIds: parentVoterIds, - ParentVoterSigData: h.ParentVoterSigData, - ProposerId: h.ProposerID[:], - ProposerSigData: h.ProposerSigData, - ChainId: h.ChainID.String(), - LastViewTc: lastViewTC, - }, nil -} - -func MessageToBlockHeader(m *entities.BlockHeader) (*flow.Header, error) { - chainId, err := MessageToChainId(m.ChainId) - if err != nil { - return nil, fmt.Errorf("failed to convert ChainId: %w", err) - } - var lastViewTC *flow.TimeoutCertificate - if m.LastViewTc != nil { - newestQC := m.LastViewTc.HighestQc - if newestQC == nil { - return nil, fmt.Errorf("invalid structure newest QC should be present") - } - lastViewTC = &flow.TimeoutCertificate{ - View: m.LastViewTc.View, - NewestQCViews: m.LastViewTc.HighQcViews, - SignerIndices: m.LastViewTc.SignerIndices, - SigData: m.LastViewTc.SigData, - NewestQC: &flow.QuorumCertificate{ - View: newestQC.View, - BlockID: MessageToIdentifier(newestQC.BlockId), - SignerIndices: newestQC.SignerIndices, - SigData: newestQC.SigData, - }, - } - } - - return &flow.Header{ - ParentID: MessageToIdentifier(m.ParentId), - Height: m.Height, - PayloadHash: MessageToIdentifier(m.PayloadHash), - Timestamp: m.Timestamp.AsTime(), - View: m.View, - ParentView: m.ParentView, - ParentVoterIndices: m.ParentVoterIndices, - ParentVoterSigData: m.ParentVoterSigData, - ProposerID: MessageToIdentifier(m.ProposerId), - ProposerSigData: m.ProposerSigData, - ChainID: *chainId, - LastViewTC: lastViewTC, - }, nil -} - -// MessageToChainId checks chainId from enumeration to prevent a panic on Chain() being called +// MessageToChainId converts the chainID from a protobuf message to a flow.ChainID +// It returns an error if the value is not a valid chainId func MessageToChainId(m string) (*flow.ChainID, error) { if !ValidChainIds[m] { return nil, fmt.Errorf("invalid chainId %s: ", m) @@ -231,200 +32,21 @@ func MessageToChainId(m string) (*flow.ChainID, error) { return &chainId, nil } -func CollectionGuaranteesToMessages(c []*flow.CollectionGuarantee) []*entities.CollectionGuarantee { - cg := make([]*entities.CollectionGuarantee, len(c)) - for i, g := range c { - cg[i] = CollectionGuaranteeToMessage(g) - } - return cg -} - -func MessagesToCollectionGuarantees(m []*entities.CollectionGuarantee) []*flow.CollectionGuarantee { - cg := make([]*flow.CollectionGuarantee, len(m)) - for i, g := range m { - cg[i] = MessageToCollectionGuarantee(g) - } - return cg -} - -func BlockSealsToMessages(b []*flow.Seal) []*entities.BlockSeal { - seals := make([]*entities.BlockSeal, len(b)) - for i, s := range b { - seals[i] = BlockSealToMessage(s) - } - return seals -} - -func MessagesToBlockSeals(m []*entities.BlockSeal) ([]*flow.Seal, error) { - seals := make([]*flow.Seal, len(m)) - for i, s := range m { - msg, err := MessageToBlockSeal(s) - if err != nil { - return nil, err - } - seals[i] = msg - } - return seals, nil -} - -func ExecutionResultsToMessages(e []*flow.ExecutionResult) ([]*entities.ExecutionResult, error) { - execResults := make([]*entities.ExecutionResult, len(e)) - for i, execRes := range e { - parsedExecResult, err := ExecutionResultToMessage(execRes) - if err != nil { - return nil, err - } - execResults[i] = parsedExecResult - } - return execResults, nil -} - -func MessagesToExecutionResults(m []*entities.ExecutionResult) ([]*flow.ExecutionResult, error) { - execResults := make([]*flow.ExecutionResult, len(m)) - for i, e := range m { - parsedExecResult, err := MessageToExecutionResult(e) - if err != nil { - return nil, fmt.Errorf("failed to convert message at index %d to execution result: %w", i, err) - } - execResults[i] = parsedExecResult - } - return execResults, nil -} - -func BlockToMessage(h *flow.Block, signerIDs flow.IdentifierList) (*entities.Block, error) { - - id := h.ID() - - parentID := h.Header.ParentID - t := timestamppb.New(h.Header.Timestamp) - cg := CollectionGuaranteesToMessages(h.Payload.Guarantees) - - seals := BlockSealsToMessages(h.Payload.Seals) - - execResults, err := ExecutionResultsToMessages(h.Payload.Results) - if err != nil { - return nil, err - } - - blockHeader, err := BlockHeaderToMessage(h.Header, signerIDs) - if err != nil { - return nil, err - } - - bh := entities.Block{ - Id: id[:], - Height: h.Header.Height, - ParentId: parentID[:], - Timestamp: t, - CollectionGuarantees: cg, - BlockSeals: seals, - Signatures: [][]byte{h.Header.ParentVoterSigData}, - ExecutionReceiptMetaList: ExecutionResultMetaListToMessages(h.Payload.Receipts), - ExecutionResultList: execResults, - BlockHeader: blockHeader, - } - - return &bh, nil -} - -func BlockToMessageLight(h *flow.Block) *entities.Block { - id := h.ID() - - parentID := h.Header.ParentID - t := timestamppb.New(h.Header.Timestamp) - cg := CollectionGuaranteesToMessages(h.Payload.Guarantees) - - return &entities.Block{ - Id: id[:], - Height: h.Header.Height, - ParentId: parentID[:], - Timestamp: t, - CollectionGuarantees: cg, - Signatures: [][]byte{h.Header.ParentVoterSigData}, - } -} - -func MessageToBlock(m *entities.Block) (*flow.Block, error) { - payload, err := PayloadFromMessage(m) - if err != nil { - return nil, fmt.Errorf("failed to extract payload data from message: %w", err) - } - header, err := MessageToBlockHeader(m.BlockHeader) - if err != nil { - return nil, fmt.Errorf("failed to convert block header: %w", err) - } - return &flow.Block{ - Header: header, - Payload: payload, - }, nil -} - -func MessagesToExecutionResultMetaList(m []*entities.ExecutionReceiptMeta) flow.ExecutionReceiptMetaList { - execMetaList := make([]*flow.ExecutionReceiptMeta, len(m)) - for i, message := range m { - execMetaList[i] = &flow.ExecutionReceiptMeta{ - ExecutorID: MessageToIdentifier(message.ExecutorId), - ResultID: MessageToIdentifier(message.ResultId), - Spocks: MessagesToSignatures(message.Spocks), - ExecutorSignature: MessageToSignature(message.ExecutorSignature), - } - } - return execMetaList[:] -} - -func ExecutionResultMetaListToMessages(e flow.ExecutionReceiptMetaList) []*entities.ExecutionReceiptMeta { - messageList := make([]*entities.ExecutionReceiptMeta, len(e)) - for i, execMeta := range e { - messageList[i] = &entities.ExecutionReceiptMeta{ - ExecutorId: IdentifierToMessage(execMeta.ExecutorID), - ResultId: IdentifierToMessage(execMeta.ResultID), - Spocks: SignaturesToMessages(execMeta.Spocks), - ExecutorSignature: MessageToSignature(execMeta.ExecutorSignature), +// AggregatedSignaturesToMessages converts a slice of AggregatedSignature structs to a corresponding +// slice of protobuf messages +func AggregatedSignaturesToMessages(a []flow.AggregatedSignature) []*entities.AggregatedSignature { + parsedMessages := make([]*entities.AggregatedSignature, len(a)) + for i, sig := range a { + parsedMessages[i] = &entities.AggregatedSignature{ + SignerIds: IdentifiersToMessages(sig.SignerIDs), + VerifierSignatures: SignaturesToMessages(sig.VerifierSignatures), } } - return messageList -} - -func PayloadFromMessage(m *entities.Block) (*flow.Payload, error) { - cgs := MessagesToCollectionGuarantees(m.CollectionGuarantees) - seals, err := MessagesToBlockSeals(m.BlockSeals) - if err != nil { - return nil, err - } - receipts := MessagesToExecutionResultMetaList(m.ExecutionReceiptMetaList) - results, err := MessagesToExecutionResults(m.ExecutionResultList) - if err != nil { - return nil, err - } - return &flow.Payload{ - Guarantees: cgs, - Seals: seals, - Receipts: receipts, - Results: results, - }, nil -} - -func CollectionGuaranteeToMessage(g *flow.CollectionGuarantee) *entities.CollectionGuarantee { - id := g.ID() - - return &entities.CollectionGuarantee{ - CollectionId: id[:], - Signatures: [][]byte{g.Signature}, - ReferenceBlockId: IdentifierToMessage(g.ReferenceBlockID), - Signature: g.Signature, - SignerIndices: g.SignerIndices, - } -} - -func MessageToCollectionGuarantee(m *entities.CollectionGuarantee) *flow.CollectionGuarantee { - return &flow.CollectionGuarantee{ - CollectionID: MessageToIdentifier(m.CollectionId), - ReferenceBlockID: MessageToIdentifier(m.ReferenceBlockId), - SignerIndices: m.SignerIndices, - Signature: MessageToSignature(m.Signature), - } + return parsedMessages } +// MessagesToAggregatedSignatures converts a slice of protobuf messages to their corresponding +// AggregatedSignature structs func MessagesToAggregatedSignatures(m []*entities.AggregatedSignature) []flow.AggregatedSignature { parsedSignatures := make([]flow.AggregatedSignature, len(m)) for i, message := range m { @@ -436,29 +58,17 @@ func MessagesToAggregatedSignatures(m []*entities.AggregatedSignature) []flow.Ag return parsedSignatures } -func AggregatedSignaturesToMessages(a []flow.AggregatedSignature) []*entities.AggregatedSignature { - parsedMessages := make([]*entities.AggregatedSignature, len(a)) - for i, sig := range a { - parsedMessages[i] = &entities.AggregatedSignature{ - SignerIds: IdentifiersToMessages(sig.SignerIDs), - VerifierSignatures: SignaturesToMessages(sig.VerifierSignatures), - } - } - return parsedMessages -} - -func MessagesToSignatures(m [][]byte) []crypto.Signature { - signatures := make([]crypto.Signature, len(m)) - for i, message := range m { - signatures[i] = MessageToSignature(message) - } - return signatures +// SignatureToMessage converts a crypto.Signature to a byte slice for inclusion in a protobuf message +func SignatureToMessage(s crypto.Signature) []byte { + return s[:] } +// MessageToSignature converts a byte slice from a protobuf message to a crypto.Signature func MessageToSignature(m []byte) crypto.Signature { return m[:] } +// SignaturesToMessages converts a slice of crypto.Signatures to a slice of byte slices for inclusion in a protobuf message func SignaturesToMessages(s []crypto.Signature) [][]byte { messages := make([][]byte, len(s)) for i, sig := range s { @@ -467,208 +77,26 @@ func SignaturesToMessages(s []crypto.Signature) [][]byte { return messages } -func SignatureToMessage(s crypto.Signature) []byte { - return s[:] -} - -func BlockSealToMessage(s *flow.Seal) *entities.BlockSeal { - id := s.BlockID - result := s.ResultID - return &entities.BlockSeal{ - BlockId: id[:], - ExecutionReceiptId: result[:], - ExecutionReceiptSignatures: [][]byte{}, // filling seals signature with zero - FinalState: StateCommitmentToMessage(s.FinalState), - AggregatedApprovalSigs: AggregatedSignaturesToMessages(s.AggregatedApprovalSigs), - ResultId: IdentifierToMessage(s.ResultID), - } -} - -func MessageToBlockSeal(m *entities.BlockSeal) (*flow.Seal, error) { - finalState, err := MessageToStateCommitment(m.FinalState) - if err != nil { - return nil, fmt.Errorf("failed to convert message to block seal: %w", err) - } - return &flow.Seal{ - BlockID: MessageToIdentifier(m.BlockId), - ResultID: MessageToIdentifier(m.ResultId), - FinalState: finalState, - AggregatedApprovalSigs: MessagesToAggregatedSignatures(m.AggregatedApprovalSigs), - }, nil -} - -func CollectionToMessage(c *flow.Collection) (*entities.Collection, error) { - if c == nil || c.Transactions == nil { - return nil, fmt.Errorf("invalid collection") - } - - transactionsIDs := make([][]byte, len(c.Transactions)) - for i, t := range c.Transactions { - id := t.ID() - transactionsIDs[i] = id[:] - } - - collectionID := c.ID() - - ce := &entities.Collection{ - Id: collectionID[:], - TransactionIds: transactionsIDs, - } - - return ce, nil -} - -func LightCollectionToMessage(c *flow.LightCollection) (*entities.Collection, error) { - if c == nil || c.Transactions == nil { - return nil, fmt.Errorf("invalid collection") - } - - collectionID := c.ID() - - return &entities.Collection{ - Id: collectionID[:], - TransactionIds: IdentifiersToMessages(c.Transactions), - }, nil -} - -func EventToMessage(e flow.Event) *entities.Event { - return &entities.Event{ - Type: string(e.Type), - TransactionId: e.TransactionID[:], - TransactionIndex: e.TransactionIndex, - EventIndex: e.EventIndex, - Payload: e.Payload, - } -} - -func MessageToAccount(m *entities.Account) (*flow.Account, error) { - if m == nil { - return nil, ErrEmptyMessage - } - - accountKeys := make([]flow.AccountPublicKey, len(m.GetKeys())) - for i, key := range m.GetKeys() { - accountKey, err := MessageToAccountKey(key) - if err != nil { - return nil, err - } - - accountKeys[i] = *accountKey - } - - return &flow.Account{ - Address: flow.BytesToAddress(m.GetAddress()), - Balance: m.GetBalance(), - Keys: accountKeys, - Contracts: m.Contracts, - }, nil -} - -func AccountToMessage(a *flow.Account) (*entities.Account, error) { - keys := make([]*entities.AccountKey, len(a.Keys)) - for i, k := range a.Keys { - messageKey, err := AccountKeyToMessage(k) - if err != nil { - return nil, err - } - keys[i] = messageKey - } - - return &entities.Account{ - Address: a.Address.Bytes(), - Balance: a.Balance, - Code: nil, - Keys: keys, - Contracts: a.Contracts, - }, nil -} - -func MessageToAccountKey(m *entities.AccountKey) (*flow.AccountPublicKey, error) { - if m == nil { - return nil, ErrEmptyMessage - } - - sigAlgo := crypto.SigningAlgorithm(m.GetSignAlgo()) - hashAlgo := hash.HashingAlgorithm(m.GetHashAlgo()) - - publicKey, err := crypto.DecodePublicKey(sigAlgo, m.GetPublicKey()) - if err != nil { - return nil, err - } - - return &flow.AccountPublicKey{ - Index: int(m.GetIndex()), - PublicKey: publicKey, - SignAlgo: sigAlgo, - HashAlgo: hashAlgo, - Weight: int(m.GetWeight()), - SeqNumber: uint64(m.GetSequenceNumber()), - Revoked: m.GetRevoked(), - }, nil -} - -func AccountKeyToMessage(a flow.AccountPublicKey) (*entities.AccountKey, error) { - publicKey := a.PublicKey.Encode() - return &entities.AccountKey{ - Index: uint32(a.Index), - PublicKey: publicKey, - SignAlgo: uint32(a.SignAlgo), - HashAlgo: uint32(a.HashAlgo), - Weight: uint32(a.Weight), - SequenceNumber: uint32(a.SeqNumber), - Revoked: a.Revoked, - }, nil -} - -func MessagesToEvents(l []*entities.Event) []flow.Event { - events := make([]flow.Event, len(l)) - - for i, m := range l { - events[i] = MessageToEvent(m) - } - - return events -} - -func MessageToEvent(m *entities.Event) flow.Event { - return flow.Event{ - Type: flow.EventType(m.GetType()), - TransactionID: flow.HashToID(m.GetTransactionId()), - TransactionIndex: m.GetTransactionIndex(), - EventIndex: m.GetEventIndex(), - Payload: m.GetPayload(), - } -} - -func EventsToMessages(flowEvents []flow.Event) []*entities.Event { - events := make([]*entities.Event, len(flowEvents)) - for i, e := range flowEvents { - event := EventToMessage(e) - events[i] = event +// MessagesToSignatures converts a slice of byte slices from a protobuf message to a slice of crypto.Signatures +func MessagesToSignatures(m [][]byte) []crypto.Signature { + signatures := make([]crypto.Signature, len(m)) + for i, message := range m { + signatures[i] = MessageToSignature(message) } - return events + return signatures } +// IdentifierToMessage converts a flow.Identifier to a byte slice for inclusion in a protobuf message func IdentifierToMessage(i flow.Identifier) []byte { return i[:] } +// MessageToIdentifier converts a byte slice from a protobuf message to a flow.Identifier func MessageToIdentifier(b []byte) flow.Identifier { return flow.HashToID(b) } -func StateCommitmentToMessage(s flow.StateCommitment) []byte { - return s[:] -} - -func MessageToStateCommitment(bytes []byte) (sc flow.StateCommitment, err error) { - if len(bytes) != len(sc) { - return sc, fmt.Errorf("invalid state commitment length. got %d expected %d", len(bytes), len(sc)) - } - copy(sc[:], bytes) - return -} - +// IdentifiersToMessages converts a slice of flow.Identifiers to a slice of byte slices for inclusion in a protobuf message func IdentifiersToMessages(l []flow.Identifier) [][]byte { results := make([][]byte, len(l)) for i, item := range l { @@ -677,6 +105,7 @@ func IdentifiersToMessages(l []flow.Identifier) [][]byte { return results } +// MessagesToIdentifiers converts a slice of byte slices from a protobuf message to a slice of flow.Identifiers func MessagesToIdentifiers(l [][]byte) []flow.Identifier { results := make([]flow.Identifier, len(l)) for i, item := range l { @@ -685,409 +114,16 @@ func MessagesToIdentifiers(l [][]byte) []flow.Identifier { return results } -// SnapshotToBytes converts a `protocol.Snapshot` to bytes, encoded as JSON -func SnapshotToBytes(snapshot protocol.Snapshot) ([]byte, error) { - serializable, err := inmem.FromSnapshot(snapshot) - if err != nil { - return nil, err - } - - data, err := json.Marshal(serializable.Encodable()) - if err != nil { - return nil, err - } - - return data, nil -} - -// BytesToInmemSnapshot converts an array of bytes to `inmem.Snapshot` -func BytesToInmemSnapshot(bytes []byte) (*inmem.Snapshot, error) { - var encodable inmem.EncodableSnapshot - err := json.Unmarshal(bytes, &encodable) - if err != nil { - return nil, fmt.Errorf("could not unmarshal decoded snapshot: %w", err) - } - - return inmem.SnapshotFromEncodable(encodable), nil -} - -func MessagesToChunkList(m []*entities.Chunk) (flow.ChunkList, error) { - parsedChunks := make(flow.ChunkList, len(m)) - for i, chunk := range m { - parsedChunk, err := MessageToChunk(chunk) - if err != nil { - return nil, fmt.Errorf("failed to parse message at index %d to chunk: %w", i, err) - } - parsedChunks[i] = parsedChunk - } - return parsedChunks, nil -} - -func MessagesToServiceEventList(m []*entities.ServiceEvent) (flow.ServiceEventList, error) { - parsedServiceEvents := make(flow.ServiceEventList, len(m)) - for i, serviceEvent := range m { - parsedServiceEvent, err := MessageToServiceEvent(serviceEvent) - if err != nil { - return nil, fmt.Errorf("failed to parse service event at index %d from message: %w", i, err) - } - parsedServiceEvents[i] = *parsedServiceEvent - } - return parsedServiceEvents, nil -} - -func MessageToExecutionResult(m *entities.ExecutionResult) (*flow.ExecutionResult, error) { - // convert Chunks - parsedChunks, err := MessagesToChunkList(m.Chunks) - if err != nil { - return nil, fmt.Errorf("failed to parse messages to ChunkList: %w", err) - } - // convert ServiceEvents - parsedServiceEvents, err := MessagesToServiceEventList(m.ServiceEvents) - if err != nil { - return nil, err - } - return &flow.ExecutionResult{ - PreviousResultID: MessageToIdentifier(m.PreviousResultId), - BlockID: MessageToIdentifier(m.BlockId), - Chunks: parsedChunks, - ServiceEvents: parsedServiceEvents, - ExecutionDataID: MessageToIdentifier(m.ExecutionDataId), - }, nil -} - -func ExecutionResultToMessage(er *flow.ExecutionResult) (*entities.ExecutionResult, error) { - - chunks := make([]*entities.Chunk, len(er.Chunks)) - - for i, chunk := range er.Chunks { - chunks[i] = ChunkToMessage(chunk) - } - - serviceEvents := make([]*entities.ServiceEvent, len(er.ServiceEvents)) - var err error - for i, serviceEvent := range er.ServiceEvents { - serviceEvents[i], err = ServiceEventToMessage(serviceEvent) - if err != nil { - return nil, fmt.Errorf("error while convering service event %d: %w", i, err) - } - } - - return &entities.ExecutionResult{ - PreviousResultId: IdentifierToMessage(er.PreviousResultID), - BlockId: IdentifierToMessage(er.BlockID), - Chunks: chunks, - ServiceEvents: serviceEvents, - ExecutionDataId: IdentifierToMessage(er.ExecutionDataID), - }, nil -} - -func ServiceEventToMessage(event flow.ServiceEvent) (*entities.ServiceEvent, error) { - - bytes, err := json.Marshal(event.Event) - if err != nil { - return nil, fmt.Errorf("cannot marshal service event: %w", err) - } - - return &entities.ServiceEvent{ - Type: event.Type, - Payload: bytes, - }, nil -} - -func MessageToServiceEvent(m *entities.ServiceEvent) (*flow.ServiceEvent, error) { - var event interface{} - rawEvent := m.Payload - // map keys correctly - switch m.Type { - case flow.ServiceEventSetup: - setup := new(flow.EpochSetup) - err := json.Unmarshal(rawEvent, setup) - if err != nil { - return nil, fmt.Errorf("failed to marshal to EpochSetup event: %w", err) - } - event = setup - case flow.ServiceEventCommit: - commit := new(flow.EpochCommit) - err := json.Unmarshal(rawEvent, commit) - if err != nil { - return nil, fmt.Errorf("failed to marshal to EpochCommit event: %w", err) - } - event = commit - default: - return nil, fmt.Errorf("invalid event type: %s", m.Type) - } - return &flow.ServiceEvent{ - Type: m.Type, - Event: event, - }, nil -} - -func ChunkToMessage(chunk *flow.Chunk) *entities.Chunk { - return &entities.Chunk{ - CollectionIndex: uint32(chunk.CollectionIndex), - StartState: StateCommitmentToMessage(chunk.StartState), - EventCollection: IdentifierToMessage(chunk.EventCollection), - BlockId: IdentifierToMessage(chunk.BlockID), - TotalComputationUsed: chunk.TotalComputationUsed, - NumberOfTransactions: uint32(chunk.NumberOfTransactions), - Index: chunk.Index, - EndState: StateCommitmentToMessage(chunk.EndState), - } -} - -func MessageToChunk(m *entities.Chunk) (*flow.Chunk, error) { - startState, err := flow.ToStateCommitment(m.StartState) - if err != nil { - return nil, fmt.Errorf("failed to parse Message start state to Chunk: %w", err) - } - endState, err := flow.ToStateCommitment(m.EndState) - if err != nil { - return nil, fmt.Errorf("failed to parse Message end state to Chunk: %w", err) - } - chunkBody := flow.ChunkBody{ - CollectionIndex: uint(m.CollectionIndex), - StartState: startState, - EventCollection: MessageToIdentifier(m.EventCollection), - BlockID: MessageToIdentifier(m.BlockId), - TotalComputationUsed: m.TotalComputationUsed, - NumberOfTransactions: uint64(m.NumberOfTransactions), - } - return &flow.Chunk{ - ChunkBody: chunkBody, - Index: m.Index, - EndState: endState, - }, nil -} - -func BlockExecutionDataToMessage(data *execution_data.BlockExecutionData) (*entities.BlockExecutionData, error) { - chunkExecutionDatas := make([]*entities.ChunkExecutionData, len(data.ChunkExecutionDatas)) - for i, chunk := range data.ChunkExecutionDatas { - chunkMessage, err := ChunkExecutionDataToMessage(chunk) - if err != nil { - return nil, err - } - chunkExecutionDatas[i] = chunkMessage - } - return &entities.BlockExecutionData{ - BlockId: IdentifierToMessage(data.BlockID), - ChunkExecutionData: chunkExecutionDatas, - }, nil -} - -func ChunkExecutionDataToMessage(data *execution_data.ChunkExecutionData) (*entities.ChunkExecutionData, error) { - collection := &entities.ExecutionDataCollection{} - if data.Collection != nil { - collection = &entities.ExecutionDataCollection{ - Transactions: TransactionsToMessages(data.Collection.Transactions), - } - } - - events := EventsToMessages(data.Events) - if len(events) == 0 { - events = nil - } - - var trieUpdate *entities.TrieUpdate - if data.TrieUpdate != nil { - paths := make([][]byte, len(data.TrieUpdate.Paths)) - for i, path := range data.TrieUpdate.Paths { - paths[i] = path[:] - } - - payloads := make([]*entities.Payload, len(data.TrieUpdate.Payloads)) - for i, payload := range data.TrieUpdate.Payloads { - key, err := payload.Key() - if err != nil { - return nil, err - } - keyParts := make([]*entities.KeyPart, len(key.KeyParts)) - for j, keyPart := range key.KeyParts { - keyParts[j] = &entities.KeyPart{ - Type: uint32(keyPart.Type), - Value: keyPart.Value, - } - } - payloads[i] = &entities.Payload{ - KeyPart: keyParts, - Value: payload.Value(), - } - } - - trieUpdate = &entities.TrieUpdate{ - RootHash: data.TrieUpdate.RootHash[:], - Paths: paths, - Payloads: payloads, - } - } - - return &entities.ChunkExecutionData{ - Collection: collection, - Events: events, - TrieUpdate: trieUpdate, - }, nil -} - -func MessageToBlockExecutionData(m *entities.BlockExecutionData, chain flow.Chain) (*execution_data.BlockExecutionData, error) { - if m == nil { - return nil, ErrEmptyMessage - } - chunks := make([]*execution_data.ChunkExecutionData, len(m.ChunkExecutionData)) - for i, chunk := range m.GetChunkExecutionData() { - convertedChunk, err := MessageToChunkExecutionData(chunk, chain) - if err != nil { - return nil, err - } - chunks[i] = convertedChunk - } - - return &execution_data.BlockExecutionData{ - BlockID: MessageToIdentifier(m.GetBlockId()), - ChunkExecutionDatas: chunks, - }, nil -} - -func MessageToChunkExecutionData(m *entities.ChunkExecutionData, chain flow.Chain) (*execution_data.ChunkExecutionData, error) { - collection, err := messageToTrustedCollection(m.GetCollection(), chain) - if err != nil { - return nil, err - } - - var trieUpdate *ledger.TrieUpdate - if m.GetTrieUpdate() != nil { - trieUpdate, err = MessageToTrieUpdate(m.GetTrieUpdate()) - if err != nil { - return nil, err - } - } - - events := MessagesToEvents(m.GetEvents()) - if len(events) == 0 { - events = nil - } - - return &execution_data.ChunkExecutionData{ - Collection: collection, - Events: events, - TrieUpdate: trieUpdate, - }, nil -} - -func messageToTrustedCollection(m *entities.ExecutionDataCollection, chain flow.Chain) (*flow.Collection, error) { - messages := m.GetTransactions() - transactions := make([]*flow.TransactionBody, len(messages)) - for i, message := range messages { - transaction, err := messageToTrustedTransaction(message, chain) - if err != nil { - return nil, fmt.Errorf("could not convert transaction %d: %w", i, err) - } - transactions[i] = &transaction - } - - if len(transactions) == 0 { - return nil, nil - } - - return &flow.Collection{Transactions: transactions}, nil -} - -// messageToTrustedTransaction converts a transaction message to a transaction body. -// This is useful when converting transactions from trusted state like BlockExecutionData which -// contain service transactions that do not conform to external transaction format. -func messageToTrustedTransaction(m *entities.Transaction, chain flow.Chain) (flow.TransactionBody, error) { - if m == nil { - return flow.TransactionBody{}, ErrEmptyMessage - } - - t := flow.NewTransactionBody() - - proposalKey := m.GetProposalKey() - if proposalKey != nil { - proposalAddress, err := insecureAddress(proposalKey.GetAddress(), chain) - if err != nil { - return *t, fmt.Errorf("could not convert proposer address: %w", err) - } - t.SetProposalKey(proposalAddress, uint64(proposalKey.GetKeyId()), proposalKey.GetSequenceNumber()) - } - - payer := m.GetPayer() - if payer != nil { - payerAddress, err := insecureAddress(payer, chain) - if err != nil { - return *t, fmt.Errorf("could not convert payer address: %w", err) - } - t.SetPayer(payerAddress) - } - - for _, authorizer := range m.GetAuthorizers() { - authorizerAddress, err := Address(authorizer, chain) - if err != nil { - return *t, fmt.Errorf("could not convert authorizer address: %w", err) - } - t.AddAuthorizer(authorizerAddress) - } - - for _, sig := range m.GetPayloadSignatures() { - addr, err := Address(sig.GetAddress(), chain) - if err != nil { - return *t, fmt.Errorf("could not convert payload signature address: %w", err) - } - t.AddPayloadSignature(addr, uint64(sig.GetKeyId()), sig.GetSignature()) - } - - for _, sig := range m.GetEnvelopeSignatures() { - addr, err := Address(sig.GetAddress(), chain) - if err != nil { - return *t, fmt.Errorf("could not convert envelope signature address: %w", err) - } - t.AddEnvelopeSignature(addr, uint64(sig.GetKeyId()), sig.GetSignature()) - } - - t.SetScript(m.GetScript()) - t.SetArguments(m.GetArguments()) - t.SetReferenceBlockID(flow.HashToID(m.GetReferenceBlockId())) - t.SetGasLimit(m.GetGasLimit()) - - return *t, nil -} - -func MessageToTrieUpdate(m *entities.TrieUpdate) (*ledger.TrieUpdate, error) { - rootHash, err := ledger.ToRootHash(m.GetRootHash()) - if err != nil { - return nil, fmt.Errorf("could not convert root hash: %w", err) - } - - paths := make([]ledger.Path, len(m.GetPaths())) - for i, path := range m.GetPaths() { - convertedPath, err := ledger.ToPath(path) - if err != nil { - return nil, fmt.Errorf("could not convert path %d: %w", i, err) - } - paths[i] = convertedPath - } - - payloads := make([]*ledger.Payload, len(m.Payloads)) - for i, payload := range m.GetPayloads() { - keyParts := make([]ledger.KeyPart, len(payload.GetKeyPart())) - for j, keypart := range payload.GetKeyPart() { - keyParts[j] = ledger.NewKeyPart(uint16(keypart.GetType()), keypart.GetValue()) - } - payloads[i] = ledger.NewPayload(ledger.NewKey(keyParts), payload.GetValue()) - } - - return &ledger.TrieUpdate{ - RootHash: rootHash, - Paths: paths, - Payloads: payloads, - }, nil +// StateCommitmentToMessage converts a flow.StateCommitment to a byte slice for inclusion in a protobuf message +func StateCommitmentToMessage(s flow.StateCommitment) []byte { + return s[:] } -// insecureAddress converts a raw address to a flow.Address, skipping validation -// This is useful when converting transactions from trusted state like BlockExecutionData. -// This should only be used for trusted inputs -func insecureAddress(rawAddress []byte, chain flow.Chain) (flow.Address, error) { - if len(rawAddress) == 0 { - return flow.EmptyAddress, status.Error(codes.InvalidArgument, "address cannot be empty") +// MessageToStateCommitment converts a byte slice from a protobuf message to a flow.StateCommitment +func MessageToStateCommitment(bytes []byte) (sc flow.StateCommitment, err error) { + if len(bytes) != len(sc) { + return sc, fmt.Errorf("invalid state commitment length. got %d expected %d", len(bytes), len(sc)) } - - return flow.BytesToAddress(rawAddress), nil + copy(sc[:], bytes) + return } diff --git a/engine/common/rpc/convert/convert_test.go b/engine/common/rpc/convert/convert_test.go deleted file mode 100644 index a98f828d0f6..00000000000 --- a/engine/common/rpc/convert/convert_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package convert_test - -import ( - "bytes" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/engine/common/rpc/convert" - "github.com/onflow/flow-go/fvm" - "github.com/onflow/flow-go/ledger" - "github.com/onflow/flow-go/ledger/common/testutils" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/executiondatasync/execution_data" - "github.com/onflow/flow-go/utils/unittest" -) - -func TestConvertTransaction(t *testing.T) { - tx := unittest.TransactionBodyFixture() - - msg := convert.TransactionToMessage(tx) - converted, err := convert.MessageToTransaction(msg, flow.Testnet.Chain()) - assert.Nil(t, err) - - assert.Equal(t, tx, converted) - assert.Equal(t, tx.ID(), converted.ID()) -} - -func TestConvertAccountKey(t *testing.T) { - privateKey, _ := unittest.AccountKeyDefaultFixture() - accountKey := privateKey.PublicKey(fvm.AccountKeyWeightThreshold) - - // Explicitly test if Revoked is properly converted - accountKey.Revoked = true - - msg, err := convert.AccountKeyToMessage(accountKey) - assert.Nil(t, err) - - converted, err := convert.MessageToAccountKey(msg) - assert.Nil(t, err) - - assert.Equal(t, accountKey, *converted) - assert.Equal(t, accountKey.PublicKey, converted.PublicKey) - assert.Equal(t, accountKey.Revoked, converted.Revoked) -} - -func TestConvertEvents(t *testing.T) { - t.Run("empty", func(t *testing.T) { - messages := convert.EventsToMessages(nil) - assert.Len(t, messages, 0) - }) - - t.Run("simple", func(t *testing.T) { - - txID := unittest.IdentifierFixture() - event := unittest.EventFixture(flow.EventAccountCreated, 2, 3, txID, 0) - - messages := convert.EventsToMessages([]flow.Event{event}) - - require.Len(t, messages, 1) - - message := messages[0] - - require.Equal(t, event.EventIndex, message.EventIndex) - require.Equal(t, event.TransactionIndex, message.TransactionIndex) - require.Equal(t, event.Payload, message.Payload) - require.Equal(t, event.TransactionID[:], message.TransactionId) - require.Equal(t, string(event.Type), message.Type) - }) -} - -// TestConvertBlockExecutionData checks if conversions between BlockExecutionData and it's fields are consistent. -func TestConvertBlockExecutionData(t *testing.T) { - // Initialize the BlockExecutionData object - numChunks := 5 - ced := make([]*execution_data.ChunkExecutionData, numChunks) - bed := &execution_data.BlockExecutionData{ - BlockID: unittest.IdentifierFixture(), - ChunkExecutionDatas: ced, - } - - // Fill the chunk execution datas with trie updates, collections, and events - minSerializedSize := uint64(10 * execution_data.DefaultMaxBlobSize) - for i := 0; i < numChunks; i++ { - // the service chunk sometimes does not have any trie updates - if i == numChunks-1 { - tx1 := unittest.TransactionBodyFixture() - // proposal key and payer are empty addresses for service tx - tx1.ProposalKey.Address = flow.EmptyAddress - tx1.Payer = flow.EmptyAddress - bed.ChunkExecutionDatas[i] = &execution_data.ChunkExecutionData{ - Collection: &flow.Collection{Transactions: []*flow.TransactionBody{&tx1}}, - } - continue - } - - // Initialize collection - tx1 := unittest.TransactionBodyFixture() - tx2 := unittest.TransactionBodyFixture() - col := &flow.Collection{Transactions: []*flow.TransactionBody{&tx1, &tx2}} - - // Initialize events - header := unittest.BlockHeaderFixture() - events := unittest.BlockEventsFixture(header, 5).Events - - chunk := &execution_data.ChunkExecutionData{ - Collection: col, - Events: events, - TrieUpdate: testutils.TrieUpdateFixture(1, 1, 8), - } - size := 1 - - // Fill the TrieUpdate with data - inner: - for { - buf := &bytes.Buffer{} - require.NoError(t, execution_data.DefaultSerializer.Serialize(buf, chunk)) - - if buf.Len() >= int(minSerializedSize) { - break inner - } - - v := make([]byte, size) - _, _ = rand.Read(v) - - k, err := chunk.TrieUpdate.Payloads[0].Key() - require.NoError(t, err) - - chunk.TrieUpdate.Payloads[0] = ledger.NewPayload(k, v) - size *= 2 - } - bed.ChunkExecutionDatas[i] = chunk - } - - t.Run("chunk execution data conversions", func(t *testing.T) { - chunkMsg, err := convert.ChunkExecutionDataToMessage(bed.ChunkExecutionDatas[0]) - assert.Nil(t, err) - - chunkReConverted, err := convert.MessageToChunkExecutionData(chunkMsg, flow.Testnet.Chain()) - assert.Nil(t, err) - assert.Equal(t, bed.ChunkExecutionDatas[0], chunkReConverted) - }) - - t.Run("block execution data conversions", func(t *testing.T) { - blockMsg, err := convert.BlockExecutionDataToMessage(bed) - assert.Nil(t, err) - - bedReConverted, err := convert.MessageToBlockExecutionData(blockMsg, flow.Testnet.Chain()) - assert.Nil(t, err) - assert.Equal(t, bed, bedReConverted) - }) -} diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go new file mode 100644 index 00000000000..58ccb0ed9a1 --- /dev/null +++ b/engine/common/rpc/convert/events.go @@ -0,0 +1,226 @@ +package convert + +import ( + "encoding/json" + "fmt" + + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/onflow/cadence/encoding/ccf" + jsoncdc "github.com/onflow/cadence/encoding/json" + + "github.com/onflow/flow-go/model/flow" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/entities" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" +) + +// EventToMessage converts a flow.Event to a protobuf message +// Note: this function does not convert the payload encoding +func EventToMessage(e flow.Event) *entities.Event { + return &entities.Event{ + Type: string(e.Type), + TransactionId: e.TransactionID[:], + TransactionIndex: e.TransactionIndex, + EventIndex: e.EventIndex, + Payload: e.Payload, + } +} + +// MessageToEvent converts a protobuf message to a flow.Event +// Note: this function does not convert the payload encoding +func MessageToEvent(m *entities.Event) flow.Event { + return flow.Event{ + Type: flow.EventType(m.GetType()), + TransactionID: flow.HashToID(m.GetTransactionId()), + TransactionIndex: m.GetTransactionIndex(), + EventIndex: m.GetEventIndex(), + Payload: m.GetPayload(), + } +} + +// EventsToMessages converts a slice of flow.Events to a slice of protobuf messages +// Note: this function does not convert the payload encoding +func EventsToMessages(flowEvents []flow.Event) []*entities.Event { + events := make([]*entities.Event, len(flowEvents)) + for i, e := range flowEvents { + event := EventToMessage(e) + events[i] = event + } + return events +} + +// MessagesToEvents converts a slice of protobuf messages to a slice of flow.Events +// Note: this function does not convert the payload encoding +func MessagesToEvents(l []*entities.Event) []flow.Event { + events := make([]flow.Event, len(l)) + for i, m := range l { + events[i] = MessageToEvent(m) + } + return events +} + +// MessageToEventFromVersion converts a protobuf message to a flow.Event, and converts the payload +// encoding from CCF to JSON if the input version is CCF +func MessageToEventFromVersion(m *entities.Event, inputVersion execproto.EventEncodingVersion) (*flow.Event, error) { + event := MessageToEvent(m) + switch inputVersion { + case execproto.EventEncodingVersion_CCF_V0: + convertedPayload, err := CcfPayloadToJsonPayload(event.Payload) + if err != nil { + return nil, fmt.Errorf("could not convert event payload from CCF to Json: %w", err) + } + event.Payload = convertedPayload + return &event, nil + case execproto.EventEncodingVersion_JSON_CDC_V0: + return &event, nil + default: + return nil, fmt.Errorf("invalid encoding format %d", inputVersion) + } +} + +// MessagesToEventsFromVersion converts a slice of protobuf messages to a slice of flow.Events, converting +// the payload encoding from CCF to JSON if the input version is CCF +func MessagesToEventsFromVersion(l []*entities.Event, version execproto.EventEncodingVersion) ([]flow.Event, error) { + events := make([]flow.Event, len(l)) + for i, m := range l { + event, err := MessageToEventFromVersion(m, version) + if err != nil { + return nil, fmt.Errorf("could not convert event at index %d from format %d: %w", + m.EventIndex, version, err) + } + events[i] = *event + } + return events, nil +} + +// ServiceEventToMessage converts a flow.ServiceEvent to a protobuf message +func ServiceEventToMessage(event flow.ServiceEvent) (*entities.ServiceEvent, error) { + bytes, err := json.Marshal(event.Event) + if err != nil { + return nil, fmt.Errorf("cannot marshal service event: %w", err) + } + + return &entities.ServiceEvent{ + Type: event.Type.String(), + Payload: bytes, + }, nil +} + +// MessageToServiceEvent converts a protobuf message to a flow.ServiceEvent +func MessageToServiceEvent(m *entities.ServiceEvent) (*flow.ServiceEvent, error) { + rawEvent := m.Payload + eventType := flow.ServiceEventType(m.Type) + se, err := flow.ServiceEventJSONMarshaller.UnmarshalWithType(rawEvent, eventType) + + return &se, err +} + +// ServiceEventListToMessages converts a slice of flow.ServiceEvents to a slice of protobuf messages +func ServiceEventListToMessages(list flow.ServiceEventList) ( + []*entities.ServiceEvent, + error, +) { + entities := make([]*entities.ServiceEvent, len(list)) + for i, serviceEvent := range list { + m, err := ServiceEventToMessage(serviceEvent) + if err != nil { + return nil, fmt.Errorf("failed to convert service event at index %d to message: %w", i, err) + } + entities[i] = m + } + return entities, nil +} + +// ServiceEventsToMessages converts a slice of flow.ServiceEvents to a slice of protobuf messages +func MessagesToServiceEventList(m []*entities.ServiceEvent) ( + flow.ServiceEventList, + error, +) { + parsedServiceEvents := make(flow.ServiceEventList, len(m)) + for i, serviceEvent := range m { + parsedServiceEvent, err := MessageToServiceEvent(serviceEvent) + if err != nil { + return nil, fmt.Errorf("failed to parse service event at index %d from message: %w", i, err) + } + parsedServiceEvents[i] = *parsedServiceEvent + } + return parsedServiceEvents, nil +} + +// CcfPayloadToJsonPayload converts a CCF-encoded payload to a JSON-encoded payload +func CcfPayloadToJsonPayload(p []byte) ([]byte, error) { + val, err := ccf.Decode(nil, p) + if err != nil { + return nil, fmt.Errorf("unable to decode from ccf format: %w", err) + } + res, err := jsoncdc.Encode(val) + if err != nil { + return nil, fmt.Errorf("unable to encode to json-cdc format: %w", err) + } + return res, nil +} + +// CcfEventToJsonEvent returns a new event with the payload converted from CCF to JSON +func CcfEventToJsonEvent(e flow.Event) (*flow.Event, error) { + convertedPayload, err := CcfPayloadToJsonPayload(e.Payload) + if err != nil { + return nil, err + } + return &flow.Event{ + Type: e.Type, + TransactionID: e.TransactionID, + TransactionIndex: e.TransactionIndex, + EventIndex: e.EventIndex, + Payload: convertedPayload, + }, nil +} + +// MessagesToBlockEvents converts a protobuf EventsResponse_Result messages to a slice of flow.BlockEvents. +func MessagesToBlockEvents(blocksEvents []*accessproto.EventsResponse_Result) []flow.BlockEvents { + evs := make([]flow.BlockEvents, len(blocksEvents)) + for i, ev := range blocksEvents { + evs[i] = MessageToBlockEvents(ev) + } + + return evs +} + +// MessageToBlockEvents converts a protobuf EventsResponse_Result message to a flow.BlockEvents. +func MessageToBlockEvents(blockEvents *accessproto.EventsResponse_Result) flow.BlockEvents { + return flow.BlockEvents{ + BlockHeight: blockEvents.BlockHeight, + BlockID: MessageToIdentifier(blockEvents.BlockId), + BlockTimestamp: blockEvents.BlockTimestamp.AsTime(), + Events: MessagesToEvents(blockEvents.Events), + } +} + +func BlockEventsToMessages(blocks []flow.BlockEvents) ([]*accessproto.EventsResponse_Result, error) { + results := make([]*accessproto.EventsResponse_Result, len(blocks)) + + for i, block := range blocks { + event, err := BlockEventsToMessage(block) + if err != nil { + return nil, err + } + results[i] = event + } + + return results, nil +} + +func BlockEventsToMessage(block flow.BlockEvents) (*accessproto.EventsResponse_Result, error) { + eventMessages := make([]*entities.Event, len(block.Events)) + for i, event := range block.Events { + eventMessages[i] = EventToMessage(event) + } + timestamp := timestamppb.New(block.BlockTimestamp) + return &accessproto.EventsResponse_Result{ + BlockId: block.BlockID[:], + BlockHeight: block.BlockHeight, + BlockTimestamp: timestamp, + Events: eventMessages, + }, nil +} diff --git a/engine/common/rpc/convert/events_test.go b/engine/common/rpc/convert/events_test.go new file mode 100644 index 00000000000..879db710f8b --- /dev/null +++ b/engine/common/rpc/convert/events_test.go @@ -0,0 +1,216 @@ +package convert_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" + jsoncdc "github.com/onflow/cadence/encoding/json" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" + + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestConvertEventWithoutPayloadConversion tests converting events to and from protobuf messages +// with no payload modification +func TestConvertEventWithoutPayloadConversion(t *testing.T) { + t.Parallel() + + txID := unittest.IdentifierFixture() + cadenceValue, err := cadence.NewValue(2) + require.NoError(t, err) + + t.Run("convert empty event", func(t *testing.T) { + event := unittest.EventFixture(flow.EventAccountCreated, 2, 3, txID, 0) + + msg := convert.EventToMessage(event) + converted := convert.MessageToEvent(msg) + + assert.Equal(t, event, converted) + }) + + t.Run("convert json cdc encoded event", func(t *testing.T) { + ccfPayload, err := ccf.Encode(cadenceValue) + require.NoError(t, err) + + event := unittest.EventFixture(flow.EventAccountCreated, 2, 3, txID, 0) + event.Payload = ccfPayload + + msg := convert.EventToMessage(event) + converted := convert.MessageToEvent(msg) + + assert.Equal(t, event, converted) + }) + + t.Run("convert json cdc encoded event", func(t *testing.T) { + jsonPayload, err := jsoncdc.Encode(cadenceValue) + require.NoError(t, err) + + event := unittest.EventFixture(flow.EventAccountCreated, 2, 3, txID, 0) + event.Payload = jsonPayload + + msg := convert.EventToMessage(event) + converted := convert.MessageToEvent(msg) + + assert.Equal(t, event.Type, converted.Type) + }) +} + +// TestConvertEventWithPayloadConversion tests converting events to and from protobuf messages +// with payload modification +func TestConvertEventWithPayloadConversion(t *testing.T) { + t.Parallel() + + txID := unittest.IdentifierFixture() + cadenceValue, err := cadence.NewValue(2) + require.NoError(t, err) + + ccfEvent := unittest.EventFixture(flow.EventAccountCreated, 2, 3, txID, 0) + ccfEvent.Payload, err = ccf.Encode(cadenceValue) + require.NoError(t, err) + + jsonEvent := unittest.EventFixture(flow.EventAccountCreated, 2, 3, txID, 0) + jsonEvent.Payload, err = jsoncdc.Encode(cadenceValue) + require.NoError(t, err) + + t.Run("convert payload from ccf to jsoncdc", func(t *testing.T) { + message := convert.EventToMessage(ccfEvent) + convertedEvent, err := convert.MessageToEventFromVersion(message, execproto.EventEncodingVersion_CCF_V0) + assert.NoError(t, err) + + assert.Equal(t, jsonEvent, *convertedEvent) + }) + + t.Run("convert payload from jsoncdc to jsoncdc", func(t *testing.T) { + message := convert.EventToMessage(jsonEvent) + convertedEvent, err := convert.MessageToEventFromVersion(message, execproto.EventEncodingVersion_JSON_CDC_V0) + assert.NoError(t, err) + + assert.Equal(t, jsonEvent, *convertedEvent) + }) +} + +func TestConvertEvents(t *testing.T) { + t.Parallel() + + eventCount := 3 + txID := unittest.IdentifierFixture() + + events := make([]flow.Event, eventCount) + ccfEvents := make([]flow.Event, eventCount) + jsonEvents := make([]flow.Event, eventCount) + for i := 0; i < eventCount; i++ { + cadenceValue, err := cadence.NewValue(i) + require.NoError(t, err) + + eventIndex := 3 + uint32(i) + + event := unittest.EventFixture(flow.EventAccountCreated, 2, eventIndex, txID, 0) + ccfEvent := unittest.EventFixture(flow.EventAccountCreated, 2, eventIndex, txID, 0) + jsonEvent := unittest.EventFixture(flow.EventAccountCreated, 2, eventIndex, txID, 0) + + ccfEvent.Payload, err = ccf.Encode(cadenceValue) + require.NoError(t, err) + + jsonEvent.Payload, err = jsoncdc.Encode(cadenceValue) + require.NoError(t, err) + + events[i] = event + ccfEvents[i] = ccfEvent + jsonEvents[i] = jsonEvent + } + + t.Run("empty", func(t *testing.T) { + messages := convert.EventsToMessages(nil) + assert.Len(t, messages, 0) + }) + + t.Run("convert with passthrough payload conversion", func(t *testing.T) { + messages := convert.EventsToMessages(events) + require.Len(t, messages, len(events)) + + for i, message := range messages { + event := events[i] + require.Equal(t, event.EventIndex, message.EventIndex) + require.Equal(t, event.TransactionIndex, message.TransactionIndex) + require.Equal(t, event.Payload, message.Payload) + require.Equal(t, event.TransactionID[:], message.TransactionId) + require.Equal(t, string(event.Type), message.Type) + } + + converted := convert.MessagesToEvents(messages) + assert.Equal(t, events, converted) + }) + + t.Run("convert event from ccf to jsoncdc", func(t *testing.T) { + messages := convert.EventsToMessages(ccfEvents) + converted, err := convert.MessagesToEventsFromVersion(messages, execproto.EventEncodingVersion_CCF_V0) + assert.NoError(t, err) + + assert.Equal(t, jsonEvents, converted) + }) + + t.Run("convert event from jsoncdc", func(t *testing.T) { + messages := convert.EventsToMessages(jsonEvents) + converted, err := convert.MessagesToEventsFromVersion(messages, execproto.EventEncodingVersion_JSON_CDC_V0) + assert.NoError(t, err) + + assert.Equal(t, jsonEvents, converted) + }) +} + +func TestConvertServiceEvent(t *testing.T) { + t.Parallel() + + serviceEvents := unittest.ServiceEventsFixture(1) + require.Len(t, serviceEvents, 1) + + msg, err := convert.ServiceEventToMessage(serviceEvents[0]) + require.NoError(t, err) + + converted, err := convert.MessageToServiceEvent(msg) + require.NoError(t, err) + + assert.Equal(t, serviceEvents[0], *converted) +} + +func TestConvertServiceEventList(t *testing.T) { + t.Parallel() + + serviceEvents := unittest.ServiceEventsFixture(5) + require.Len(t, serviceEvents, 5) + + msg, err := convert.ServiceEventListToMessages(serviceEvents) + require.NoError(t, err) + + converted, err := convert.MessagesToServiceEventList(msg) + require.NoError(t, err) + + assert.Equal(t, serviceEvents, converted) +} + +// TestConvertMessagesToBlockEvents tests that converting a protobuf EventsResponse_Result message to and from block events in the same +// block +func TestConvertMessagesToBlockEvents(t *testing.T) { + t.Parallel() + + count := 2 + blockEvents := make([]flow.BlockEvents, count) + for i := 0; i < count; i++ { + header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(uint64(i))) + blockEvents[i] = unittest.BlockEventsFixture(header, 2) + } + + msg, err := convert.BlockEventsToMessages(blockEvents) + require.NoError(t, err) + + converted := convert.MessagesToBlockEvents(msg) + require.NoError(t, err) + + assert.Equal(t, blockEvents, converted) +} diff --git a/engine/common/rpc/convert/execution_data.go b/engine/common/rpc/convert/execution_data.go new file mode 100644 index 00000000000..1e9c6031b2c --- /dev/null +++ b/engine/common/rpc/convert/execution_data.go @@ -0,0 +1,281 @@ +package convert + +import ( + "fmt" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/onflow/flow/protobuf/go/flow/entities" + + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" +) + +// BlockExecutionDataToMessage converts a BlockExecutionData to a protobuf message +func BlockExecutionDataToMessage(data *execution_data.BlockExecutionData) ( + *entities.BlockExecutionData, + error, +) { + chunkExecutionDatas := make([]*entities.ChunkExecutionData, len(data.ChunkExecutionDatas)) + for i, chunk := range data.ChunkExecutionDatas { + chunkMessage, err := ChunkExecutionDataToMessage(chunk) + if err != nil { + return nil, err + } + chunkExecutionDatas[i] = chunkMessage + } + return &entities.BlockExecutionData{ + BlockId: IdentifierToMessage(data.BlockID), + ChunkExecutionData: chunkExecutionDatas, + }, nil +} + +// MessageToBlockExecutionData converts a protobuf message to a BlockExecutionData +func MessageToBlockExecutionData( + m *entities.BlockExecutionData, + chain flow.Chain, +) (*execution_data.BlockExecutionData, error) { + if m == nil { + return nil, ErrEmptyMessage + } + chunks := make([]*execution_data.ChunkExecutionData, len(m.ChunkExecutionData)) + for i, chunk := range m.GetChunkExecutionData() { + convertedChunk, err := MessageToChunkExecutionData(chunk, chain) + if err != nil { + return nil, err + } + chunks[i] = convertedChunk + } + + return &execution_data.BlockExecutionData{ + BlockID: MessageToIdentifier(m.GetBlockId()), + ChunkExecutionDatas: chunks, + }, nil +} + +// ChunkExecutionDataToMessage converts a ChunkExecutionData to a protobuf message +func ChunkExecutionDataToMessage(data *execution_data.ChunkExecutionData) ( + *entities.ChunkExecutionData, + error, +) { + collection := &entities.ExecutionDataCollection{} + if data.Collection != nil { + collection = &entities.ExecutionDataCollection{ + Transactions: TransactionsToMessages(data.Collection.Transactions), + } + } + + events := EventsToMessages(data.Events) + if len(events) == 0 { + events = nil + } + + trieUpdate, err := TrieUpdateToMessage(data.TrieUpdate) + if err != nil { + return nil, err + } + + return &entities.ChunkExecutionData{ + Collection: collection, + Events: events, + TrieUpdate: trieUpdate, + }, nil +} + +// MessageToChunkExecutionData converts a protobuf message to a ChunkExecutionData +func MessageToChunkExecutionData( + m *entities.ChunkExecutionData, + chain flow.Chain, +) (*execution_data.ChunkExecutionData, error) { + collection, err := messageToTrustedCollection(m.GetCollection(), chain) + if err != nil { + return nil, err + } + + var trieUpdate *ledger.TrieUpdate + if m.GetTrieUpdate() != nil { + trieUpdate, err = MessageToTrieUpdate(m.GetTrieUpdate()) + if err != nil { + return nil, err + } + } + + events := MessagesToEvents(m.GetEvents()) + if len(events) == 0 { + events = nil + } + + return &execution_data.ChunkExecutionData{ + Collection: collection, + Events: events, + TrieUpdate: trieUpdate, + }, nil +} + +// MessageToTrieUpdate converts a protobuf message to a TrieUpdate +func MessageToTrieUpdate(m *entities.TrieUpdate) (*ledger.TrieUpdate, error) { + rootHash, err := ledger.ToRootHash(m.GetRootHash()) + if err != nil { + return nil, fmt.Errorf("could not convert root hash: %w", err) + } + + paths := make([]ledger.Path, len(m.GetPaths())) + for i, path := range m.GetPaths() { + convertedPath, err := ledger.ToPath(path) + if err != nil { + return nil, fmt.Errorf("could not convert path %d: %w", i, err) + } + paths[i] = convertedPath + } + + payloads := make([]*ledger.Payload, len(m.Payloads)) + for i, payload := range m.GetPayloads() { + keyParts := make([]ledger.KeyPart, len(payload.GetKeyPart())) + for j, keypart := range payload.GetKeyPart() { + keyParts[j] = ledger.NewKeyPart(uint16(keypart.GetType()), keypart.GetValue()) + } + payloads[i] = ledger.NewPayload(ledger.NewKey(keyParts), payload.GetValue()) + } + + return &ledger.TrieUpdate{ + RootHash: rootHash, + Paths: paths, + Payloads: payloads, + }, nil +} + +// TrieUpdateToMessage converts a TrieUpdate to a protobuf message +func TrieUpdateToMessage(t *ledger.TrieUpdate) (*entities.TrieUpdate, error) { + if t == nil { + return nil, nil + } + + paths := make([][]byte, len(t.Paths)) + for i, path := range t.Paths { + paths[i] = path[:] + } + + payloads := make([]*entities.Payload, len(t.Payloads)) + for i, payload := range t.Payloads { + key, err := payload.Key() + if err != nil { + return nil, fmt.Errorf("could not convert payload %d: %w", i, err) + } + keyParts := make([]*entities.KeyPart, len(key.KeyParts)) + for j, keyPart := range key.KeyParts { + keyParts[j] = &entities.KeyPart{ + Type: uint32(keyPart.Type), + Value: keyPart.Value, + } + } + payloads[i] = &entities.Payload{ + KeyPart: keyParts, + Value: payload.Value(), + } + } + + return &entities.TrieUpdate{ + RootHash: t.RootHash[:], + Paths: paths, + Payloads: payloads, + }, nil +} + +// messageToTrustedCollection converts a protobuf message to a collection using the +// messageToTrustedTransaction converter to support service transactions. +func messageToTrustedCollection( + m *entities.ExecutionDataCollection, + chain flow.Chain, +) (*flow.Collection, error) { + messages := m.GetTransactions() + transactions := make([]*flow.TransactionBody, len(messages)) + for i, message := range messages { + transaction, err := messageToTrustedTransaction(message, chain) + if err != nil { + return nil, fmt.Errorf("could not convert transaction %d: %w", i, err) + } + transactions[i] = &transaction + } + + if len(transactions) == 0 { + return nil, nil + } + + return &flow.Collection{Transactions: transactions}, nil +} + +// messageToTrustedTransaction converts a transaction message to a transaction body. +// This is useful when converting transactions from trusted state like BlockExecutionData which +// contain service transactions that do not conform to external transaction format. +func messageToTrustedTransaction( + m *entities.Transaction, + chain flow.Chain, +) (flow.TransactionBody, error) { + if m == nil { + return flow.TransactionBody{}, ErrEmptyMessage + } + + t := flow.NewTransactionBody() + + proposalKey := m.GetProposalKey() + if proposalKey != nil { + proposalAddress, err := insecureAddress(proposalKey.GetAddress(), chain) + if err != nil { + return *t, fmt.Errorf("could not convert proposer address: %w", err) + } + t.SetProposalKey(proposalAddress, uint64(proposalKey.GetKeyId()), proposalKey.GetSequenceNumber()) + } + + payer := m.GetPayer() + if payer != nil { + payerAddress, err := insecureAddress(payer, chain) + if err != nil { + return *t, fmt.Errorf("could not convert payer address: %w", err) + } + t.SetPayer(payerAddress) + } + + for _, authorizer := range m.GetAuthorizers() { + authorizerAddress, err := Address(authorizer, chain) + if err != nil { + return *t, fmt.Errorf("could not convert authorizer address: %w", err) + } + t.AddAuthorizer(authorizerAddress) + } + + for _, sig := range m.GetPayloadSignatures() { + addr, err := Address(sig.GetAddress(), chain) + if err != nil { + return *t, fmt.Errorf("could not convert payload signature address: %w", err) + } + t.AddPayloadSignature(addr, uint64(sig.GetKeyId()), sig.GetSignature()) + } + + for _, sig := range m.GetEnvelopeSignatures() { + addr, err := Address(sig.GetAddress(), chain) + if err != nil { + return *t, fmt.Errorf("could not convert envelope signature address: %w", err) + } + t.AddEnvelopeSignature(addr, uint64(sig.GetKeyId()), sig.GetSignature()) + } + + t.SetScript(m.GetScript()) + t.SetArguments(m.GetArguments()) + t.SetReferenceBlockID(flow.HashToID(m.GetReferenceBlockId())) + t.SetGasLimit(m.GetGasLimit()) + + return *t, nil +} + +// insecureAddress converts a raw address to a flow.Address, skipping validation +// This is useful when converting transactions from trusted state like BlockExecutionData. +// This should only be used for trusted inputs +func insecureAddress(rawAddress []byte, chain flow.Chain) (flow.Address, error) { + if len(rawAddress) == 0 { + return flow.EmptyAddress, status.Error(codes.InvalidArgument, "address cannot be empty") + } + + return flow.BytesToAddress(rawAddress), nil +} diff --git a/engine/common/rpc/convert/execution_data_test.go b/engine/common/rpc/convert/execution_data_test.go new file mode 100644 index 00000000000..2ebfd915c28 --- /dev/null +++ b/engine/common/rpc/convert/execution_data_test.go @@ -0,0 +1,60 @@ +package convert_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestConvertBlockExecutionData1(t *testing.T) { + t.Parallel() + + chain := flow.Testnet.Chain() // this is used by the AddressFixture + events := unittest.EventsFixture(5) + + chunks := 5 + chunkData := make([]*execution_data.ChunkExecutionData, 0, chunks) + for i := 0; i < chunks-1; i++ { + chunkData = append(chunkData, unittest.ChunkExecutionDataFixture(t, execution_data.DefaultMaxBlobSize/5, unittest.WithChunkEvents(events))) + } + makeServiceTx := func(ced *execution_data.ChunkExecutionData) { + // proposal key and payer are empty addresses for service tx + collection := unittest.CollectionFixture(1) + collection.Transactions[0].ProposalKey.Address = flow.EmptyAddress + collection.Transactions[0].Payer = flow.EmptyAddress + ced.Collection = &collection + + // the service chunk sometimes does not have any trie updates + ced.TrieUpdate = nil + } + chunk := unittest.ChunkExecutionDataFixture(t, execution_data.DefaultMaxBlobSize/5, unittest.WithChunkEvents(events), makeServiceTx) + chunkData = append(chunkData, chunk) + + blockData := unittest.BlockExecutionDataFixture(unittest.WithChunkExecutionDatas(chunkData...)) + + t.Run("chunk execution data conversions", func(t *testing.T) { + chunkMsg, err := convert.ChunkExecutionDataToMessage(chunkData[0]) + require.NoError(t, err) + + chunkReConverted, err := convert.MessageToChunkExecutionData(chunkMsg, flow.Testnet.Chain()) + require.NoError(t, err) + + assert.Equal(t, chunkData[0], chunkReConverted) + }) + + t.Run("block execution data conversions", func(t *testing.T) { + msg, err := convert.BlockExecutionDataToMessage(blockData) + require.NoError(t, err) + + converted, err := convert.MessageToBlockExecutionData(msg, chain) + require.NoError(t, err) + + assert.Equal(t, blockData, converted) + }) +} diff --git a/engine/common/rpc/convert/execution_results.go b/engine/common/rpc/convert/execution_results.go new file mode 100644 index 00000000000..bbe7541edeb --- /dev/null +++ b/engine/common/rpc/convert/execution_results.go @@ -0,0 +1,174 @@ +package convert + +import ( + "fmt" + + "github.com/onflow/flow/protobuf/go/flow/entities" + + "github.com/onflow/flow-go/model/flow" +) + +// ExecutionResultToMessage converts an execution result to a protobuf message +func ExecutionResultToMessage(er *flow.ExecutionResult) ( + *entities.ExecutionResult, + error, +) { + chunks := make([]*entities.Chunk, len(er.Chunks)) + + for i, chunk := range er.Chunks { + chunks[i] = ChunkToMessage(chunk) + } + + serviceEvents := make([]*entities.ServiceEvent, len(er.ServiceEvents)) + var err error + for i, serviceEvent := range er.ServiceEvents { + serviceEvents[i], err = ServiceEventToMessage(serviceEvent) + if err != nil { + return nil, fmt.Errorf("error while convering service event %d: %w", i, err) + } + } + + return &entities.ExecutionResult{ + PreviousResultId: IdentifierToMessage(er.PreviousResultID), + BlockId: IdentifierToMessage(er.BlockID), + Chunks: chunks, + ServiceEvents: serviceEvents, + ExecutionDataId: IdentifierToMessage(er.ExecutionDataID), + }, nil +} + +// MessageToExecutionResult converts a protobuf message to an execution result +func MessageToExecutionResult(m *entities.ExecutionResult) ( + *flow.ExecutionResult, + error, +) { + // convert Chunks + parsedChunks, err := MessagesToChunkList(m.Chunks) + if err != nil { + return nil, fmt.Errorf("failed to parse messages to ChunkList: %w", err) + } + // convert ServiceEvents + parsedServiceEvents, err := MessagesToServiceEventList(m.ServiceEvents) + if err != nil { + return nil, err + } + return &flow.ExecutionResult{ + PreviousResultID: MessageToIdentifier(m.PreviousResultId), + BlockID: MessageToIdentifier(m.BlockId), + Chunks: parsedChunks, + ServiceEvents: parsedServiceEvents, + ExecutionDataID: MessageToIdentifier(m.ExecutionDataId), + }, nil +} + +// ExecutionResultsToMessages converts a slice of execution results to a slice of protobuf messages +func ExecutionResultsToMessages(e []*flow.ExecutionResult) ( + []*entities.ExecutionResult, + error, +) { + execResults := make([]*entities.ExecutionResult, len(e)) + for i, execRes := range e { + parsedExecResult, err := ExecutionResultToMessage(execRes) + if err != nil { + return nil, err + } + execResults[i] = parsedExecResult + } + return execResults, nil +} + +// MessagesToExecutionResults converts a slice of protobuf messages to a slice of execution results +func MessagesToExecutionResults(m []*entities.ExecutionResult) ( + []*flow.ExecutionResult, + error, +) { + execResults := make([]*flow.ExecutionResult, len(m)) + for i, e := range m { + parsedExecResult, err := MessageToExecutionResult(e) + if err != nil { + return nil, fmt.Errorf("failed to convert message at index %d to execution result: %w", i, err) + } + execResults[i] = parsedExecResult + } + return execResults, nil +} + +// ExecutionResultMetaListToMessages converts an execution result meta list to a slice of protobuf messages +func ExecutionResultMetaListToMessages(e flow.ExecutionReceiptMetaList) []*entities.ExecutionReceiptMeta { + messageList := make([]*entities.ExecutionReceiptMeta, len(e)) + for i, execMeta := range e { + messageList[i] = &entities.ExecutionReceiptMeta{ + ExecutorId: IdentifierToMessage(execMeta.ExecutorID), + ResultId: IdentifierToMessage(execMeta.ResultID), + Spocks: SignaturesToMessages(execMeta.Spocks), + ExecutorSignature: MessageToSignature(execMeta.ExecutorSignature), + } + } + return messageList +} + +// MessagesToExecutionResultMetaList converts a slice of protobuf messages to an execution result meta list +func MessagesToExecutionResultMetaList(m []*entities.ExecutionReceiptMeta) flow.ExecutionReceiptMetaList { + execMetaList := make([]*flow.ExecutionReceiptMeta, len(m)) + for i, message := range m { + execMetaList[i] = &flow.ExecutionReceiptMeta{ + ExecutorID: MessageToIdentifier(message.ExecutorId), + ResultID: MessageToIdentifier(message.ResultId), + Spocks: MessagesToSignatures(message.Spocks), + ExecutorSignature: MessageToSignature(message.ExecutorSignature), + } + } + return execMetaList[:] +} + +// ChunkToMessage converts a chunk to a protobuf message +func ChunkToMessage(chunk *flow.Chunk) *entities.Chunk { + return &entities.Chunk{ + CollectionIndex: uint32(chunk.CollectionIndex), + StartState: StateCommitmentToMessage(chunk.StartState), + EventCollection: IdentifierToMessage(chunk.EventCollection), + BlockId: IdentifierToMessage(chunk.BlockID), + TotalComputationUsed: chunk.TotalComputationUsed, + NumberOfTransactions: uint32(chunk.NumberOfTransactions), + Index: chunk.Index, + EndState: StateCommitmentToMessage(chunk.EndState), + } +} + +// MessageToChunk converts a protobuf message to a chunk +func MessageToChunk(m *entities.Chunk) (*flow.Chunk, error) { + startState, err := flow.ToStateCommitment(m.StartState) + if err != nil { + return nil, fmt.Errorf("failed to parse Message start state to Chunk: %w", err) + } + endState, err := flow.ToStateCommitment(m.EndState) + if err != nil { + return nil, fmt.Errorf("failed to parse Message end state to Chunk: %w", err) + } + chunkBody := flow.ChunkBody{ + CollectionIndex: uint(m.CollectionIndex), + StartState: startState, + EventCollection: MessageToIdentifier(m.EventCollection), + BlockID: MessageToIdentifier(m.BlockId), + TotalComputationUsed: m.TotalComputationUsed, + NumberOfTransactions: uint64(m.NumberOfTransactions), + } + return &flow.Chunk{ + ChunkBody: chunkBody, + Index: m.Index, + EndState: endState, + }, nil +} + +// MessagesToChunkList converts a slice of protobuf messages to a chunk list +func MessagesToChunkList(m []*entities.Chunk) (flow.ChunkList, error) { + parsedChunks := make(flow.ChunkList, len(m)) + for i, chunk := range m { + parsedChunk, err := MessageToChunk(chunk) + if err != nil { + return nil, fmt.Errorf("failed to parse message at index %d to chunk: %w", i, err) + } + parsedChunks[i] = parsedChunk + } + return parsedChunks, nil +} diff --git a/engine/common/rpc/convert/execution_results_test.go b/engine/common/rpc/convert/execution_results_test.go new file mode 100644 index 00000000000..cce0bd175e6 --- /dev/null +++ b/engine/common/rpc/convert/execution_results_test.go @@ -0,0 +1,57 @@ +package convert_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestConvertExecutionResult(t *testing.T) { + t.Parallel() + + er := unittest.ExecutionResultFixture(unittest.WithServiceEvents(3)) + + msg, err := convert.ExecutionResultToMessage(er) + require.NoError(t, err) + + converted, err := convert.MessageToExecutionResult(msg) + require.NoError(t, err) + + assert.Equal(t, er, converted) +} + +func TestConvertExecutionResults(t *testing.T) { + t.Parallel() + + results := []*flow.ExecutionResult{ + unittest.ExecutionResultFixture(unittest.WithServiceEvents(3)), + unittest.ExecutionResultFixture(unittest.WithServiceEvents(3)), + unittest.ExecutionResultFixture(unittest.WithServiceEvents(3)), + } + + msg, err := convert.ExecutionResultsToMessages(results) + require.NoError(t, err) + + converted, err := convert.MessagesToExecutionResults(msg) + require.NoError(t, err) + + assert.Equal(t, results, converted) +} + +func TestConvertExecutionResultMetaList(t *testing.T) { + t.Parallel() + + block := unittest.FullBlockFixture() + block.SetPayload(unittest.PayloadFixture(unittest.WithAllTheFixins)) + metaList := block.Payload.Receipts + + msg := convert.ExecutionResultMetaListToMessages(metaList) + converted := convert.MessagesToExecutionResultMetaList(msg) + + assert.Equal(t, metaList, converted) +} diff --git a/engine/common/rpc/convert/headers.go b/engine/common/rpc/convert/headers.go new file mode 100644 index 00000000000..b45686c853d --- /dev/null +++ b/engine/common/rpc/convert/headers.go @@ -0,0 +1,97 @@ +package convert + +import ( + "fmt" + + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/onflow/flow/protobuf/go/flow/entities" + + "github.com/onflow/flow-go/model/flow" +) + +// BlockHeaderToMessage converts a flow.Header to a protobuf message +func BlockHeaderToMessage( + h *flow.Header, + signerIDs flow.IdentifierList, +) (*entities.BlockHeader, error) { + id := h.ID() + + t := timestamppb.New(h.Timestamp) + var lastViewTC *entities.TimeoutCertificate + if h.LastViewTC != nil { + newestQC := h.LastViewTC.NewestQC + lastViewTC = &entities.TimeoutCertificate{ + View: h.LastViewTC.View, + HighQcViews: h.LastViewTC.NewestQCViews, + SignerIndices: h.LastViewTC.SignerIndices, + SigData: h.LastViewTC.SigData, + HighestQc: &entities.QuorumCertificate{ + View: newestQC.View, + BlockId: newestQC.BlockID[:], + SignerIndices: newestQC.SignerIndices, + SigData: newestQC.SigData, + }, + } + } + parentVoterIds := IdentifiersToMessages(signerIDs) + + return &entities.BlockHeader{ + Id: id[:], + ParentId: h.ParentID[:], + Height: h.Height, + PayloadHash: h.PayloadHash[:], + Timestamp: t, + View: h.View, + ParentView: h.ParentView, + ParentVoterIndices: h.ParentVoterIndices, + ParentVoterIds: parentVoterIds, + ParentVoterSigData: h.ParentVoterSigData, + ProposerId: h.ProposerID[:], + ProposerSigData: h.ProposerSigData, + ChainId: h.ChainID.String(), + LastViewTc: lastViewTC, + }, nil +} + +// MessageToBlockHeader converts a protobuf message to a flow.Header +func MessageToBlockHeader(m *entities.BlockHeader) (*flow.Header, error) { + chainId, err := MessageToChainId(m.ChainId) + if err != nil { + return nil, fmt.Errorf("failed to convert ChainId: %w", err) + } + var lastViewTC *flow.TimeoutCertificate + if m.LastViewTc != nil { + newestQC := m.LastViewTc.HighestQc + if newestQC == nil { + return nil, fmt.Errorf("invalid structure newest QC should be present") + } + lastViewTC = &flow.TimeoutCertificate{ + View: m.LastViewTc.View, + NewestQCViews: m.LastViewTc.HighQcViews, + SignerIndices: m.LastViewTc.SignerIndices, + SigData: m.LastViewTc.SigData, + NewestQC: &flow.QuorumCertificate{ + View: newestQC.View, + BlockID: MessageToIdentifier(newestQC.BlockId), + SignerIndices: newestQC.SignerIndices, + SigData: newestQC.SigData, + }, + } + } + + return &flow.Header{ + ParentID: MessageToIdentifier(m.ParentId), + Height: m.Height, + PayloadHash: MessageToIdentifier(m.PayloadHash), + Timestamp: m.Timestamp.AsTime(), + View: m.View, + ParentView: m.ParentView, + ParentVoterIndices: m.ParentVoterIndices, + ParentVoterSigData: m.ParentVoterSigData, + ProposerID: MessageToIdentifier(m.ProposerId), + ProposerSigData: m.ProposerSigData, + ChainID: *chainId, + LastViewTC: lastViewTC, + }, nil +} diff --git a/engine/common/rpc/convert/headers_test.go b/engine/common/rpc/convert/headers_test.go new file mode 100644 index 00000000000..d74b930b307 --- /dev/null +++ b/engine/common/rpc/convert/headers_test.go @@ -0,0 +1,29 @@ +package convert_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestConvertBlockHeader tests that converting a header to and from a protobuf message results in the same +// header +func TestConvertBlockHeader(t *testing.T) { + t.Parallel() + + header := unittest.BlockHeaderFixture() + + signerIDs := unittest.IdentifierListFixture(5) + + msg, err := convert.BlockHeaderToMessage(header, signerIDs) + require.NoError(t, err) + + converted, err := convert.MessageToBlockHeader(msg) + require.NoError(t, err) + + assert.Equal(t, header, converted) +} diff --git a/engine/common/rpc/convert/snapshots.go b/engine/common/rpc/convert/snapshots.go new file mode 100644 index 00000000000..963f95dbd09 --- /dev/null +++ b/engine/common/rpc/convert/snapshots.go @@ -0,0 +1,35 @@ +package convert + +import ( + "encoding/json" + "fmt" + + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/inmem" +) + +// SnapshotToBytes converts a `protocol.Snapshot` to bytes, encoded as JSON +func SnapshotToBytes(snapshot protocol.Snapshot) ([]byte, error) { + serializable, err := inmem.FromSnapshot(snapshot) + if err != nil { + return nil, err + } + + data, err := json.Marshal(serializable.Encodable()) + if err != nil { + return nil, err + } + + return data, nil +} + +// BytesToInmemSnapshot converts an array of bytes to `inmem.Snapshot` +func BytesToInmemSnapshot(bytes []byte) (*inmem.Snapshot, error) { + var encodable inmem.EncodableSnapshot + err := json.Unmarshal(bytes, &encodable) + if err != nil { + return nil, fmt.Errorf("could not unmarshal decoded snapshot: %w", err) + } + + return inmem.SnapshotFromEncodable(encodable), nil +} diff --git a/engine/common/rpc/convert/snapshots_test.go b/engine/common/rpc/convert/snapshots_test.go new file mode 100644 index 00000000000..2e1d4ce91e1 --- /dev/null +++ b/engine/common/rpc/convert/snapshots_test.go @@ -0,0 +1,27 @@ +package convert_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestConvertSnapshot(t *testing.T) { + t.Parallel() + + identities := unittest.CompleteIdentitySet() + snapshot := unittest.RootSnapshotFixtureWithChainID(identities, flow.Testnet.Chain().ChainID()) + + msg, err := convert.SnapshotToBytes(snapshot) + require.NoError(t, err) + + converted, err := convert.BytesToInmemSnapshot(msg) + require.NoError(t, err) + + assert.Equal(t, snapshot, converted) +} diff --git a/engine/common/rpc/convert/transactions.go b/engine/common/rpc/convert/transactions.go new file mode 100644 index 00000000000..ce94b5bae1c --- /dev/null +++ b/engine/common/rpc/convert/transactions.go @@ -0,0 +1,123 @@ +package convert + +import ( + "github.com/onflow/flow/protobuf/go/flow/entities" + + "github.com/onflow/flow-go/model/flow" +) + +// TransactionToMessage converts a flow.TransactionBody to a protobuf message +func TransactionToMessage(tb flow.TransactionBody) *entities.Transaction { + proposalKeyMessage := &entities.Transaction_ProposalKey{ + Address: tb.ProposalKey.Address.Bytes(), + KeyId: uint32(tb.ProposalKey.KeyIndex), + SequenceNumber: tb.ProposalKey.SequenceNumber, + } + + authMessages := make([][]byte, len(tb.Authorizers)) + for i, auth := range tb.Authorizers { + authMessages[i] = auth.Bytes() + } + + payloadSigMessages := make([]*entities.Transaction_Signature, len(tb.PayloadSignatures)) + + for i, sig := range tb.PayloadSignatures { + payloadSigMessages[i] = &entities.Transaction_Signature{ + Address: sig.Address.Bytes(), + KeyId: uint32(sig.KeyIndex), + Signature: sig.Signature, + } + } + + envelopeSigMessages := make([]*entities.Transaction_Signature, len(tb.EnvelopeSignatures)) + + for i, sig := range tb.EnvelopeSignatures { + envelopeSigMessages[i] = &entities.Transaction_Signature{ + Address: sig.Address.Bytes(), + KeyId: uint32(sig.KeyIndex), + Signature: sig.Signature, + } + } + + return &entities.Transaction{ + Script: tb.Script, + Arguments: tb.Arguments, + ReferenceBlockId: tb.ReferenceBlockID[:], + GasLimit: tb.GasLimit, + ProposalKey: proposalKeyMessage, + Payer: tb.Payer.Bytes(), + Authorizers: authMessages, + PayloadSignatures: payloadSigMessages, + EnvelopeSignatures: envelopeSigMessages, + } +} + +// MessageToTransaction converts a protobuf message to a flow.TransactionBody +func MessageToTransaction( + m *entities.Transaction, + chain flow.Chain, +) (flow.TransactionBody, error) { + if m == nil { + return flow.TransactionBody{}, ErrEmptyMessage + } + + t := flow.NewTransactionBody() + + proposalKey := m.GetProposalKey() + if proposalKey != nil { + proposalAddress, err := Address(proposalKey.GetAddress(), chain) + if err != nil { + return *t, err + } + t.SetProposalKey(proposalAddress, uint64(proposalKey.GetKeyId()), proposalKey.GetSequenceNumber()) + } + + payer := m.GetPayer() + if payer != nil { + payerAddress, err := Address(payer, chain) + if err != nil { + return *t, err + } + t.SetPayer(payerAddress) + } + + for _, authorizer := range m.GetAuthorizers() { + authorizerAddress, err := Address(authorizer, chain) + if err != nil { + return *t, err + } + t.AddAuthorizer(authorizerAddress) + } + + for _, sig := range m.GetPayloadSignatures() { + addr, err := Address(sig.GetAddress(), chain) + if err != nil { + return *t, err + } + t.AddPayloadSignature(addr, uint64(sig.GetKeyId()), sig.GetSignature()) + } + + for _, sig := range m.GetEnvelopeSignatures() { + addr, err := Address(sig.GetAddress(), chain) + if err != nil { + return *t, err + } + t.AddEnvelopeSignature(addr, uint64(sig.GetKeyId()), sig.GetSignature()) + } + + t.SetScript(m.GetScript()) + t.SetArguments(m.GetArguments()) + t.SetReferenceBlockID(flow.HashToID(m.GetReferenceBlockId())) + t.SetGasLimit(m.GetGasLimit()) + + return *t, nil +} + +// TransactionsToMessages converts a slice of flow.TransactionBody to a slice of protobuf messages +func TransactionsToMessages(transactions []*flow.TransactionBody) []*entities.Transaction { + transactionMessages := make([]*entities.Transaction, len(transactions)) + for i, t := range transactions { + transactionMessages[i] = TransactionToMessage(*t) + } + return transactionMessages +} diff --git a/engine/common/rpc/convert/transactions_test.go b/engine/common/rpc/convert/transactions_test.go new file mode 100644 index 00000000000..c9c5141f9a8 --- /dev/null +++ b/engine/common/rpc/convert/transactions_test.go @@ -0,0 +1,34 @@ +package convert_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/cadence" + jsoncdc "github.com/onflow/cadence/encoding/json" + + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestConvertTransaction(t *testing.T) { + t.Parallel() + + tx := unittest.TransactionBodyFixture() + arg, err := jsoncdc.Encode(cadence.NewAddress(unittest.AddressFixture())) + require.NoError(t, err) + + // add fields not included in the fixture + tx.Arguments = append(tx.Arguments, arg) + tx.EnvelopeSignatures = append(tx.EnvelopeSignatures, unittest.TransactionSignatureFixture()) + + msg := convert.TransactionToMessage(tx) + converted, err := convert.MessageToTransaction(msg, flow.Testnet.Chain()) + require.NoError(t, err) + + assert.Equal(t, tx, converted) + assert.Equal(t, tx.ID(), converted.ID()) +} diff --git a/engine/common/rpc/errors.go b/engine/common/rpc/errors.go index 5bd0b88471c..96201b04a78 100644 --- a/engine/common/rpc/errors.go +++ b/engine/common/rpc/errors.go @@ -20,16 +20,16 @@ func ConvertError(err error, msg string, defaultCode codes.Code) error { return nil } - // Already converted - if status.Code(err) != codes.Unknown { - return err - } - // Handle multierrors separately if multiErr, ok := err.(*multierror.Error); ok { return ConvertMultiError(multiErr, msg, defaultCode) } + // Already converted + if status.Code(err) != codes.Unknown { + return err + } + if msg != "" { msg += ": " } diff --git a/engine/common/splitter/network/example_test.go b/engine/common/splitter/network/example_test.go index b94f9e8a70e..93ef74b566b 100644 --- a/engine/common/splitter/network/example_test.go +++ b/engine/common/splitter/network/example_test.go @@ -2,7 +2,6 @@ package network_test import ( "fmt" - "math/rand" "github.com/rs/zerolog" @@ -20,10 +19,11 @@ func Example() { logger := zerolog.Nop() splitterNet := splitterNetwork.NewNetwork(net, logger) - // generate a random origin ID - var id flow.Identifier - rand.Seed(0) - rand.Read(id[:]) + // generate an origin ID + id, err := flow.HexStringToIdentifier("0194fdc2fa2ffcc041d3ff12045b73c86e4ff95ff662a5eee82abdf44a2d0b75") + if err != nil { + fmt.Println(err) + } // create engines engineProcessFunc := func(engineID int) testnet.EngineProcessFunc { @@ -38,7 +38,7 @@ func Example() { // register engines with splitter network channel := channels.Channel("foo-channel") - _, err := splitterNet.Register(channel, engine1) + _, err = splitterNet.Register(channel, engine1) if err != nil { fmt.Println(err) } diff --git a/engine/common/synchronization/engine.go b/engine/common/synchronization/engine.go index 7fab624d5a4..ec3f2e941dd 100644 --- a/engine/common/synchronization/engine.go +++ b/engine/common/synchronization/engine.go @@ -3,13 +3,14 @@ package synchronization import ( + "context" "fmt" - "math/rand" "time" "github.com/hashicorp/go-multierror" "github.com/rs/zerolog" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/common/fifoqueue" "github.com/onflow/flow-go/engine/consensus" @@ -18,11 +19,15 @@ import ( "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" synccore "github.com/onflow/flow-go/module/chainsync" - "github.com/onflow/flow-go/module/lifecycle" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/events" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/rand" ) // defaultSyncResponseQueueCapacity maximum capacity of sync responses queue @@ -33,20 +38,21 @@ const defaultBlockResponseQueueCapacity = 500 // Engine is the synchronization engine, responsible for synchronizing chain state. type Engine struct { - unit *engine.Unit - lm *lifecycle.LifecycleManager - log zerolog.Logger - metrics module.EngineMetrics - me module.Local - con network.Conduit - blocks storage.Blocks - comp consensus.Compliance + component.Component + hotstuff.FinalizationConsumer + + log zerolog.Logger + metrics module.EngineMetrics + me module.Local + finalizedHeaderCache module.FinalizedHeaderCache + con network.Conduit + blocks storage.Blocks + comp consensus.Compliance pollInterval time.Duration scanInterval time.Duration core module.SyncCore participantsProvider module.IdentifierProvider - finalizedHeader *FinalizedHeaderCache requestHandler *RequestHandler // component responsible for handling requests @@ -55,16 +61,19 @@ type Engine struct { responseMessageHandler *engine.MessageHandler // message handler responsible for response processing } +var _ network.MessageProcessor = (*Engine)(nil) +var _ component.Component = (*Engine)(nil) + // New creates a new main chain synchronization engine. func New( log zerolog.Logger, metrics module.EngineMetrics, net network.Network, me module.Local, + state protocol.State, blocks storage.Blocks, comp consensus.Compliance, core module.SyncCore, - finalizedHeader *FinalizedHeaderCache, participantsProvider module.IdentifierProvider, opts ...OptionFunc, ) (*Engine, error) { @@ -78,35 +87,48 @@ func New( panic("must initialize synchronization engine with comp engine") } + finalizedHeaderCache, finalizedCacheWorker, err := events.NewFinalizedHeaderCache(state) + if err != nil { + return nil, fmt.Errorf("could not create finalized header cache: %w", err) + } + // initialize the propagation engine with its dependencies e := &Engine{ - unit: engine.NewUnit(), - lm: lifecycle.NewLifecycleManager(), + FinalizationConsumer: finalizedHeaderCache, log: log.With().Str("engine", "synchronization").Logger(), metrics: metrics, me: me, + finalizedHeaderCache: finalizedHeaderCache, blocks: blocks, comp: comp, core: core, pollInterval: opt.PollInterval, scanInterval: opt.ScanInterval, - finalizedHeader: finalizedHeader, participantsProvider: participantsProvider, } - err := e.setupResponseMessageHandler() - if err != nil { - return nil, fmt.Errorf("could not setup message handler") - } - // register the engine with the network layer and store the conduit con, err := net.Register(channels.SyncCommittee, e) if err != nil { return nil, fmt.Errorf("could not register engine: %w", err) } e.con = con + e.requestHandler = NewRequestHandler(log, metrics, NewResponseSender(con), me, finalizedHeaderCache, blocks, core, true) + + // set up worker routines + builder := component.NewComponentManagerBuilder(). + AddWorker(finalizedCacheWorker). + AddWorker(e.checkLoop). + AddWorker(e.responseProcessingLoop) + for i := 0; i < defaultEngineRequestsWorkers; i++ { + builder.AddWorker(e.requestHandler.requestProcessingWorker) + } + e.Component = builder.Build() - e.requestHandler = NewRequestHandler(log, metrics, NewResponseSender(con), me, blocks, core, finalizedHeader, true) + err = e.setupResponseMessageHandler() + if err != nil { + return nil, fmt.Errorf("could not setup message handler") + } return e, nil } @@ -160,60 +182,10 @@ func (e *Engine) setupResponseMessageHandler() error { return nil } -// Ready returns a ready channel that is closed once the engine has fully started. -func (e *Engine) Ready() <-chan struct{} { - e.lm.OnStart(func() { - <-e.finalizedHeader.Ready() - e.unit.Launch(e.checkLoop) - e.unit.Launch(e.responseProcessingLoop) - // wait for request handler to startup - <-e.requestHandler.Ready() - }) - return e.lm.Started() -} - -// Done returns a done channel that is closed once the engine has fully stopped. -func (e *Engine) Done() <-chan struct{} { - e.lm.OnStop(func() { - // signal the request handler to shutdown - requestHandlerDone := e.requestHandler.Done() - // wait for request sending and response processing routines to exit - <-e.unit.Done() - // wait for request handler shutdown to complete - <-requestHandlerDone - <-e.finalizedHeader.Done() - }) - return e.lm.Stopped() -} - -// SubmitLocal submits an event originating on the local node. -func (e *Engine) SubmitLocal(event interface{}) { - err := e.process(e.me.NodeID(), event) - if err != nil { - // receiving an input of incompatible type from a trusted internal component is fatal - e.log.Fatal().Err(err).Msg("internal error processing event") - } -} - -// Submit submits the given event from the node with the given origin ID -// for processing in a non-blocking manner. It returns instantly and logs -// a potential processing error internally when done. -func (e *Engine) Submit(channel channels.Channel, originID flow.Identifier, event interface{}) { - err := e.Process(channel, originID, event) - if err != nil { - e.log.Fatal().Err(err).Msg("internal error processing event") - } -} - -// ProcessLocal processes an event originating on the local node. -func (e *Engine) ProcessLocal(event interface{}) error { - return e.process(e.me.NodeID(), event) -} - // Process processes the given event from the node with the given origin ID in // a blocking manner. It returns the potential processing error when done. func (e *Engine) Process(channel channels.Channel, originID flow.Identifier, event interface{}) error { - err := e.process(originID, event) + err := e.process(channel, originID, event) if err != nil { if engine.IsIncompatibleInputTypeError(err) { e.log.Warn().Msgf("%v delivered unsupported message %T through %v", originID, event, channel) @@ -228,10 +200,10 @@ func (e *Engine) Process(channel channels.Channel, originID flow.Identifier, eve // Error returns: // - IncompatibleInputTypeError if input has unexpected type // - All other errors are potential symptoms of internal state corruption or bugs (fatal). -func (e *Engine) process(originID flow.Identifier, event interface{}) error { +func (e *Engine) process(channel channels.Channel, originID flow.Identifier, event interface{}) error { switch event.(type) { case *messages.RangeRequest, *messages.BatchRequest, *messages.SyncRequest: - return e.requestHandler.process(originID, event) + return e.requestHandler.Process(channel, originID, event) case *messages.SyncResponse, *messages.BlockResponse: return e.responseMessageHandler.Process(originID, event) default: @@ -240,23 +212,26 @@ func (e *Engine) process(originID flow.Identifier, event interface{}) error { } // responseProcessingLoop is a separate goroutine that performs processing of queued responses -func (e *Engine) responseProcessingLoop() { +func (e *Engine) responseProcessingLoop(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + notifier := e.responseMessageHandler.GetNotifier() + done := ctx.Done() for { select { - case <-e.unit.Quit(): + case <-done: return case <-notifier: - e.processAvailableResponses() + e.processAvailableResponses(ctx) } } } // processAvailableResponses is processor of pending events which drives events from networking layer to business logic. -func (e *Engine) processAvailableResponses() { +func (e *Engine) processAvailableResponses(ctx context.Context) { for { select { - case <-e.unit.Quit(): + case <-ctx.Done(): return default: } @@ -284,7 +259,7 @@ func (e *Engine) processAvailableResponses() { // onSyncResponse processes a synchronization response. func (e *Engine) onSyncResponse(originID flow.Identifier, res *messages.SyncResponse) { e.log.Debug().Str("origin_id", originID.String()).Msg("received sync response") - final := e.finalizedHeader.Get() + final := e.finalizedHeaderCache.Get() e.core.HandleHeight(final, res.Height) } @@ -318,7 +293,9 @@ func (e *Engine) onBlockResponse(originID flow.Identifier, res *messages.BlockRe } // checkLoop will regularly scan for items that need requesting. -func (e *Engine) checkLoop() { +func (e *Engine) checkLoop(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + pollChan := make(<-chan time.Time) if e.pollInterval > 0 { poll := time.NewTicker(e.pollInterval) @@ -326,48 +303,56 @@ func (e *Engine) checkLoop() { defer poll.Stop() } scan := time.NewTicker(e.scanInterval) + defer scan.Stop() -CheckLoop: + done := ctx.Done() for { // give the quit channel a priority to be selected select { - case <-e.unit.Quit(): - break CheckLoop + case <-done: + return default: } select { - case <-e.unit.Quit(): - break CheckLoop + case <-done: + return case <-pollChan: e.pollHeight() case <-scan.C: - head := e.finalizedHeader.Get() + final := e.finalizedHeaderCache.Get() participants := e.participantsProvider.Identifiers() - ranges, batches := e.core.ScanPending(head) + ranges, batches := e.core.ScanPending(final) e.sendRequests(participants, ranges, batches) } } - - // some minor cleanup - scan.Stop() } // pollHeight will send a synchronization request to three random nodes. func (e *Engine) pollHeight() { - head := e.finalizedHeader.Get() + final := e.finalizedHeaderCache.Get() participants := e.participantsProvider.Identifiers() + nonce, err := rand.Uint64() + if err != nil { + // TODO: this error should be returned by pollHeight() + // it is logged for now since the only error possible is related to a failure + // of the system entropy generation. Such error is going to cause failures in other + // components where it's handled properly and will lead to crashing the module. + e.log.Warn().Err(err).Msg("nonce generation failed during pollHeight") + return + } + // send the request for synchronization req := &messages.SyncRequest{ - Nonce: rand.Uint64(), - Height: head.Height, + Nonce: nonce, + Height: final.Height, } e.log.Debug(). Uint64("height", req.Height). Uint64("range_nonce", req.Nonce). Msg("sending sync request") - err := e.con.Multicast(req, synccore.DefaultPollNodes, participants...) + err = e.con.Multicast(req, synccore.DefaultPollNodes, participants...) if err != nil { e.log.Warn().Err(err).Msg("sending sync request to poll heights failed") return @@ -380,12 +365,21 @@ func (e *Engine) sendRequests(participants flow.IdentifierList, ranges []chainsy var errs *multierror.Error for _, ran := range ranges { + nonce, err := rand.Uint64() + if err != nil { + // TODO: this error should be returned by sendRequests + // it is logged for now since the only error possible is related to a failure + // of the system entropy generation. Such error is going to cause failures in other + // components where it's handled properly and will lead to crashing the module. + e.log.Error().Err(err).Msg("nonce generation failed during range request") + return + } req := &messages.RangeRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, FromHeight: ran.From, ToHeight: ran.To, } - err := e.con.Multicast(req, synccore.DefaultBlockRequestNodes, participants...) + err = e.con.Multicast(req, synccore.DefaultBlockRequestNodes, participants...) if err != nil { errs = multierror.Append(errs, fmt.Errorf("could not submit range request: %w", err)) continue @@ -400,11 +394,20 @@ func (e *Engine) sendRequests(participants flow.IdentifierList, ranges []chainsy } for _, batch := range batches { + nonce, err := rand.Uint64() + if err != nil { + // TODO: this error should be returned by sendRequests + // it is logged for now since the only error possible is related to a failure + // of the system entropy generation. Such error is going to cause failures in other + // components where it's handled properly and will lead to crashing the module. + e.log.Error().Err(err).Msg("nonce generation failed during batch request") + return + } req := &messages.BatchRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, BlockIDs: batch.BlockIDs, } - err := e.con.Multicast(req, synccore.DefaultBlockRequestNodes, participants...) + err = e.con.Multicast(req, synccore.DefaultBlockRequestNodes, participants...) if err != nil { errs = multierror.Append(errs, fmt.Errorf("could not submit batch request: %w", err)) continue diff --git a/engine/common/synchronization/engine_test.go b/engine/common/synchronization/engine_test.go index ba83046a0e3..128c4684376 100644 --- a/engine/common/synchronization/engine_test.go +++ b/engine/common/synchronization/engine_test.go @@ -1,6 +1,7 @@ package synchronization import ( + "context" "io" "math" "math/rand" @@ -13,14 +14,13 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" - "github.com/onflow/flow-go/engine" mockconsensus "github.com/onflow/flow-go/engine/consensus/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/messages" synccore "github.com/onflow/flow-go/module/chainsync" "github.com/onflow/flow-go/module/id" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" netint "github.com/onflow/flow-go/network" @@ -58,9 +58,6 @@ type SyncSuite struct { } func (ss *SyncSuite) SetupTest() { - // seed the RNG - rand.Seed(time.Now().UnixNano()) - // generate own ID ss.participants = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleConsensus)) keys := unittest.NetworkingKeys(len(ss.participants)) @@ -168,12 +165,9 @@ func (ss *SyncSuite) SetupTest() { log := zerolog.New(io.Discard) metrics := metrics.NewNoopCollector() - finalizedHeader, err := NewFinalizedHeaderCache(log, ss.state, pubsub.NewFinalizationDistributor()) - require.NoError(ss.T(), err, "could not create finalized snapshot cache") - idCache, err := cache.NewProtocolStateIDCache(log, ss.state, protocolEvents.NewDistributor()) require.NoError(ss.T(), err, "could not create protocol state identity cache") - e, err := New(log, metrics, ss.net, ss.me, ss.blocks, ss.comp, ss.core, finalizedHeader, + e, err := New(log, metrics, ss.net, ss.me, ss.state, ss.blocks, ss.comp, ss.core, id.NewIdentityFilterIdentifierProvider( filter.And( filter.HasRole(flow.RoleConsensus), @@ -515,15 +509,19 @@ func (ss *SyncSuite) TestSendRequests() { // test a synchronization engine can be started and stopped func (ss *SyncSuite) TestStartStop() { - unittest.AssertReturnsBefore(ss.T(), func() { - <-ss.e.Ready() - <-ss.e.Done() - }, time.Second) + ctx, cancel := irrecoverable.NewMockSignalerContextWithCancel(ss.T(), context.Background()) + ss.e.Start(ctx) + unittest.AssertClosesBefore(ss.T(), ss.e.Ready(), time.Second) + cancel() + unittest.AssertClosesBefore(ss.T(), ss.e.Done(), time.Second) } // TestProcessingMultipleItems tests that items are processed in async way func (ss *SyncSuite) TestProcessingMultipleItems() { - <-ss.e.Ready() + ctx, cancel := irrecoverable.NewMockSignalerContextWithCancel(ss.T(), context.Background()) + ss.e.Start(ctx) + unittest.AssertClosesBefore(ss.T(), ss.e.Ready(), time.Second) + defer cancel() originID := unittest.IdentifierFixture() for i := 0; i < 5; i++ { @@ -556,20 +554,6 @@ func (ss *SyncSuite) TestProcessingMultipleItems() { ss.core.AssertExpectations(ss.T()) } -// TestOnFinalizedBlock tests that when new finalized block is discovered engine updates cached variables -// to latest state -func (ss *SyncSuite) TestOnFinalizedBlock() { - finalizedBlock := unittest.BlockHeaderWithParentFixture(ss.head) - // change head - ss.head = finalizedBlock - - err := ss.e.finalizedHeader.updateHeader() - require.NoError(ss.T(), err) - actualHeader := ss.e.finalizedHeader.Get() - require.ElementsMatch(ss.T(), ss.e.participantsProvider.Identifiers(), ss.participants[1:].NodeIDs()) - require.Equal(ss.T(), actualHeader, finalizedBlock) -} - // TestProcessUnsupportedMessageType tests that Process and ProcessLocal correctly handle a case where invalid message type // was submitted from network layer. func (ss *SyncSuite) TestProcessUnsupportedMessageType() { @@ -580,9 +564,4 @@ func (ss *SyncSuite) TestProcessUnsupportedMessageType() { // shouldn't result in error since byzantine inputs are expected require.NoError(ss.T(), err) } - - // in case of local processing error cannot be consumed since all inputs are trusted - err := ss.e.ProcessLocal(invalidEvent) - require.Error(ss.T(), err) - require.True(ss.T(), engine.IsIncompatibleInputTypeError(err)) } diff --git a/engine/common/synchronization/finalized_snapshot.go b/engine/common/synchronization/finalized_snapshot.go deleted file mode 100644 index a98b9fe6758..00000000000 --- a/engine/common/synchronization/finalized_snapshot.go +++ /dev/null @@ -1,137 +0,0 @@ -package synchronization - -import ( - "fmt" - "sync" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" - "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/lifecycle" - "github.com/onflow/flow-go/state/protocol" -) - -// FinalizedHeaderCache represents the cached value of the latest finalized header. -// It is used in Engine to access latest valid data. -type FinalizedHeaderCache struct { - mu sync.RWMutex - - log zerolog.Logger - state protocol.State - lastFinalizedHeader *flow.Header - finalizationEventNotifier engine.Notifier // notifier for finalization events - - lm *lifecycle.LifecycleManager - stopped chan struct{} -} - -// NewFinalizedHeaderCache creates a new finalized header cache. -func NewFinalizedHeaderCache(log zerolog.Logger, state protocol.State, finalizationDistributor *pubsub.FinalizationDistributor) (*FinalizedHeaderCache, error) { - cache := &FinalizedHeaderCache{ - state: state, - lm: lifecycle.NewLifecycleManager(), - log: log.With().Str("component", "finalized_snapshot_cache").Logger(), - finalizationEventNotifier: engine.NewNotifier(), - stopped: make(chan struct{}), - } - - snapshot, err := cache.getHeader() - if err != nil { - return nil, fmt.Errorf("could not apply last finalized state") - } - - cache.lastFinalizedHeader = snapshot - - finalizationDistributor.AddOnBlockFinalizedConsumer(cache.onFinalizedBlock) - - return cache, nil -} - -// Get returns the last locally cached finalized header. -func (f *FinalizedHeaderCache) Get() *flow.Header { - f.mu.RLock() - defer f.mu.RUnlock() - return f.lastFinalizedHeader -} - -func (f *FinalizedHeaderCache) getHeader() (*flow.Header, error) { - finalSnapshot := f.state.Final() - head, err := finalSnapshot.Head() - if err != nil { - return nil, fmt.Errorf("could not get last finalized header: %w", err) - } - - return head, nil -} - -// updateHeader updates latest locally cached finalized header. -func (f *FinalizedHeaderCache) updateHeader() error { - f.log.Debug().Msg("updating header") - - head, err := f.getHeader() - if err != nil { - f.log.Err(err).Msg("failed to get header") - return err - } - - f.log.Debug(). - Str("block_id", head.ID().String()). - Uint64("height", head.Height). - Msg("got new header") - - f.mu.Lock() - defer f.mu.Unlock() - - if f.lastFinalizedHeader.Height < head.Height { - f.lastFinalizedHeader = head - } - - return nil -} - -func (f *FinalizedHeaderCache) Ready() <-chan struct{} { - f.lm.OnStart(func() { - go f.finalizationProcessingLoop() - }) - return f.lm.Started() -} - -func (f *FinalizedHeaderCache) Done() <-chan struct{} { - f.lm.OnStop(func() { - <-f.stopped - }) - return f.lm.Stopped() -} - -// onFinalizedBlock implements the `OnFinalizedBlock` callback from the `hotstuff.FinalizationConsumer` -// (1) Updates local state of last finalized snapshot. -// -// CAUTION: the input to this callback is treated as trusted; precautions should be taken that messages -// from external nodes cannot be considered as inputs to this function -func (f *FinalizedHeaderCache) onFinalizedBlock(block *model.Block) { - f.log.Debug().Str("block_id", block.BlockID.String()).Msg("received new block finalization callback") - // notify that there is new finalized block - f.finalizationEventNotifier.Notify() -} - -// finalizationProcessingLoop is a separate goroutine that performs processing of finalization events -func (f *FinalizedHeaderCache) finalizationProcessingLoop() { - defer close(f.stopped) - - f.log.Debug().Msg("starting finalization processing loop") - notifier := f.finalizationEventNotifier.Channel() - for { - select { - case <-f.lm.ShutdownSignal(): - return - case <-notifier: - err := f.updateHeader() - if err != nil { - f.log.Fatal().Err(err).Msg("could not process latest finalized block") - } - } - } -} diff --git a/engine/common/synchronization/request_handler.go b/engine/common/synchronization/request_handler.go index 462f0d20835..cb5a3bde749 100644 --- a/engine/common/synchronization/request_handler.go +++ b/engine/common/synchronization/request_handler.go @@ -1,6 +1,7 @@ package synchronization import ( + "context" "errors" "fmt" @@ -11,7 +12,9 @@ import ( "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/chainsync" - "github.com/onflow/flow-go/module/lifecycle" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/events" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/storage" @@ -30,18 +33,26 @@ const defaultBatchRequestQueueCapacity = 500 // defaultEngineRequestsWorkers number of workers to dispatch events for requests const defaultEngineRequestsWorkers = 8 +// RequestHandler encapsulates message queues and processing logic for the sync engine. +// It logically separates request processing from active participation (sending requests), +// primarily to simplify nodes which bridge the public and private networks. +// +// The RequestHandlerEngine embeds RequestHandler to create an engine which only responds +// to requests on the public network (does not send requests over this network). +// The Engine embeds RequestHandler and additionally includes logic for sending sync requests. +// +// Although the RequestHandler defines a notifier, message queue, and processing worker logic, +// it is not itself a component.Component and does not manage any worker threads. The containing +// engine is responsible for starting the worker threads for processing requests. type RequestHandler struct { - lm *lifecycle.LifecycleManager - unit *engine.Unit - me module.Local log zerolog.Logger metrics module.EngineMetrics - blocks storage.Blocks - core module.SyncCore - finalizedHeader *FinalizedHeaderCache - responseSender ResponseSender + blocks storage.Blocks + finalizedHeaderCache module.FinalizedHeaderCache + core module.SyncCore + responseSender ResponseSender pendingSyncRequests engine.MessageStore // message store for *message.SyncRequest pendingBatchRequests engine.MessageStore // message store for *message.BatchRequest @@ -56,22 +67,20 @@ func NewRequestHandler( metrics module.EngineMetrics, responseSender ResponseSender, me module.Local, + finalizedHeaderCache *events.FinalizedHeaderCache, blocks storage.Blocks, core module.SyncCore, - finalizedHeader *FinalizedHeaderCache, queueMissingHeights bool, ) *RequestHandler { r := &RequestHandler{ - unit: engine.NewUnit(), - lm: lifecycle.NewLifecycleManager(), - me: me, - log: log.With().Str("engine", "synchronization").Logger(), - metrics: metrics, - blocks: blocks, - core: core, - finalizedHeader: finalizedHeader, - responseSender: responseSender, - queueMissingHeights: queueMissingHeights, + me: me, + log: log.With().Str("engine", "synchronization").Logger(), + metrics: metrics, + finalizedHeaderCache: finalizedHeaderCache, + blocks: blocks, + core: core, + responseSender: responseSender, + queueMissingHeights: queueMissingHeights, } r.setupRequestMessageHandler() @@ -79,10 +88,10 @@ func NewRequestHandler( return r } -// Process processes the given event from the node with the given origin ID in -// a blocking manner. It returns the potential processing error when done. +// Process processes the given event from the node with the given origin ID in a blocking manner. +// No errors are expected during normal operation. func (r *RequestHandler) Process(channel channels.Channel, originID flow.Identifier, event interface{}) error { - err := r.process(originID, event) + err := r.requestMessageHandler.Process(originID, event) if err != nil { if engine.IsIncompatibleInputTypeError(err) { r.log.Warn().Msgf("%v delivered unsupported message %T through %v", originID, event, channel) @@ -93,14 +102,6 @@ func (r *RequestHandler) Process(channel channels.Channel, originID flow.Identif return nil } -// process processes events for the synchronization request handler engine. -// Error returns: -// - IncompatibleInputTypeError if input has unexpected type -// - All other errors are potential symptoms of internal state corruption or bugs (fatal). -func (r *RequestHandler) process(originID flow.Identifier, event interface{}) error { - return r.requestMessageHandler.Process(originID, event) -} - // setupRequestMessageHandler initializes the inbound queues and the MessageHandler for UNTRUSTED requests. func (r *RequestHandler) setupRequestMessageHandler() { // RequestHeap deduplicates requests by keeping only one sync request for each requester. @@ -148,29 +149,30 @@ func (r *RequestHandler) setupRequestMessageHandler() { // onSyncRequest processes an outgoing handshake; if we have a higher height, we // inform the other node of it, so they can organize their block downloads. If // we have a lower height, we add the difference to our own download queue. +// No errors are expected during normal operation. func (r *RequestHandler) onSyncRequest(originID flow.Identifier, req *messages.SyncRequest) error { - final := r.finalizedHeader.Get() + finalizedHeader := r.finalizedHeaderCache.Get() logger := r.log.With().Str("origin_id", originID.String()).Logger() logger.Debug(). Uint64("origin_height", req.Height). - Uint64("local_height", final.Height). + Uint64("local_height", finalizedHeader.Height). Msg("received new sync request") if r.queueMissingHeights { // queue any missing heights as needed - r.core.HandleHeight(final, req.Height) + r.core.HandleHeight(finalizedHeader, req.Height) } // don't bother sending a response if we're within tolerance or if we're // behind the requester - if r.core.WithinTolerance(final, req.Height) || req.Height > final.Height { + if r.core.WithinTolerance(finalizedHeader, req.Height) || req.Height > finalizedHeader.Height { return nil } // if we're sufficiently ahead of the requester, send a response res := &messages.SyncResponse{ - Height: final.Height, + Height: finalizedHeader.Height, Nonce: req.Nonce, } err := r.responseSender.SendResponse(res, originID) @@ -184,15 +186,16 @@ func (r *RequestHandler) onSyncRequest(originID flow.Identifier, req *messages.S } // onRangeRequest processes a request for a range of blocks by height. +// No errors are expected during normal operation. func (r *RequestHandler) onRangeRequest(originID flow.Identifier, req *messages.RangeRequest) error { logger := r.log.With().Str("origin_id", originID.String()).Logger() logger.Debug().Msg("received new range request") // get the latest final state to know if we can fulfill the request - head := r.finalizedHeader.Get() + finalizedHeader := r.finalizedHeaderCache.Get() // if we don't have anything to send, we can bail right away - if head.Height < req.FromHeight || req.FromHeight > req.ToHeight { + if finalizedHeader.Height < req.FromHeight || req.FromHeight > req.ToHeight { return nil } @@ -217,7 +220,7 @@ func (r *RequestHandler) onRangeRequest(originID flow.Identifier, req *messages. req.ToHeight = maxHeight } - // get all of the blocks, one by one + // get all the blocks, one by one blocks := make([]messages.UntrustedBlock, 0, req.ToHeight-req.FromHeight+1) for height := req.FromHeight; height <= req.ToHeight; height++ { block, err := r.blocks.ByHeight(height) @@ -325,10 +328,10 @@ func (r *RequestHandler) onBatchRequest(originID flow.Identifier, req *messages. } // processAvailableRequests is processor of pending events which drives events from networking layer to business logic. -func (r *RequestHandler) processAvailableRequests() error { +func (r *RequestHandler) processAvailableRequests(ctx context.Context) error { for { select { - case <-r.unit.Quit(): + case <-ctx.Done(): return nil default: } @@ -366,39 +369,24 @@ func (r *RequestHandler) processAvailableRequests() error { } } -// requestProcessingLoop is a separate goroutine that performs processing of queued requests -func (r *RequestHandler) requestProcessingLoop() { +// requestProcessingWorker is a separate goroutine that performs processing of queued requests. +// Multiple instances may be invoked. It is invoked and managed by the Engine or RequestHandlerEngine +// which embeds this RequestHandler. +func (r *RequestHandler) requestProcessingWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + notifier := r.requestMessageHandler.GetNotifier() + done := ctx.Done() for { select { - case <-r.unit.Quit(): + case <-done: return case <-notifier: - err := r.processAvailableRequests() + err := r.processAvailableRequests(ctx) if err != nil { - r.log.Fatal().Err(err).Msg("internal error processing queued requests") + r.log.Err(err).Msg("internal error processing queued requests") + ctx.Throw(err) } } } } - -// Ready returns a ready channel that is closed once the engine has fully started. -func (r *RequestHandler) Ready() <-chan struct{} { - r.lm.OnStart(func() { - <-r.finalizedHeader.Ready() - for i := 0; i < defaultEngineRequestsWorkers; i++ { - r.unit.Launch(r.requestProcessingLoop) - } - }) - return r.lm.Started() -} - -// Done returns a done channel that is closed once the engine has fully stopped. -func (r *RequestHandler) Done() <-chan struct{} { - r.lm.OnStop(func() { - // wait for all request processing workers to exit - <-r.unit.Done() - <-r.finalizedHeader.Done() - }) - return r.lm.Stopped() -} diff --git a/engine/common/synchronization/request_handler_engine.go b/engine/common/synchronization/request_handler_engine.go index 95c49fd4442..ebbced56455 100644 --- a/engine/common/synchronization/request_handler_engine.go +++ b/engine/common/synchronization/request_handler_engine.go @@ -5,11 +5,15 @@ import ( "github.com/rs/zerolog" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/events" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" ) @@ -46,20 +50,29 @@ func NewResponseSender(con network.Conduit) *ResponseSenderImpl { } } +// RequestHandlerEngine is an engine which operates only the request-handling portion of the block sync protocol. +// It is used by Access/Observer nodes attached to the public network, enabling them +// to provide block synchronization data to nodes on the public network, but not +// requesting any data from these nodes. (Requests are sent only on the private network.) type RequestHandlerEngine struct { + component.Component + hotstuff.FinalizationConsumer + requestHandler *RequestHandler } var _ network.MessageProcessor = (*RequestHandlerEngine)(nil) +var _ component.Component = (*RequestHandlerEngine)(nil) +var _ hotstuff.FinalizationConsumer = (*RequestHandlerEngine)(nil) func NewRequestHandlerEngine( logger zerolog.Logger, metrics module.EngineMetrics, net network.Network, me module.Local, + state protocol.State, blocks storage.Blocks, core module.SyncCore, - finalizedHeader *FinalizedHeaderCache, ) (*RequestHandlerEngine, error) { e := &RequestHandlerEngine{} @@ -68,16 +81,26 @@ func NewRequestHandlerEngine( return nil, fmt.Errorf("could not register engine: %w", err) } + finalizedHeaderCache, finalizedCacheWorker, err := events.NewFinalizedHeaderCache(state) + if err != nil { + return nil, fmt.Errorf("could not initialize finalized header cache: %w", err) + } + e.FinalizationConsumer = finalizedHeaderCache e.requestHandler = NewRequestHandler( logger, metrics, NewResponseSender(con), me, + finalizedHeaderCache, blocks, core, - finalizedHeader, false, ) + builder := component.NewComponentManagerBuilder().AddWorker(finalizedCacheWorker) + for i := 0; i < defaultEngineRequestsWorkers; i++ { + builder.AddWorker(e.requestHandler.requestProcessingWorker) + } + e.Component = builder.Build() return e, nil } @@ -85,11 +108,3 @@ func NewRequestHandlerEngine( func (r *RequestHandlerEngine) Process(channel channels.Channel, originID flow.Identifier, event interface{}) error { return r.requestHandler.Process(channel, originID, event) } - -func (r *RequestHandlerEngine) Ready() <-chan struct{} { - return r.requestHandler.Ready() -} - -func (r *RequestHandlerEngine) Done() <-chan struct{} { - return r.requestHandler.Done() -} diff --git a/engine/consensus/approvals/request_tracker.go b/engine/consensus/approvals/request_tracker.go index 02520d10ee7..7669199c0c0 100644 --- a/engine/consensus/approvals/request_tracker.go +++ b/engine/consensus/approvals/request_tracker.go @@ -2,13 +2,13 @@ package approvals import ( "fmt" - "math/rand" "sync" "time" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/rand" ) /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -28,30 +28,45 @@ type RequestTrackerItem struct { // NewRequestTrackerItem instantiates a new RequestTrackerItem where the // NextTimeout is evaluated to the current time plus a random blackout period // contained between min and max. -func NewRequestTrackerItem(blackoutPeriodMin, blackoutPeriodMax int) RequestTrackerItem { +func NewRequestTrackerItem(blackoutPeriodMin, blackoutPeriodMax int) (RequestTrackerItem, error) { item := RequestTrackerItem{ blackoutPeriodMin: blackoutPeriodMin, blackoutPeriodMax: blackoutPeriodMax, } - item.NextTimeout = randBlackout(blackoutPeriodMin, blackoutPeriodMax) - return item + var err error + item.NextTimeout, err = randBlackout(blackoutPeriodMin, blackoutPeriodMax) + if err != nil { + return RequestTrackerItem{}, err + } + + return item, err } // Update creates a _new_ RequestTrackerItem with incremented request number and updated NextTimeout. -func (i RequestTrackerItem) Update() RequestTrackerItem { +// No errors are expected during normal operation. +func (i RequestTrackerItem) Update() (RequestTrackerItem, error) { i.Requests++ - i.NextTimeout = randBlackout(i.blackoutPeriodMin, i.blackoutPeriodMax) - return i + var err error + i.NextTimeout, err = randBlackout(i.blackoutPeriodMin, i.blackoutPeriodMax) + if err != nil { + return RequestTrackerItem{}, fmt.Errorf("could not get next timeout: %w", err) + } + return i, nil } func (i RequestTrackerItem) IsBlackout() bool { return time.Now().Before(i.NextTimeout) } -func randBlackout(min int, max int) time.Time { - blackoutSeconds := rand.Intn(max-min+1) + min +// No errors are expected during normal operation. +func randBlackout(min int, max int) (time.Time, error) { + random, err := rand.Uint64n(uint64(max - min + 1)) + if err != nil { + return time.Now(), fmt.Errorf("failed to generate blackout: %w", err) + } + blackoutSeconds := random + uint64(min) blackout := time.Now().Add(time.Duration(blackoutSeconds) * time.Second) - return blackout + return blackout, nil } /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -93,10 +108,14 @@ func (rt *RequestTracker) TryUpdate(result *flow.ExecutionResult, incorporatedBl rt.lock.Lock() defer rt.lock.Unlock() item, ok := rt.index[resultID][incorporatedBlockID][chunkIndex] + var err error if !ok { - item = NewRequestTrackerItem(rt.blackoutPeriodMin, rt.blackoutPeriodMax) - err := rt.set(resultID, result.BlockID, incorporatedBlockID, chunkIndex, item) + item, err = NewRequestTrackerItem(rt.blackoutPeriodMin, rt.blackoutPeriodMax) + if err != nil { + return item, false, fmt.Errorf("could not create tracker item: %w", err) + } + err = rt.set(resultID, result.BlockID, incorporatedBlockID, chunkIndex, item) if err != nil { return item, false, fmt.Errorf("could not set created tracker item: %w", err) } @@ -104,7 +123,10 @@ func (rt *RequestTracker) TryUpdate(result *flow.ExecutionResult, incorporatedBl canUpdate := !item.IsBlackout() if canUpdate { - item = item.Update() + item, err = item.Update() + if err != nil { + return item, false, fmt.Errorf("could not update tracker item: %w", err) + } rt.index[resultID][incorporatedBlockID][chunkIndex] = item } diff --git a/engine/consensus/approvals/verifying_assignment_collector.go b/engine/consensus/approvals/verifying_assignment_collector.go index 118627db3bc..a78131783f5 100644 --- a/engine/consensus/approvals/verifying_assignment_collector.go +++ b/engine/consensus/approvals/verifying_assignment_collector.go @@ -2,7 +2,6 @@ package approvals import ( "fmt" - "math/rand" "sync" "github.com/rs/zerolog" @@ -15,6 +14,7 @@ import ( "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/utils/rand" ) // **Emergency-sealing parameters** @@ -360,9 +360,14 @@ func (ac *VerifyingAssignmentCollector) RequestMissingApprovals(observation cons ) } + nonce, err := rand.Uint64() + if err != nil { + return 0, fmt.Errorf("nonce generation failed during request missing approvals: %w", err) + } + // prepare the request req := &messages.ApprovalRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, ResultID: ac.ResultID(), ChunkIndex: chunkIndex, } diff --git a/engine/consensus/compliance/core.go b/engine/consensus/compliance/core.go index d38e2b78dd4..0a0e8f1400b 100644 --- a/engine/consensus/compliance/core.go +++ b/engine/consensus/compliance/core.go @@ -14,11 +14,11 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/compliance" + "github.com/onflow/flow-go/module/counters" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/trace" @@ -36,16 +36,17 @@ import ( // - The only exception is calls to `ProcessFinalizedView`, which is the only concurrency-safe // method of compliance.Core type Core struct { - log zerolog.Logger // used to log relevant actions with context - config compliance.Config - engineMetrics module.EngineMetrics - mempoolMetrics module.MempoolMetrics - hotstuffMetrics module.HotstuffMetrics - complianceMetrics module.ComplianceMetrics - tracer module.Tracer - headers storage.Headers - payloads storage.Payloads - state protocol.ParticipantState + log zerolog.Logger // used to log relevant actions with context + config compliance.Config + engineMetrics module.EngineMetrics + mempoolMetrics module.MempoolMetrics + hotstuffMetrics module.HotstuffMetrics + complianceMetrics module.ComplianceMetrics + proposalViolationNotifier hotstuff.ProposalViolationConsumer + tracer module.Tracer + headers storage.Headers + payloads storage.Payloads + state protocol.ParticipantState // track latest finalized view/height - used to efficiently drop outdated or too-far-ahead blocks finalizedView counters.StrictMonotonousCounter finalizedHeight counters.StrictMonotonousCounter @@ -64,6 +65,7 @@ func NewCore( mempool module.MempoolMetrics, hotstuffMetrics module.HotstuffMetrics, complianceMetrics module.ComplianceMetrics, + proposalViolationNotifier hotstuff.ProposalViolationConsumer, tracer module.Tracer, headers storage.Headers, payloads storage.Payloads, @@ -74,31 +76,27 @@ func NewCore( hotstuff module.HotStuff, voteAggregator hotstuff.VoteAggregator, timeoutAggregator hotstuff.TimeoutAggregator, - opts ...compliance.Opt, + config compliance.Config, ) (*Core, error) { - config := compliance.DefaultConfig() - for _, apply := range opts { - apply(&config) - } - c := &Core{ - log: log.With().Str("compliance", "core").Logger(), - config: config, - engineMetrics: collector, - tracer: tracer, - mempoolMetrics: mempool, - hotstuffMetrics: hotstuffMetrics, - complianceMetrics: complianceMetrics, - headers: headers, - payloads: payloads, - state: state, - pending: pending, - sync: sync, - hotstuff: hotstuff, - validator: validator, - voteAggregator: voteAggregator, - timeoutAggregator: timeoutAggregator, + log: log.With().Str("compliance", "core").Logger(), + config: config, + engineMetrics: collector, + tracer: tracer, + mempoolMetrics: mempool, + hotstuffMetrics: hotstuffMetrics, + complianceMetrics: complianceMetrics, + proposalViolationNotifier: proposalViolationNotifier, + headers: headers, + payloads: payloads, + state: state, + pending: pending, + sync: sync, + hotstuff: hotstuff, + validator: validator, + voteAggregator: voteAggregator, + timeoutAggregator: timeoutAggregator, } // initialize finalized boundary cache @@ -116,9 +114,12 @@ func NewCore( // OnBlockProposal handles incoming block proposals. // No errors are expected during normal operation. All returned exceptions // are potential symptoms of internal state corruption and should be fatal. -func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.BlockProposal) error { - block := proposal.Block.ToInternal() - header := block.Header +func (c *Core) OnBlockProposal(proposal flow.Slashable[*messages.BlockProposal]) error { + block := flow.Slashable[*flow.Block]{ + OriginID: proposal.OriginID, + Message: proposal.Message.Block.ToInternal(), + } + header := block.Message.Header blockID := header.ID() finalHeight := c.finalizedHeight.Value() finalView := c.finalizedView.Value() @@ -126,14 +127,14 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc span, _ := c.tracer.StartBlockSpan(context.Background(), header.ID(), trace.CONCompOnBlockProposal) span.SetAttributes( attribute.Int64("view", int64(header.View)), - attribute.String("origin_id", originID.String()), + attribute.String("origin_id", proposal.OriginID.String()), attribute.String("proposer", header.ProposerID.String()), ) traceID := span.SpanContext().TraceID().String() defer span.End() log := c.log.With(). - Hex("origin_id", originID[:]). + Hex("origin_id", proposal.OriginID[:]). Str("chain_id", header.ChainID.String()). Uint64("block_height", header.Height). Uint64("block_view", header.View). @@ -155,12 +156,13 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc return nil } + skipNewProposalsThreshold := c.config.GetSkipNewProposalsThreshold() // ignore proposals which are too far ahead of our local finalized state // instead, rely on sync engine to catch up finalization more effectively, and avoid // large subtree of blocks to be cached. - if header.View > finalView+c.config.SkipNewProposalsThreshold { + if header.View > finalView+skipNewProposalsThreshold { log.Debug(). - Uint64("skip_new_proposals_threshold", c.config.SkipNewProposalsThreshold). + Uint64("skip_new_proposals_threshold", skipNewProposalsThreshold). Msg("dropping block too far ahead of locally finalized view") return nil } @@ -201,7 +203,7 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc _, found := c.pending.ByID(header.ParentID) if found { // add the block to the cache - _ = c.pending.Add(originID, block) + _ = c.pending.Add(block) c.mempoolMetrics.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) return nil @@ -210,18 +212,18 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc // if the proposal is connected to a block that is neither in the cache, nor // in persistent storage, its direct parent is missing; cache the proposal // and request the parent - parent, err := c.headers.ByBlockID(header.ParentID) - if errors.Is(err, storage.ErrNotFound) { - _ = c.pending.Add(originID, block) + exists, err := c.headers.Exists(header.ParentID) + if err != nil { + return fmt.Errorf("could not check parent exists: %w", err) + } + if !exists { + _ = c.pending.Add(block) c.mempoolMetrics.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) c.sync.RequestBlock(header.ParentID, header.Height-1) log.Debug().Msg("requesting missing parent for proposal") return nil } - if err != nil { - return fmt.Errorf("could not check parent: %w", err) - } // At this point, we should be able to connect the proposal to the finalized // state and should process it to see whether to forward to hotstuff or not. @@ -229,7 +231,7 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc // execution of the entire recursion, which might include processing the // proposal's pending children. There is another span within // processBlockProposal that measures the time spent for a single proposal. - err = c.processBlockAndDescendants(block, parent) + err = c.processBlockAndDescendants(block) c.mempoolMetrics.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) if err != nil { return fmt.Errorf("could not process block proposal: %w", err) @@ -244,25 +246,34 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc // processed as well. // No errors are expected during normal operation. All returned exceptions // are potential symptoms of internal state corruption and should be fatal. -func (c *Core) processBlockAndDescendants(proposal *flow.Block, parent *flow.Header) error { - blockID := proposal.Header.ID() +func (c *Core) processBlockAndDescendants(proposal flow.Slashable[*flow.Block]) error { + header := proposal.Message.Header + blockID := header.ID() log := c.log.With(). Str("block_id", blockID.String()). - Uint64("block_height", proposal.Header.Height). - Uint64("block_view", proposal.Header.View). - Uint64("parent_view", parent.View). + Uint64("block_height", header.Height). + Uint64("block_view", header.View). + Uint64("parent_view", header.ParentView). Logger() // process block itself - err := c.processBlockProposal(proposal, parent) + err := c.processBlockProposal(proposal.Message) if err != nil { - if checkForAndLogOutdatedInputError(err, log) { + if checkForAndLogOutdatedInputError(err, log) || checkForAndLogUnverifiableInputError(err, log) { return nil } - if checkForAndLogInvalidInputError(err, log) { + if invalidBlockErr, ok := model.AsInvalidProposalError(err); ok { + log.Err(err).Msg("received invalid block from other node (potential slashing evidence?)") + + // notify consumers about invalid block + c.proposalViolationNotifier.OnInvalidBlockDetected(flow.Slashable[model.InvalidProposalError]{ + OriginID: proposal.OriginID, + Message: *invalidBlockErr, + }) + // notify VoteAggregator about the invalid block - err = c.voteAggregator.InvalidBlock(model.ProposalFromFlow(proposal.Header)) + err = c.voteAggregator.InvalidBlock(model.ProposalFromFlow(header)) if err != nil { if mempool.IsBelowPrunedThresholdError(err) { log.Warn().Msg("received invalid block, but is below pruned threshold") @@ -284,7 +295,7 @@ func (c *Core) processBlockAndDescendants(proposal *flow.Block, parent *flow.Hea return nil } for _, child := range children { - cpr := c.processBlockAndDescendants(child.Message, proposal.Header) + cpr := c.processBlockAndDescendants(child) if cpr != nil { // unexpected error: potentially corrupted internal state => abort processing and escalate error return cpr @@ -301,8 +312,9 @@ func (c *Core) processBlockAndDescendants(proposal *flow.Block, parent *flow.Hea // the finalized state. // Expected errors during normal operations: // - engine.OutdatedInputError if the block proposal is outdated (e.g. orphaned) -// - engine.InvalidInputError if the block proposal is invalid -func (c *Core) processBlockProposal(proposal *flow.Block, parent *flow.Header) error { +// - model.InvalidProposalError if the block proposal is invalid +// - engine.UnverifiableInputError if the block proposal cannot be verified +func (c *Core) processBlockProposal(proposal *flow.Block) error { startTime := time.Now() defer func() { c.hotstuffMetrics.BlockProcessingDuration(time.Since(startTime)) @@ -320,21 +332,21 @@ func (c *Core) processBlockProposal(proposal *flow.Block, parent *flow.Header) e hotstuffProposal := model.ProposalFromFlow(header) err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { - if model.IsInvalidBlockError(err) { - return engine.NewInvalidInputErrorf("invalid block proposal: %w", err) + if model.IsInvalidProposalError(err) { + return err } if errors.Is(err, model.ErrViewForUnknownEpoch) { // We have received a proposal, but we don't know the epoch its view is within. // We know: // - the parent of this block is valid and was appended to the state (ie. we knew the epoch for it) // - if we then see this for the child, one of two things must have happened: - // 1. the proposer malicious created the block for a view very far in the future (it's invalid) + // 1. the proposer maliciously created the block for a view very far in the future (it's invalid) // -> in this case we can disregard the block - // 2. no blocks have been finalized within the epoch commitment deadline, and the epoch ended + // 2. no blocks have been finalized within the epoch commitment deadline, and the epoch ended // (breaking a critical assumption - see EpochCommitSafetyThreshold in protocol.Params for details) // -> in this case, the network has encountered a critical failure // - we assume in general that Case 2 will not happen, therefore this must be Case 1 - an invalid block - return engine.NewInvalidInputErrorf("invalid proposal with view from unknown epoch: %w", err) + return engine.NewUnverifiableInputError("unverifiable proposal with view from unknown epoch: %w", err) } return fmt.Errorf("unexpected error validating proposal: %w", err) } @@ -361,7 +373,7 @@ func (c *Core) processBlockProposal(proposal *flow.Block, parent *flow.Header) e if err != nil { if state.IsInvalidExtensionError(err) { // if the block proposes an invalid extension of the protocol state, then the block is invalid - return engine.NewInvalidInputErrorf("invalid extension of protocol state (block: %x, height: %d): %w", blockID, header.Height, err) + return model.NewInvalidProposalErrorf(hotstuffProposal, "invalid extension of protocol state (block: %x, height: %d): %w", blockID, header.Height, err) } if state.IsOutdatedExtensionError(err) { // protocol state aborted processing of block as it is on an abandoned fork: block is outdated @@ -409,14 +421,15 @@ func checkForAndLogOutdatedInputError(err error, log zerolog.Logger) bool { return false } -// checkForAndLogInvalidInputError checks whether error is an `engine.InvalidInputError`. +// checkForAndLogUnverifiableInputError checks whether error is an `engine.UnverifiableInputError`. // If this is the case, we emit a log message and return true. -// For any error other than `engine.InvalidInputError`, this function is a no-op +// For any error other than `engine.UnverifiableInputError`, this function is a no-op // and returns false. -func checkForAndLogInvalidInputError(err error, log zerolog.Logger) bool { - if engine.IsInvalidInputError(err) { - // the block is invalid; log as error as we desire honest participation - log.Err(err).Msg("received invalid block from other node (potential slashing evidence?)") +func checkForAndLogUnverifiableInputError(err error, log zerolog.Logger) bool { + if engine.IsUnverifiableInputError(err) { + // the block cannot be validated + log.Warn().Err(err).Msg("received unverifiable block proposal; " + + "this might be an indicator that a malicious proposer is generating detached blocks very far ahead") return true } return false diff --git a/engine/consensus/compliance/core_test.go b/engine/consensus/compliance/core_test.go index 34bc9e3570c..36dddda317d 100644 --- a/engine/consensus/compliance/core_test.go +++ b/engine/consensus/compliance/core_test.go @@ -2,9 +2,7 @@ package compliance import ( "errors" - "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -58,31 +56,29 @@ type CommonSuite struct { childrenDB map[flow.Identifier][]flow.Slashable[*flow.Block] // mocked dependencies - me *module.Local - metrics *metrics.NoopCollector - tracer realModule.Tracer - headers *storage.Headers - payloads *storage.Payloads - state *protocol.ParticipantState - snapshot *protocol.Snapshot - con *mocknetwork.Conduit - net *mocknetwork.Network - prov *consensus.ProposalProvider - pending *module.PendingBlockBuffer - hotstuff *module.HotStuff - sync *module.BlockRequester - validator *hotstuff.Validator - voteAggregator *hotstuff.VoteAggregator - timeoutAggregator *hotstuff.TimeoutAggregator + me *module.Local + metrics *metrics.NoopCollector + tracer realModule.Tracer + headers *storage.Headers + payloads *storage.Payloads + state *protocol.ParticipantState + snapshot *protocol.Snapshot + con *mocknetwork.Conduit + net *mocknetwork.Network + prov *consensus.ProposalProvider + pending *module.PendingBlockBuffer + hotstuff *module.HotStuff + sync *module.BlockRequester + proposalViolationNotifier *hotstuff.ProposalViolationConsumer + validator *hotstuff.Validator + voteAggregator *hotstuff.VoteAggregator + timeoutAggregator *hotstuff.TimeoutAggregator // engine under test core *Core } func (cs *CommonSuite) SetupTest() { - // seed the RNG - rand.Seed(time.Now().UnixNano()) - // initialize the paramaters cs.participants = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleConsensus), @@ -130,6 +126,13 @@ func (cs *CommonSuite) SetupTest() { return nil }, ) + cs.headers.On("Exists", mock.Anything).Return( + func(blockID flow.Identifier) bool { + _, exists := cs.headerDB[blockID] + return exists + }, func(blockID flow.Identifier) error { + return nil + }) // set up payload storage mock cs.payloads = &storage.Payloads{} @@ -244,6 +247,9 @@ func (cs *CommonSuite) SetupTest() { // set up no-op tracer cs.tracer = trace.NewNoopTracer() + // set up notifier for reporting protocol violations + cs.proposalViolationNotifier = hotstuff.NewProposalViolationConsumer(cs.T()) + // initialize the engine e, err := NewCore( unittest.Logger(), @@ -251,6 +257,7 @@ func (cs *CommonSuite) SetupTest() { cs.metrics, cs.metrics, cs.metrics, + cs.proposalViolationNotifier, cs.tracer, cs.headers, cs.payloads, @@ -261,6 +268,7 @@ func (cs *CommonSuite) SetupTest() { cs.hotstuff, cs.voteAggregator, cs.timeoutAggregator, + compliance.DefaultConfig(), ) require.NoError(cs.T(), err, "engine initialization should pass") @@ -283,7 +291,10 @@ func (cs *CoreSuite) TestOnBlockProposalValidParent() { cs.hotstuff.On("SubmitProposal", hotstuffProposal) // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "valid block proposal should pass") // we should extend the state with the header @@ -309,7 +320,10 @@ func (cs *CoreSuite) TestOnBlockProposalValidAncestor() { cs.hotstuff.On("SubmitProposal", hotstuffProposal) // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "valid block proposal should pass") // we should extend the state with the header @@ -324,7 +338,10 @@ func (cs *CoreSuite) TestOnBlockProposalSkipProposalThreshold() { block.Header.View = cs.head.View + compliance.DefaultConfig().SkipNewProposalsThreshold + 1 proposal := unittest.ProposalFromBlock(&block) - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err) // block should be dropped - not added to state or cache @@ -355,12 +372,20 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.Run("invalid block error", func() { // the block fails HotStuff validation *cs.validator = *hotstuff.NewValidator(cs.T()) - cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.InvalidBlockError{}) + sentinelError := model.NewInvalidProposalErrorf(hotstuffProposal, "") + cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError) + cs.proposalViolationNotifier.On("OnInvalidBlockDetected", flow.Slashable[model.InvalidProposalError]{ + OriginID: originID, + Message: sentinelError.(model.InvalidProposalError), + }).Return().Once() // we should notify VoteAggregator about the invalid block cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) // the expected error should be handled within the Core - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "proposal with invalid extension should fail") // we should not extend the state with the header @@ -375,7 +400,10 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.ErrViewForUnknownEpoch) // the expected error should be handled within the Core - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "proposal with invalid extension should fail") // we should not extend the state with the header @@ -391,7 +419,10 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { cs.validator.On("ValidateProposal", hotstuffProposal).Return(unexpectedErr) // the error should be propagated - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.ErrorIs(cs.T(), err, unexpectedErr) // we should not extend the state with the header @@ -427,12 +458,22 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { // make sure we fail to extend the state *cs.state = protocol.ParticipantState{} cs.state.On("Final").Return(func() protint.Snapshot { return cs.snapshot }) - cs.state.On("Extend", mock.Anything, mock.Anything).Return(state.NewInvalidExtensionError("")) + sentinelErr := state.NewInvalidExtensionError("") + cs.state.On("Extend", mock.Anything, mock.Anything).Return(sentinelErr) + cs.proposalViolationNotifier.On("OnInvalidBlockDetected", mock.Anything).Run(func(args mock.Arguments) { + err := args.Get(0).(flow.Slashable[model.InvalidProposalError]) + require.ErrorIs(cs.T(), err.Message, sentinelErr) + require.Equal(cs.T(), err.Message.InvalidProposal, hotstuffProposal) + require.Equal(cs.T(), err.OriginID, originID) + }).Return().Once() // we should notify VoteAggregator about the invalid block cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) // the expected error should be handled within the Core - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "proposal with invalid extension should fail") // we should extend the state with the header @@ -450,7 +491,10 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { cs.state.On("Extend", mock.Anything, mock.Anything).Return(state.NewOutdatedExtensionError("")) // the expected error should be handled within the Core - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "proposal with invalid extension should fail") // we should extend the state with the header @@ -469,7 +513,10 @@ func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { cs.state.On("Extend", mock.Anything, mock.Anything).Return(unexpectedErr) // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.ErrorIs(cs.T(), err, unexpectedErr) // we should extend the state with the header @@ -511,7 +558,10 @@ func (cs *CoreSuite) TestProcessBlockAndDescendants() { } // execute the connected children handling - err := cs.core.processBlockAndDescendants(parent, cs.head) + err := cs.core.processBlockAndDescendants(flow.Slashable[*flow.Block]{ + OriginID: unittest.IdentifierFixture(), + Message: parent, + }) require.NoError(cs.T(), err, "should pass handling children") // make sure we drop the cache after trying to process @@ -544,7 +594,10 @@ func (cs *CoreSuite) TestProposalBufferingOrder() { // process all the descendants for _, proposal := range proposals { // process and make sure no error occurs (as they are unverifiable) - err := cs.core.OnBlockProposal(originID, proposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: proposal, + }) require.NoError(cs.T(), err, "proposal buffering should pass") // make sure no block is forwarded to hotstuff @@ -580,7 +633,10 @@ func (cs *CoreSuite) TestProposalBufferingOrder() { cs.voteAggregator.On("AddBlock", mock.Anything).Times(4) // process the root proposal - err := cs.core.OnBlockProposal(originID, missingProposal) + err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: originID, + Message: missingProposal, + }) require.NoError(cs.T(), err, "root proposal should pass") // all proposals should be processed diff --git a/engine/consensus/compliance/engine.go b/engine/consensus/compliance/engine.go index d1a2b530e65..4a14a4e5aa9 100644 --- a/engine/consensus/compliance/engine.go +++ b/engine/consensus/compliance/engine.go @@ -5,8 +5,8 @@ import ( "github.com/rs/zerolog" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/consensus/hotstuff/tracker" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/common/fifoqueue" "github.com/onflow/flow-go/engine/consensus" @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/events" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/state/protocol" @@ -29,20 +30,20 @@ const defaultBlockQueueCapacity = 10_000 // `compliance.Core` implements the actual compliance logic. // Implements consensus.Compliance interface. type Engine struct { - *component.ComponentManager - log zerolog.Logger - mempoolMetrics module.MempoolMetrics - engineMetrics module.EngineMetrics - me module.Local - headers storage.Headers - payloads storage.Payloads - tracer module.Tracer - state protocol.State - core *Core - pendingBlocks *fifoqueue.FifoQueue // queue for processing inbound blocks - pendingBlocksNotifier engine.Notifier - finalizedBlockTracker *tracker.NewestBlockTracker - finalizedBlockNotifier engine.Notifier + component.Component + hotstuff.FinalizationConsumer + + log zerolog.Logger + mempoolMetrics module.MempoolMetrics + engineMetrics module.EngineMetrics + me module.Local + headers storage.Headers + payloads storage.Payloads + tracer module.Tracer + state protocol.State + core *Core + pendingBlocks *fifoqueue.FifoQueue // queue for processing inbound blocks + pendingBlocksNotifier engine.Notifier } var _ consensus.Compliance = (*Engine)(nil) @@ -63,25 +64,24 @@ func NewEngine( } eng := &Engine{ - log: log.With().Str("compliance", "engine").Logger(), - me: me, - mempoolMetrics: core.mempoolMetrics, - engineMetrics: core.engineMetrics, - headers: core.headers, - payloads: core.payloads, - pendingBlocks: blocksQueue, - state: core.state, - tracer: core.tracer, - core: core, - pendingBlocksNotifier: engine.NewNotifier(), - finalizedBlockTracker: tracker.NewNewestBlockTracker(), - finalizedBlockNotifier: engine.NewNotifier(), + log: log.With().Str("compliance", "engine").Logger(), + me: me, + mempoolMetrics: core.mempoolMetrics, + engineMetrics: core.engineMetrics, + headers: core.headers, + payloads: core.payloads, + pendingBlocks: blocksQueue, + state: core.state, + tracer: core.tracer, + core: core, + pendingBlocksNotifier: engine.NewNotifier(), } - + finalizationActor, finalizationWorker := events.NewFinalizationActor(eng.processOnFinalizedBlock) + eng.FinalizationConsumer = finalizationActor // create the component manager and worker threads - eng.ComponentManager = component.NewComponentManagerBuilder(). + eng.Component = component.NewComponentManagerBuilder(). AddWorker(eng.processBlocksLoop). - AddWorker(eng.finalizationProcessingLoop). + AddWorker(finalizationWorker). Build() return eng, nil @@ -122,7 +122,10 @@ func (e *Engine) processQueuedBlocks(doneSignal <-chan struct{}) error { if ok { batch := msg.(flow.Slashable[[]*messages.BlockProposal]) for _, block := range batch.Message { - err := e.core.OnBlockProposal(batch.OriginID, block) + err := e.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{ + OriginID: batch.OriginID, + Message: block, + }) e.core.engineMetrics.MessageHandled(metrics.EngineCompliance, metrics.MessageBlockProposal) if err != nil { return fmt.Errorf("could not handle block proposal: %w", err) @@ -137,17 +140,6 @@ func (e *Engine) processQueuedBlocks(doneSignal <-chan struct{}) error { } } -// OnFinalizedBlock implements the `OnFinalizedBlock` callback from the `hotstuff.FinalizationConsumer` -// It informs compliance.Core about finalization of the respective block. -// -// CAUTION: the input to this callback is treated as trusted; precautions should be taken that messages -// from external nodes cannot be considered as inputs to this function -func (e *Engine) OnFinalizedBlock(block *model.Block) { - if e.finalizedBlockTracker.Track(block) { - e.finalizedBlockNotifier.Notify() - } -} - // OnBlockProposal feeds a new block proposal into the processing pipeline. // Incoming proposals are queued and eventually dispatched by worker. func (e *Engine) OnBlockProposal(proposal flow.Slashable[*messages.BlockProposal]) { @@ -175,23 +167,16 @@ func (e *Engine) OnSyncedBlocks(blocks flow.Slashable[[]*messages.BlockProposal] } } -// finalizationProcessingLoop is a separate goroutine that performs processing of finalization events -func (e *Engine) finalizationProcessingLoop(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - ready() - - doneSignal := ctx.Done() - blockFinalizedSignal := e.finalizedBlockNotifier.Channel() - for { - select { - case <-doneSignal: - return - case <-blockFinalizedSignal: - // retrieve the latest finalized header, so we know the height - finalHeader, err := e.headers.ByBlockID(e.finalizedBlockTracker.NewestBlock().BlockID) - if err != nil { // no expected errors - ctx.Throw(err) - } - e.core.ProcessFinalizedBlock(finalHeader) - } +// processOnFinalizedBlock informs compliance.Core about finalization of the respective block. +// The input to this callback is treated as trusted. This method should be executed on +// `OnFinalizedBlock` notifications from the node-internal consensus instance. +// No errors expected during normal operations. +func (e *Engine) processOnFinalizedBlock(block *model.Block) error { + // retrieve the latest finalized header, so we know the height + finalHeader, err := e.headers.ByBlockID(block.BlockID) + if err != nil { // no expected errors + return fmt.Errorf("could not get finalized header: %w", err) } + e.core.ProcessFinalizedBlock(finalHeader) + return nil } diff --git a/engine/consensus/compliance/engine_test.go b/engine/consensus/compliance/engine_test.go index b2f899ccce7..a82ccc558c7 100644 --- a/engine/consensus/compliance/engine_test.go +++ b/engine/consensus/compliance/engine_test.go @@ -70,7 +70,6 @@ func (cs *EngineSuite) TestSubmittingMultipleEntries() { for i := 0; i < blockCount; i++ { block := unittest.BlockWithParentFixture(cs.head) proposal := messages.NewBlockProposal(block) - cs.headerDB[block.Header.ParentID] = cs.head hotstuffProposal := model.ProposalFromFlow(block.Header) cs.hotstuff.On("SubmitProposal", hotstuffProposal).Return().Once() cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() @@ -89,8 +88,6 @@ func (cs *EngineSuite) TestSubmittingMultipleEntries() { block := unittest.BlockWithParentFixture(cs.head) proposal := unittest.ProposalFromBlock(block) - // store the data for retrieval - cs.headerDB[block.Header.ParentID] = cs.head hotstuffProposal := model.ProposalFromFlow(block.Header) cs.hotstuff.On("SubmitProposal", hotstuffProposal).Return().Once() cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() @@ -128,6 +125,7 @@ func (cs *EngineSuite) TestOnFinalizedBlock() { Run(func(_ mock.Arguments) { wg.Done() }). Return(uint(0)).Once() - cs.engine.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock)) + err := cs.engine.processOnFinalizedBlock(model.BlockFromFlow(finalizedBlock)) + require.NoError(cs.T(), err) unittest.AssertReturnsBefore(cs.T(), wg.Wait, time.Second, "an expected call to block buffer wasn't made") } diff --git a/engine/consensus/dkg/messaging_engine.go b/engine/consensus/dkg/messaging_engine.go index 80705d93b51..18862110083 100644 --- a/engine/consensus/dkg/messaging_engine.go +++ b/engine/consensus/dkg/messaging_engine.go @@ -2,6 +2,7 @@ package dkg import ( "context" + "errors" "fmt" "time" @@ -9,178 +10,236 @@ import ( "github.com/sethvargo/go-retry" "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/common/fifoqueue" "github.com/onflow/flow-go/model/flow" msg "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/dkg" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/utils/logging" ) -// retryMax is the maximum number of times the engine will attempt to forward -// a message before permanently giving up. -const retryMax = 9 - -// retryBaseWait is the duration to wait between the two first tries. -// With 9 attempts and exponential backoff, this will retry for about -// 8m before giving up. -const retryBaseWait = 1 * time.Second +// MessagingEngineConfig configures outbound message submission. +type MessagingEngineConfig struct { + // RetryMax is the maximum number of times the engine will attempt to send + // an outbound message before permanently giving up. + RetryMax uint64 + // RetryBaseWait is the duration to wait between the two first send attempts. + RetryBaseWait time.Duration + // RetryJitterPercent is the percent jitter to add to each inter-retry wait. + RetryJitterPercent uint64 +} -// retryJitterPct is the percent jitter to add to each inter-retry wait. -const retryJitterPct = 25 +// DefaultMessagingEngineConfig returns the config defaults. With 9 attempts and +// exponential backoff, this will retry for about 8m before giving up. +func DefaultMessagingEngineConfig() MessagingEngineConfig { + return MessagingEngineConfig{ + RetryMax: 9, + RetryBaseWait: time.Second, + RetryJitterPercent: 25, + } +} -// MessagingEngine is a network engine that enables DKG nodes to exchange -// private messages over the network. +// MessagingEngine is an engine which sends and receives all DKG private messages. +// The same engine instance is used for the lifetime of a node and will be used +// for different DKG instances. The ReactorEngine is responsible for the lifecycle +// of components which are scoped one DKG instance, for example the DKGController. +// The dkg.BrokerTunnel handles routing messages to/from the current DKG instance. type MessagingEngine struct { - unit *engine.Unit log zerolog.Logger - me module.Local // local object to identify the node - conduit network.Conduit // network conduit for sending and receiving private messages - tunnel *dkg.BrokerTunnel // tunnel for relaying private messages to and from controllers + me module.Local // local object to identify the node + conduit network.Conduit // network conduit for sending and receiving private messages + tunnel *dkg.BrokerTunnel // tunnel for relaying private messages to and from controllers + config MessagingEngineConfig // config for outbound message transmission + + messageHandler *engine.MessageHandler // encapsulates enqueueing messages from network + notifier engine.Notifier // notifies inbound messages available for forwarding + inbound *fifoqueue.FifoQueue // messages from the network, to be processed by DKG Controller + + component.Component + cm *component.ComponentManager } -// NewMessagingEngine returns a new engine. +var _ network.MessageProcessor = (*MessagingEngine)(nil) +var _ component.Component = (*MessagingEngine)(nil) + +// NewMessagingEngine returns a new MessagingEngine. func NewMessagingEngine( - logger zerolog.Logger, + log zerolog.Logger, net network.Network, me module.Local, - tunnel *dkg.BrokerTunnel) (*MessagingEngine, error) { + tunnel *dkg.BrokerTunnel, + collector module.MempoolMetrics, + config MessagingEngineConfig, +) (*MessagingEngine, error) { + log = log.With().Str("engine", "dkg_messaging").Logger() + + inbound, err := fifoqueue.NewFifoQueue( + 1000, + fifoqueue.WithLengthMetricObserver(metrics.ResourceDKGMessage, collector.MempoolEntries)) + if err != nil { + return nil, fmt.Errorf("could not create inbound fifoqueue: %w", err) + } - log := logger.With().Str("engine", "dkg-processor").Logger() + notifier := engine.NewNotifier() + messageHandler := engine.NewMessageHandler(log, notifier, engine.Pattern{ + Match: engine.MatchType[*msg.DKGMessage], + Store: &engine.FifoMessageStore{FifoQueue: inbound}, + }) eng := MessagingEngine{ - unit: engine.NewUnit(), - log: log, - me: me, - tunnel: tunnel, + log: log, + me: me, + tunnel: tunnel, + messageHandler: messageHandler, + notifier: notifier, + inbound: inbound, + config: config, } - var err error - eng.conduit, err = net.Register(channels.DKGCommittee, &eng) + conduit, err := net.Register(channels.DKGCommittee, &eng) if err != nil { return nil, fmt.Errorf("could not register dkg network engine: %w", err) } + eng.conduit = conduit - eng.unit.Launch(eng.forwardOutgoingMessages) + eng.cm = component.NewComponentManagerBuilder(). + AddWorker(eng.forwardInboundMessagesWorker). + AddWorker(eng.forwardOutboundMessagesWorker). + Build() + eng.Component = eng.cm return &eng, nil } -// Ready implements the module ReadyDoneAware interface. It returns a channel -// that will close when the engine has successfully -// started. -func (e *MessagingEngine) Ready() <-chan struct{} { - return e.unit.Ready() -} - -// Done implements the module ReadyDoneAware interface. It returns a channel -// that will close when the engine has successfully stopped. -func (e *MessagingEngine) Done() <-chan struct{} { - return e.unit.Done() -} - -// SubmitLocal implements the network Engine interface -func (e *MessagingEngine) SubmitLocal(event interface{}) { - e.unit.Launch(func() { - err := e.process(e.me.NodeID(), event) - if err != nil { - e.log.Fatal().Err(err).Str("origin", e.me.NodeID().String()).Msg("failed to submit local message") +// Process processes messages from the networking layer. +// No errors are expected during normal operation. +func (e *MessagingEngine) Process(channel channels.Channel, originID flow.Identifier, message any) error { + err := e.messageHandler.Process(originID, message) + if err != nil { + if errors.Is(err, engine.IncompatibleInputTypeError) { + e.log.Warn().Bool(logging.KeySuspicious, true).Msgf("%v delivered unsupported message %T through %v", originID, message, channel) + return nil } - }) + return fmt.Errorf("unexpected failure to process inbound dkg message: %w", err) + } + return nil } -// Submit implements the network Engine interface -func (e *MessagingEngine) Submit(_ channels.Channel, originID flow.Identifier, event interface{}) { - e.unit.Launch(func() { - err := e.process(originID, event) - if engine.IsInvalidInputError(err) { - e.log.Error().Err(err).Str("origin", originID.String()).Msg("failed to submit dropping invalid input message") - } else if err != nil { - e.log.Fatal().Err(err).Str("origin", originID.String()).Msg("failed to submit message unknown error") +// forwardInboundMessagesWorker reads queued inbound messages and forwards them +// through the broker tunnel to the DKG Controller for processing. +// This is a worker routine which runs for the lifetime of the engine. +func (e *MessagingEngine) forwardInboundMessagesWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + done := ctx.Done() + wake := e.notifier.Channel() + for { + select { + case <-done: + return + case <-wake: + e.forwardInboundMessagesWhileAvailable(ctx) } - }) + } } -// ProcessLocal implements the network Engine interface -func (e *MessagingEngine) ProcessLocal(event interface{}) error { - return e.unit.Do(func() error { - err := e.process(e.me.NodeID(), event) - if err != nil { - e.log.Fatal().Err(err).Str("origin", e.me.NodeID().String()).Msg("failed to process local message") - } +// popNextInboundMessage pops one message from the queue and returns it as the +// appropriate type expected by the DKG controller. +func (e *MessagingEngine) popNextInboundMessage() (msg.PrivDKGMessageIn, bool) { + nextMessage, ok := e.inbound.Pop() + if !ok { + return msg.PrivDKGMessageIn{}, false + } + asEngineWrapper := nextMessage.(*engine.Message) + asDKGMsg := asEngineWrapper.Payload.(*msg.DKGMessage) + originID := asEngineWrapper.OriginID - return nil - }) + message := msg.PrivDKGMessageIn{ + DKGMessage: *asDKGMsg, + OriginID: originID, + } + return message, true } -// Process implements the network Engine interface -func (e *MessagingEngine) Process(_ channels.Channel, originID flow.Identifier, event interface{}) error { - return e.unit.Do(func() error { - return e.process(originID, event) - }) -} +// forwardInboundMessagesWhileAvailable retrieves all inbound messages from the queue and +// sends to the DKG Controller over the broker tunnel. Exists when the queue is empty. +func (e *MessagingEngine) forwardInboundMessagesWhileAvailable(ctx context.Context) { + for { + started := time.Now() + message, ok := e.popNextInboundMessage() + if !ok { + return + } -func (e *MessagingEngine) process(originID flow.Identifier, event interface{}) error { - switch v := event.(type) { - case *msg.DKGMessage: - // messages are forwarded async rather than sync, because otherwise the message queue - // might get full when it's slow to process DKG messages synchronously and impact - // block rate. - e.forwardInboundMessageAsync(originID, v) - return nil - default: - return engine.NewInvalidInputErrorf("expecting input with type msg.DKGMessage, but got %T", event) + select { + case <-ctx.Done(): + return + case e.tunnel.MsgChIn <- message: + e.log.Debug().Dur("waited", time.Since(started)).Msg("forwarded DKG message to Broker") + continue + } } } -// forwardInboundMessageAsync forwards a private DKG message from another DKG -// participant to the DKG controller. -func (e *MessagingEngine) forwardInboundMessageAsync(originID flow.Identifier, message *msg.DKGMessage) { - e.unit.Launch(func() { - e.tunnel.SendIn( - msg.PrivDKGMessageIn{ - DKGMessage: *message, - OriginID: originID, - }, - ) - }) -} +// forwardOutboundMessagesWorker reads outbound DKG messages created by our DKG Controller +// and sends them to the appropriate other DKG participant. Each outbound message is sent +// async in an ad-hoc goroutine, which internally manages retry backoff for the message. +// This is a worker routine which runs for the lifetime of the engine. +func (e *MessagingEngine) forwardOutboundMessagesWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() -func (e *MessagingEngine) forwardOutgoingMessages() { + done := ctx.Done() for { select { - case msg := <-e.tunnel.MsgChOut: - e.forwardOutboundMessageAsync(msg) - case <-e.unit.Quit(): + case <-done: return + case message := <-e.tunnel.MsgChOut: + go e.forwardOutboundMessage(ctx, message) } } } -// forwardOutboundMessageAsync asynchronously attempts to forward a private -// DKG message to a single other DKG participant, on a best effort basis. -func (e *MessagingEngine) forwardOutboundMessageAsync(message msg.PrivDKGMessageOut) { - e.unit.Launch(func() { - backoff := retry.NewExponential(retryBaseWait) - backoff = retry.WithMaxRetries(retryMax, backoff) - backoff = retry.WithJitterPercent(retryJitterPct, backoff) - - attempts := 1 - err := retry.Do(e.unit.Ctx(), backoff, func(ctx context.Context) error { - err := e.conduit.Unicast(&message.DKGMessage, message.DestID) - if err != nil { - e.log.Warn().Err(err).Msgf("error sending dkg message retrying (%d)", attempts) - } - - attempts++ - return retry.RetryableError(err) - }) - - // Various network conditions can result in errors while forwarding outbound messages. - // Because the overall DKG is resilient to individual message failures most of time. - // it is acceptable to log the error and move on. +// forwardOutboundMessage transmits message to the target DKG participant. +// Upon any error from the Unicast, we will retry with an exponential backoff. +// After a limited number of attempts, we will log an error and exit. +// The DKG protocol tolerates a number of failed private messages - these will +// be resolved by broadcasting complaints in later phases. +// Must be invoked as a goroutine. +func (e *MessagingEngine) forwardOutboundMessage(ctx context.Context, message msg.PrivDKGMessageOut) { + backoff := retry.NewExponential(e.config.RetryBaseWait) + backoff = retry.WithMaxRetries(e.config.RetryMax, backoff) + backoff = retry.WithJitterPercent(e.config.RetryJitterPercent, backoff) + + started := time.Now() + log := e.log.With().Str("target", message.DestID.String()).Logger() + + attempts := 0 + err := retry.Do(ctx, backoff, func(ctx context.Context) error { + attempts++ + err := e.conduit.Unicast(&message.DKGMessage, message.DestID) + // TODO Unicast does not document expected errors, therefore we treat all errors as benign networking failures here if err != nil { - e.log.Error().Err(err).Msgf("error sending private dkg message after %d attempts", attempts) + log.Warn(). + Err(err). + Int("attempt", attempts). + Dur("send_time", time.Since(started)). + Msgf("error sending dkg message on attempt %d - will retry...", attempts) } + + return retry.RetryableError(err) }) + + // TODO Unicast does not document expected errors, therefore we treat all errors as benign networking failures here + if err != nil { + log.Error(). + Err(err). + Int("total_attempts", attempts). + Dur("total_send_time", time.Since(started)). + Msgf("failed to send private dkg message after %d attempts - will not retry", attempts) + } } diff --git a/engine/consensus/dkg/messaging_engine_test.go b/engine/consensus/dkg/messaging_engine_test.go index 1c7d1c6e7fb..b3ca1e42ff3 100644 --- a/engine/consensus/dkg/messaging_engine_test.go +++ b/engine/consensus/dkg/messaging_engine_test.go @@ -1,51 +1,70 @@ package dkg import ( + "context" "testing" "time" - "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" msg "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/dkg" - module "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/utils/unittest" ) -// Helper function to initialise an engine. -func createTestEngine(t *testing.T) *MessagingEngine { +// MessagingEngineSuite encapsulates unit tests for the MessagingEngine. +type MessagingEngineSuite struct { + suite.Suite + + conduit *mocknetwork.Conduit + network *mocknetwork.Network + me *mockmodule.Local + + engine *MessagingEngine +} + +func TestMessagingEngine(t *testing.T) { + suite.Run(t, new(MessagingEngineSuite)) +} + +func (ms *MessagingEngineSuite) SetupTest() { // setup mock conduit - conduit := &mocknetwork.Conduit{} - network := new(mocknetwork.Network) - network.On("Register", mock.Anything, mock.Anything). - Return(conduit, nil). + ms.conduit = mocknetwork.NewConduit(ms.T()) + ms.network = mocknetwork.NewNetwork(ms.T()) + ms.network.On("Register", mock.Anything, mock.Anything). + Return(ms.conduit, nil). Once() // setup local with nodeID nodeID := unittest.IdentifierFixture() - me := new(module.Local) - me.On("NodeID").Return(nodeID) + ms.me = mockmodule.NewLocal(ms.T()) + ms.me.On("NodeID").Return(nodeID).Maybe() engine, err := NewMessagingEngine( - zerolog.Logger{}, - network, - me, + unittest.Logger(), + ms.network, + ms.me, dkg.NewBrokerTunnel(), + metrics.NewNoopCollector(), + DefaultMessagingEngineConfig(), ) - require.NoError(t, err) - - return engine + require.NoError(ms.T(), err) + ms.engine = engine } // TestForwardOutgoingMessages checks that the engine correctly forwards // outgoing messages from the tunnel's Out channel to the network conduit. -func TestForwardOutgoingMessages(t *testing.T) { - // sender engine - engine := createTestEngine(t) +func (ms *MessagingEngineSuite) TestForwardOutgoingMessages() { + ctx, cancel := irrecoverable.NewMockSignalerContextWithCancel(ms.T(), context.Background()) + ms.engine.Start(ctx) + defer cancel() // expected DKGMessage destinationID := unittest.IdentifierFixture() @@ -54,29 +73,26 @@ func TestForwardOutgoingMessages(t *testing.T) { "dkg-123", ) - // override the conduit to check that the Unicast call matches the expected - // message and destination ID - conduit := &mocknetwork.Conduit{} - conduit.On("Unicast", &expectedMsg, destinationID). + done := make(chan struct{}) + ms.conduit.On("Unicast", &expectedMsg, destinationID). + Run(func(_ mock.Arguments) { close(done) }). Return(nil). Once() - engine.conduit = conduit - engine.tunnel.SendOut(msg.PrivDKGMessageOut{ + ms.engine.tunnel.SendOut(msg.PrivDKGMessageOut{ DKGMessage: expectedMsg, DestID: destinationID, }) - time.Sleep(5 * time.Millisecond) - - conduit.AssertExpectations(t) + unittest.RequireCloseBefore(ms.T(), done, time.Second, "message not sent") } -// TestForwardIncomingMessages checks that the engine correclty forwards -// messages from the conduit to the tunnel's In channel. -func TestForwardIncomingMessages(t *testing.T) { - // sender engine - e := createTestEngine(t) +// TestForwardIncomingMessages checks that the engine correctly forwards +// messages from the conduit to the tunnel's MsgChIn channel. +func (ms *MessagingEngineSuite) TestForwardIncomingMessages() { + ctx, cancel := irrecoverable.NewMockSignalerContextWithCancel(ms.T(), context.Background()) + ms.engine.Start(ctx) + defer cancel() originID := unittest.IdentifierFixture() expectedMsg := msg.PrivDKGMessageIn{ @@ -84,17 +100,16 @@ func TestForwardIncomingMessages(t *testing.T) { OriginID: originID, } - // launch a background routine to capture messages forwarded to the tunnel's - // In channel - doneCh := make(chan struct{}) + // launch a background routine to capture messages forwarded to the tunnel's MsgChIn channel + done := make(chan struct{}) go func() { - receivedMsg := <-e.tunnel.MsgChIn - require.Equal(t, expectedMsg, receivedMsg) - close(doneCh) + receivedMsg := <-ms.engine.tunnel.MsgChIn + require.Equal(ms.T(), expectedMsg, receivedMsg) + close(done) }() - err := e.Process(channels.DKGCommittee, originID, &expectedMsg.DKGMessage) - require.NoError(t, err) + err := ms.engine.Process(channels.DKGCommittee, originID, &expectedMsg.DKGMessage) + require.NoError(ms.T(), err) - unittest.RequireCloseBefore(t, doneCh, time.Second, "message not received") + unittest.RequireCloseBefore(ms.T(), done, time.Second, "message not received") } diff --git a/engine/consensus/ingestion/core_test.go b/engine/consensus/ingestion/core_test.go index 7ca7737052e..6167f6d55ee 100644 --- a/engine/consensus/ingestion/core_test.go +++ b/engine/consensus/ingestion/core_test.go @@ -15,6 +15,7 @@ import ( "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/state/protocol" mockprotocol "github.com/onflow/flow-go/state/protocol/mock" mockstorage "github.com/onflow/flow-go/storage/mock" @@ -37,6 +38,9 @@ type IngestionCoreSuite struct { finalIdentities flow.IdentityList // identities at finalized state refIdentities flow.IdentityList // identities at reference block state + epochCounter uint64 // epoch for the cluster originating the guarantee + clusterMembers flow.IdentityList // members of the cluster originating the guarantee + clusterID flow.ChainID // chain ID of the cluster originating the guarantee final *mockprotocol.Snapshot // finalized state snapshot ref *mockprotocol.Snapshot // state snapshot w.r.t. reference block @@ -66,7 +70,9 @@ func (suite *IngestionCoreSuite) SetupTest() { suite.execID = exec.NodeID suite.verifID = verif.NodeID - clusters := flow.IdentityList{coll} + suite.epochCounter = 1 + suite.clusterMembers = flow.IdentityList{coll} + suite.clusterID = cluster.CanonicalClusterID(suite.epochCounter, suite.clusterMembers.NodeIDs()) identities := flow.IdentityList{access, con, coll, exec, verif} suite.finalIdentities = identities.Copy() @@ -109,8 +115,20 @@ func (suite *IngestionCoreSuite) SetupTest() { ) ref.On("Epochs").Return(suite.query) suite.query.On("Current").Return(suite.epoch) - cluster.On("Members").Return(clusters) - suite.epoch.On("ClusterByChainID", head.ChainID).Return(cluster, nil) + cluster.On("Members").Return(suite.clusterMembers) + suite.epoch.On("ClusterByChainID", mock.Anything).Return( + func(chainID flow.ChainID) protocol.Cluster { + if chainID == suite.clusterID { + return cluster + } + return nil + }, + func(chainID flow.ChainID) error { + if chainID == suite.clusterID { + return nil + } + return protocol.ErrClusterNotFound + }) state.On("AtBlockID", mock.Anything).Return(ref) ref.On("Identity", mock.Anything).Return( @@ -234,7 +252,23 @@ func (suite *IngestionCoreSuite) TestOnGuaranteeExpired() { err := suite.core.OnGuarantee(suite.collID, guarantee) suite.Assert().Error(err, "should error with expired collection") suite.Assert().True(engine.IsOutdatedInputError(err)) +} + +// TestOnGuaranteeReferenceBlockFromWrongEpoch validates that guarantees which contain a ChainID +// that is inconsistent with the reference block (ie. the ChainID either refers to a non-existent +// cluster, or a cluster for a different epoch) should be considered invalid inputs. +func (suite *IngestionCoreSuite) TestOnGuaranteeReferenceBlockFromWrongEpoch() { + // create a guarantee from a cluster in a different epoch + guarantee := suite.validGuarantee() + guarantee.ChainID = cluster.CanonicalClusterID(suite.epochCounter+1, suite.clusterMembers.NodeIDs()) + // the guarantee is not part of the memory pool + suite.pool.On("Has", guarantee.ID()).Return(false) + + // submit the guarantee as if it was sent by a collection node + err := suite.core.OnGuarantee(suite.collID, guarantee) + suite.Assert().Error(err, "should error with expired collection") + suite.Assert().True(engine.IsInvalidInputError(err)) } // TestOnGuaranteeInvalidGuarantor verifiers that collections with any _unknown_ @@ -306,7 +340,7 @@ func (suite *IngestionCoreSuite) TestOnGuaranteeUnknownOrigin() { // validGuarantee returns a valid collection guarantee based on the suite state. func (suite *IngestionCoreSuite) validGuarantee() *flow.CollectionGuarantee { guarantee := unittest.CollectionGuaranteeFixture() - guarantee.ChainID = suite.head.ChainID + guarantee.ChainID = suite.clusterID signerIndices, err := signature.EncodeSignersToIndices( []flow.Identifier{suite.collID}, []flow.Identifier{suite.collID}) diff --git a/engine/consensus/message_hub/message_hub.go b/engine/consensus/message_hub/message_hub.go index 3e4da058b26..6c674c219ff 100644 --- a/engine/consensus/message_hub/message_hub.go +++ b/engine/consensus/message_hub/message_hub.go @@ -473,7 +473,12 @@ func (h *MessageHub) Process(channel channels.Channel, originID flow.Identifier, } h.forwardToOwnTimeoutAggregator(t) default: - h.log.Warn().Msgf("%v delivered unsupported message %T through %v", originID, message, channel) + h.log.Warn(). + Bool(logging.KeySuspicious, true). + Hex("origin_id", logging.ID(originID)). + Str("message_type", fmt.Sprintf("%T", message)). + Str("channel", channel.String()). + Msgf("delivered unsupported message type") } return nil } diff --git a/engine/consensus/message_hub/message_hub_test.go b/engine/consensus/message_hub/message_hub_test.go index 16896be4de8..a68ce9eeb7a 100644 --- a/engine/consensus/message_hub/message_hub_test.go +++ b/engine/consensus/message_hub/message_hub_test.go @@ -2,7 +2,6 @@ package message_hub import ( "context" - "math/rand" "sync" "testing" "time" @@ -65,9 +64,6 @@ type MessageHubSuite struct { } func (s *MessageHubSuite) SetupTest() { - // seed the RNG - rand.Seed(time.Now().UnixNano()) - // initialize the paramaters s.participants = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleConsensus), diff --git a/engine/consensus/sealing/core.go b/engine/consensus/sealing/core.go index 942d489e971..1bf9350e09f 100644 --- a/engine/consensus/sealing/core.go +++ b/engine/consensus/sealing/core.go @@ -17,9 +17,9 @@ import ( "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/consensus" "github.com/onflow/flow-go/engine/consensus/approvals" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/counters" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/trace" "github.com/onflow/flow-go/network" @@ -137,7 +137,7 @@ func (c *Core) RepopulateAssignmentCollectorTree(payloads storage.Payloads) erro // Get the root block of our local state - we allow references to unknown // blocks below the root height - rootHeader, err := c.state.Params().Root() + rootHeader, err := c.state.Params().FinalizedRoot() if err != nil { return fmt.Errorf("could not retrieve root header: %w", err) } diff --git a/engine/consensus/sealing/core_test.go b/engine/consensus/sealing/core_test.go index 4dfbc31d50c..264c3a652ea 100644 --- a/engine/consensus/sealing/core_test.go +++ b/engine/consensus/sealing/core_test.go @@ -60,7 +60,7 @@ func (s *ApprovalProcessingCoreTestSuite) SetupTest() { params := new(mockstate.Params) s.State.On("Sealed").Return(unittest.StateSnapshotForKnownBlock(s.ParentBlock, nil)).Maybe() s.State.On("Params").Return(params) - params.On("Root").Return( + params.On("FinalizedRoot").Return( func() *flow.Header { return s.rootHeader }, func() error { return nil }, ) diff --git a/engine/consensus/sealing/engine.go b/engine/consensus/sealing/engine.go index ae432725bd6..60d38a57fe7 100644 --- a/engine/consensus/sealing/engine.go +++ b/engine/consensus/sealing/engine.go @@ -104,7 +104,7 @@ func NewEngine(log zerolog.Logger, sealsMempool mempool.IncorporatedResultSeals, requiredApprovalsForSealConstructionGetter module.SealingConfigsGetter, ) (*Engine, error) { - rootHeader, err := state.Params().Root() + rootHeader, err := state.Params().FinalizedRoot() if err != nil { return nil, fmt.Errorf("could not retrieve root block: %w", err) } diff --git a/engine/enqueue.go b/engine/enqueue.go index 7e4d645f24c..2999cf5cd9a 100644 --- a/engine/enqueue.go +++ b/engine/enqueue.go @@ -43,6 +43,11 @@ type MatchFunc func(*Message) bool type MapFunc func(*Message) (*Message, bool) +func MatchType[T any](m *Message) bool { + _, ok := m.Payload.(T) + return ok +} + type MessageHandler struct { log zerolog.Logger notifier Notifier diff --git a/engine/execution/block_result.go b/engine/execution/block_result.go new file mode 100644 index 00000000000..d2e57641d16 --- /dev/null +++ b/engine/execution/block_result.go @@ -0,0 +1,218 @@ +package execution + +import ( + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/mempool/entity" +) + +// BlockExecutionResult captures artifacts of execution of block collections +type BlockExecutionResult struct { + *entity.ExecutableBlock + + collectionExecutionResults []CollectionExecutionResult +} + +// NewPopulatedBlockExecutionResult constructs a new BlockExecutionResult, +// pre-populated with `chunkCounts` number of collection results +func NewPopulatedBlockExecutionResult(eb *entity.ExecutableBlock) *BlockExecutionResult { + chunkCounts := len(eb.CompleteCollections) + 1 + return &BlockExecutionResult{ + ExecutableBlock: eb, + collectionExecutionResults: make([]CollectionExecutionResult, chunkCounts), + } +} + +// Size returns the size of collection execution results +func (er *BlockExecutionResult) Size() int { + return len(er.collectionExecutionResults) +} + +func (er *BlockExecutionResult) CollectionExecutionResultAt(colIndex int) *CollectionExecutionResult { + if colIndex < 0 && colIndex > len(er.collectionExecutionResults) { + return nil + } + return &er.collectionExecutionResults[colIndex] +} + +func (er *BlockExecutionResult) AllEvents() flow.EventsList { + res := make(flow.EventsList, 0) + for _, ce := range er.collectionExecutionResults { + if len(ce.events) > 0 { + res = append(res, ce.events...) + } + } + return res +} + +func (er *BlockExecutionResult) AllServiceEvents() flow.EventsList { + res := make(flow.EventsList, 0) + for _, ce := range er.collectionExecutionResults { + if len(ce.serviceEvents) > 0 { + res = append(res, ce.serviceEvents...) + } + } + return res +} + +func (er *BlockExecutionResult) TransactionResultAt(txIdx int) *flow.TransactionResult { + allTxResults := er.AllTransactionResults() // TODO: optimize me + if txIdx > len(allTxResults) { + return nil + } + return &allTxResults[txIdx] +} + +func (er *BlockExecutionResult) AllTransactionResults() flow.TransactionResults { + res := make(flow.TransactionResults, 0) + for _, ce := range er.collectionExecutionResults { + if len(ce.transactionResults) > 0 { + res = append(res, ce.transactionResults...) + } + } + return res +} + +func (er *BlockExecutionResult) AllExecutionSnapshots() []*snapshot.ExecutionSnapshot { + res := make([]*snapshot.ExecutionSnapshot, 0) + for _, ce := range er.collectionExecutionResults { + es := ce.ExecutionSnapshot() + res = append(res, es) + } + return res +} + +func (er *BlockExecutionResult) AllConvertedServiceEvents() flow.ServiceEventList { + res := make(flow.ServiceEventList, 0) + for _, ce := range er.collectionExecutionResults { + if len(ce.convertedServiceEvents) > 0 { + res = append(res, ce.convertedServiceEvents...) + } + } + return res +} + +// BlockAttestationResult holds collection attestation results +type BlockAttestationResult struct { + *BlockExecutionResult + + collectionAttestationResults []CollectionAttestationResult + + // TODO(ramtin): move this to the outside, everything needed for create this + // should be available as part of computation result and most likely trieUpdate + // was the reason this is kept here, long term we don't need this data and should + // act based on register deltas + *execution_data.BlockExecutionData +} + +func NewEmptyBlockAttestationResult( + blockExecutionResult *BlockExecutionResult, +) *BlockAttestationResult { + colSize := blockExecutionResult.Size() + return &BlockAttestationResult{ + BlockExecutionResult: blockExecutionResult, + collectionAttestationResults: make([]CollectionAttestationResult, 0, colSize), + BlockExecutionData: &execution_data.BlockExecutionData{ + BlockID: blockExecutionResult.ID(), + ChunkExecutionDatas: make( + []*execution_data.ChunkExecutionData, + 0, + colSize), + }, + } +} + +// CollectionAttestationResultAt returns CollectionAttestationResult at collection index +func (ar *BlockAttestationResult) CollectionAttestationResultAt(colIndex int) *CollectionAttestationResult { + if colIndex < 0 && colIndex > len(ar.collectionAttestationResults) { + return nil + } + return &ar.collectionAttestationResults[colIndex] +} + +func (ar *BlockAttestationResult) AppendCollectionAttestationResult( + startStateCommit flow.StateCommitment, + endStateCommit flow.StateCommitment, + stateProof flow.StorageProof, + eventCommit flow.Identifier, + chunkExecutionDatas *execution_data.ChunkExecutionData, +) { + ar.collectionAttestationResults = append(ar.collectionAttestationResults, + CollectionAttestationResult{ + startStateCommit: startStateCommit, + endStateCommit: endStateCommit, + stateProof: stateProof, + eventCommit: eventCommit, + }, + ) + ar.ChunkExecutionDatas = append(ar.ChunkExecutionDatas, chunkExecutionDatas) +} + +func (ar *BlockAttestationResult) AllChunks() []*flow.Chunk { + chunks := make([]*flow.Chunk, len(ar.collectionAttestationResults)) + for i := 0; i < len(ar.collectionAttestationResults); i++ { + chunks[i] = ar.ChunkAt(i) // TODO(ramtin): cache and optimize this + } + return chunks +} + +func (ar *BlockAttestationResult) ChunkAt(index int) *flow.Chunk { + if index < 0 || index >= len(ar.collectionAttestationResults) { + return nil + } + + execRes := ar.collectionExecutionResults[index] + attestRes := ar.collectionAttestationResults[index] + + return flow.NewChunk( + ar.Block.ID(), + index, + attestRes.startStateCommit, + len(execRes.TransactionResults()), + attestRes.eventCommit, + attestRes.endStateCommit, + ) +} + +func (ar *BlockAttestationResult) AllChunkDataPacks() []*flow.ChunkDataPack { + chunkDataPacks := make([]*flow.ChunkDataPack, len(ar.collectionAttestationResults)) + for i := 0; i < len(ar.collectionAttestationResults); i++ { + chunkDataPacks[i] = ar.ChunkDataPackAt(i) // TODO(ramtin): cache and optimize this + } + return chunkDataPacks +} + +func (ar *BlockAttestationResult) ChunkDataPackAt(index int) *flow.ChunkDataPack { + if index < 0 || index >= len(ar.collectionAttestationResults) { + return nil + } + + // Note: There's some inconsistency in how chunk execution data and + // chunk data pack populate their collection fields when the collection + // is the system collection. + // collectionAt would return nil if the collection is system collection + collection := ar.CollectionAt(index) + + attestRes := ar.collectionAttestationResults[index] + + return flow.NewChunkDataPack( + ar.ChunkAt(index).ID(), // TODO(ramtin): optimize this + attestRes.startStateCommit, + attestRes.stateProof, + collection, + ) +} + +func (ar *BlockAttestationResult) AllEventCommitments() []flow.Identifier { + res := make([]flow.Identifier, 0) + for _, ca := range ar.collectionAttestationResults { + res = append(res, ca.EventCommitment()) + } + return res +} + +// Size returns the size of collection attestation results +func (ar *BlockAttestationResult) Size() int { + return len(ar.collectionAttestationResults) +} diff --git a/engine/execution/collection_result.go b/engine/execution/collection_result.go new file mode 100644 index 00000000000..cbe43813b8c --- /dev/null +++ b/engine/execution/collection_result.go @@ -0,0 +1,108 @@ +package execution + +import ( + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/model/flow" +) + +// CollectionExecutionResult holds aggregated artifacts (events, tx resutls, ...) +// generated during collection execution +type CollectionExecutionResult struct { + events flow.EventsList + serviceEvents flow.EventsList + convertedServiceEvents flow.ServiceEventList + transactionResults flow.TransactionResults + executionSnapshot *snapshot.ExecutionSnapshot +} + +// NewEmptyCollectionExecutionResult constructs a new CollectionExecutionResult +func NewEmptyCollectionExecutionResult() *CollectionExecutionResult { + return &CollectionExecutionResult{ + events: make(flow.EventsList, 0), + serviceEvents: make(flow.EventsList, 0), + convertedServiceEvents: make(flow.ServiceEventList, 0), + transactionResults: make(flow.TransactionResults, 0), + } +} + +func (c *CollectionExecutionResult) AppendTransactionResults( + events flow.EventsList, + serviceEvents flow.EventsList, + convertedServiceEvents flow.ServiceEventList, + transactionResult flow.TransactionResult, +) { + c.events = append(c.events, events...) + c.serviceEvents = append(c.serviceEvents, serviceEvents...) + c.convertedServiceEvents = append(c.convertedServiceEvents, convertedServiceEvents...) + c.transactionResults = append(c.transactionResults, transactionResult) +} + +func (c *CollectionExecutionResult) UpdateExecutionSnapshot( + executionSnapshot *snapshot.ExecutionSnapshot, +) { + c.executionSnapshot = executionSnapshot +} + +func (c *CollectionExecutionResult) ExecutionSnapshot() *snapshot.ExecutionSnapshot { + return c.executionSnapshot +} + +func (c *CollectionExecutionResult) Events() flow.EventsList { + return c.events +} + +func (c *CollectionExecutionResult) ServiceEventList() flow.EventsList { + return c.serviceEvents +} + +func (c *CollectionExecutionResult) ConvertedServiceEvents() flow.ServiceEventList { + return c.convertedServiceEvents +} + +func (c *CollectionExecutionResult) TransactionResults() flow.TransactionResults { + return c.transactionResults +} + +// CollectionAttestationResult holds attestations generated during post-processing +// phase of collect execution. +type CollectionAttestationResult struct { + startStateCommit flow.StateCommitment + endStateCommit flow.StateCommitment + stateProof flow.StorageProof + eventCommit flow.Identifier +} + +func NewCollectionAttestationResult( + startStateCommit flow.StateCommitment, + endStateCommit flow.StateCommitment, + stateProof flow.StorageProof, + eventCommit flow.Identifier, +) *CollectionAttestationResult { + return &CollectionAttestationResult{ + startStateCommit: startStateCommit, + endStateCommit: endStateCommit, + stateProof: stateProof, + eventCommit: eventCommit, + } +} + +func (a *CollectionAttestationResult) StartStateCommitment() flow.StateCommitment { + return a.startStateCommit +} + +func (a *CollectionAttestationResult) EndStateCommitment() flow.StateCommitment { + return a.endStateCommit +} + +func (a *CollectionAttestationResult) StateProof() flow.StorageProof { + return a.stateProof +} + +func (a *CollectionAttestationResult) EventCommitment() flow.Identifier { + return a.eventCommit +} + +// TODO(ramtin): depricate in the future, temp method, needed for uploader for now +func (a *CollectionAttestationResult) UpdateEndStateCommitment(endState flow.StateCommitment) { + a.endStateCommit = endState +} diff --git a/engine/execution/computation/committer/committer.go b/engine/execution/computation/committer/committer.go index 504a8b1ca65..7856acab4f3 100644 --- a/engine/execution/computation/committer/committer.go +++ b/engine/execution/computation/committer/committer.go @@ -7,7 +7,7 @@ import ( "github.com/hashicorp/go-multierror" execState "github.com/onflow/flow-go/engine/execution/state" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -29,7 +29,7 @@ func NewLedgerViewCommitter( } func (committer *LedgerViewCommitter) CommitView( - snapshot *state.ExecutionSnapshot, + snapshot *snapshot.ExecutionSnapshot, baseState flow.StateCommitment, ) ( newCommit flow.StateCommitment, @@ -61,13 +61,20 @@ func (committer *LedgerViewCommitter) CommitView( } func (committer *LedgerViewCommitter) collectProofs( - snapshot *state.ExecutionSnapshot, + snapshot *snapshot.ExecutionSnapshot, baseState flow.StateCommitment, ) ( proof []byte, err error, ) { - // get all deduplicated register IDs + // Reason for including AllRegisterIDs (read and written registers) instead of ReadRegisterIDs (only read registers): + // AllRegisterIDs returns deduplicated register IDs that were touched by both + // reads and writes during the block execution. + // Verification nodes only need the registers in the storage proof that were touched by reads + // in order to execute transactions in a chunk. However, without the registers touched + // by writes, especially the interim trie nodes for them, verification nodes won't be + // able to reconstruct the trie root hash of the execution state post execution. That's why + // the storage proof needs both read registers and write registers, which specifically is AllRegisterIDs allIds := snapshot.AllRegisterIDs() keys := make([]ledger.Key, 0, len(allIds)) for _, id := range allIds { diff --git a/engine/execution/computation/committer/committer_test.go b/engine/execution/computation/committer/committer_test.go index a340eaeaa65..18657a67f13 100644 --- a/engine/execution/computation/committer/committer_test.go +++ b/engine/execution/computation/committer/committer_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/engine/execution/computation/committer" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" led "github.com/onflow/flow-go/ledger" ledgermock "github.com/onflow/flow-go/ledger/mock" "github.com/onflow/flow-go/model/flow" @@ -34,7 +34,7 @@ func TestLedgerViewCommitter(t *testing.T) { Once() newState, proof, _, err := com.CommitView( - &state.ExecutionSnapshot{ + &snapshot.ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ flow.NewRegisterID("owner", "key"): []byte{1}, }, diff --git a/engine/execution/computation/committer/noop.go b/engine/execution/computation/committer/noop.go index 82d2d234cea..dcdefbac634 100644 --- a/engine/execution/computation/committer/noop.go +++ b/engine/execution/computation/committer/noop.go @@ -1,7 +1,7 @@ package committer import ( - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" ) @@ -14,7 +14,7 @@ func NewNoopViewCommitter() *NoopViewCommitter { } func (NoopViewCommitter) CommitView( - _ *state.ExecutionSnapshot, + _ *snapshot.ExecutionSnapshot, s flow.StateCommitment, ) ( flow.StateCommitment, diff --git a/engine/execution/computation/computer/computer.go b/engine/execution/computation/computer/computer.go index d291050ccfd..4c3749f9e8f 100644 --- a/engine/execution/computation/computer/computer.go +++ b/engine/execution/computation/computer/computer.go @@ -3,7 +3,7 @@ package computer import ( "context" "fmt" - "time" + "sync" "github.com/rs/zerolog" "go.opentelemetry.io/otel/attribute" @@ -15,15 +15,16 @@ import ( "github.com/onflow/flow-go/engine/execution/utils" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/fvm/blueprints" - "github.com/onflow/flow-go/fvm/state" - "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/errors" + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/executiondatasync/provider" "github.com/onflow/flow-go/module/mempool/entity" "github.com/onflow/flow-go/module/trace" - "github.com/onflow/flow-go/utils/debug" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/utils/logging" ) @@ -41,53 +42,7 @@ type collectionInfo struct { isSystemTransaction bool } -func newTransactions( - collection collectionInfo, - collectionCtx fvm.Context, - startTxnIndex int, -) []transaction { - txns := make([]transaction, 0, len(collection.Transactions)) - - logger := collectionCtx.Logger.With(). - Str("block_id", collection.blockIdStr). - Uint64("height", collectionCtx.BlockHeader.Height). - Bool("system_chunk", collection.isSystemTransaction). - Bool("system_transaction", collection.isSystemTransaction). - Logger() - - for idx, txnBody := range collection.Transactions { - txnId := txnBody.ID() - txnIdStr := txnId.String() - txnIndex := uint32(startTxnIndex + idx) - txns = append( - txns, - transaction{ - collectionInfo: collection, - txnId: txnId, - txnIdStr: txnIdStr, - txnIndex: txnIndex, - ctx: fvm.NewContextFromParent( - collectionCtx, - fvm.WithLogger( - logger.With(). - Str("tx_id", txnIdStr). - Uint32("tx_index", txnIndex). - Logger())), - TransactionProcedure: fvm.NewTransaction( - txnId, - txnIndex, - txnBody), - }) - } - - if len(txns) > 0 { - txns[len(txns)-1].lastTransactionInCollection = true - } - - return txns -} - -type transaction struct { +type TransactionRequest struct { collectionInfo txnId flow.Identifier @@ -101,13 +56,44 @@ type transaction struct { *fvm.TransactionProcedure } +func newTransactionRequest( + collection collectionInfo, + collectionCtx fvm.Context, + collectionLogger zerolog.Logger, + txnIndex uint32, + txnBody *flow.TransactionBody, + lastTransactionInCollection bool, +) TransactionRequest { + txnId := txnBody.ID() + txnIdStr := txnId.String() + + return TransactionRequest{ + collectionInfo: collection, + txnId: txnId, + txnIdStr: txnIdStr, + txnIndex: txnIndex, + ctx: fvm.NewContextFromParent( + collectionCtx, + fvm.WithLogger( + collectionLogger.With(). + Str("tx_id", txnIdStr). + Uint32("tx_index", txnIndex). + Logger())), + TransactionProcedure: fvm.NewTransaction( + txnId, + txnIndex, + txnBody), + lastTransactionInCollection: lastTransactionInCollection, + } +} + // A BlockComputer executes the transactions in a block. type BlockComputer interface { ExecuteBlock( ctx context.Context, parentBlockExecutionResultID flow.Identifier, block *entity.ExecutableBlock, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, derivedBlockData *derived.DerivedBlockData, ) ( *execution.ComputationResult, @@ -128,6 +114,8 @@ type blockComputer struct { spockHasher hash.Hasher receiptHasher hash.Hasher colResCons []result.ExecutedCollectionConsumer + protocolState protocol.State + maxConcurrency int } func SystemChunkContext(vmCtx fvm.Context, logger zerolog.Logger) fvm.Context { @@ -155,7 +143,12 @@ func NewBlockComputer( signer module.Local, executionDataProvider *provider.Provider, colResCons []result.ExecutedCollectionConsumer, + state protocol.State, + maxConcurrency int, ) (BlockComputer, error) { + if maxConcurrency < 1 { + return nil, fmt.Errorf("invalid maxConcurrency: %d", maxConcurrency) + } systemChunkCtx := SystemChunkContext(vmCtx, logger) vmCtx = fvm.NewContextFromParent( vmCtx, @@ -174,6 +167,8 @@ func NewBlockComputer( spockHasher: utils.NewSPOCKHasher(), receiptHasher: utils.NewExecutionReceiptHasher(), colResCons: colResCons, + protocolState: state, + maxConcurrency: maxConcurrency, }, nil } @@ -182,7 +177,7 @@ func (e *blockComputer) ExecuteBlock( ctx context.Context, parentBlockExecutionResultID flow.Identifier, block *entity.ExecutableBlock, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, derivedBlockData *derived.DerivedBlockData, ) ( *execution.ComputationResult, @@ -201,78 +196,107 @@ func (e *blockComputer) ExecuteBlock( return results, nil } -func (e *blockComputer) getRootSpanAndTransactions( - block *entity.ExecutableBlock, - derivedBlockData *derived.DerivedBlockData, -) ( - otelTrace.Span, - []transaction, - error, +func (e *blockComputer) queueTransactionRequests( + blockId flow.Identifier, + blockIdStr string, + blockHeader *flow.Header, + rawCollections []*entity.CompleteCollection, + systemTxnBody *flow.TransactionBody, + requestQueue chan TransactionRequest, ) { - rawCollections := block.Collections() - var transactions []transaction + txnIndex := uint32(0) - blockId := block.ID() - blockIdStr := blockId.String() - - blockCtx := fvm.NewContextFromParent( + collectionCtx := fvm.NewContextFromParent( e.vmCtx, - fvm.WithBlockHeader(block.Block.Header), - fvm.WithDerivedBlockData(derivedBlockData)) + fvm.WithBlockHeader(blockHeader), + // `protocol.Snapshot` implements `EntropyProvider` interface + // Note that `Snapshot` possible errors for RandomSource() are: + // - storage.ErrNotFound if the QC is unknown. + // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown + // However, at this stage, snapshot reference block should be known and the QC should also be known, + // so no error is expected in normal operations, as required by `EntropyProvider`. + fvm.WithEntropyProvider(e.protocolState.AtBlockID(blockId)), + ) - startTxnIndex := 0 for idx, collection := range rawCollections { - transactions = append( - transactions, - newTransactions( - collectionInfo{ - blockId: blockId, - blockIdStr: blockIdStr, - collectionIndex: idx, - CompleteCollection: collection, - isSystemTransaction: false, - }, - blockCtx, - startTxnIndex)...) - startTxnIndex += len(collection.Transactions) - } + collectionLogger := collectionCtx.Logger.With(). + Str("block_id", blockIdStr). + Uint64("height", blockHeader.Height). + Bool("system_chunk", false). + Bool("system_transaction", false). + Logger() + + collectionInfo := collectionInfo{ + blockId: blockId, + blockIdStr: blockIdStr, + collectionIndex: idx, + CompleteCollection: collection, + isSystemTransaction: false, + } + + for i, txnBody := range collection.Transactions { + requestQueue <- newTransactionRequest( + collectionInfo, + collectionCtx, + collectionLogger, + txnIndex, + txnBody, + i == len(collection.Transactions)-1) + txnIndex += 1 + } - systemTxn, err := blueprints.SystemChunkTransaction(e.vmCtx.Chain) - if err != nil { - return trace.NoopSpan, nil, fmt.Errorf( - "could not get system chunk transaction: %w", - err) } systemCtx := fvm.NewContextFromParent( e.systemChunkCtx, - fvm.WithBlockHeader(block.Block.Header), - fvm.WithDerivedBlockData(derivedBlockData)) - systemCollection := &entity.CompleteCollection{ - Transactions: []*flow.TransactionBody{systemTxn}, + fvm.WithBlockHeader(blockHeader), + // `protocol.Snapshot` implements `EntropyProvider` interface + // Note that `Snapshot` possible errors for RandomSource() are: + // - storage.ErrNotFound if the QC is unknown. + // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown + // However, at this stage, snapshot reference block should be known and the QC should also be known, + // so no error is expected in normal operations, as required by `EntropyProvider`. + fvm.WithEntropyProvider(e.protocolState.AtBlockID(blockId)), + ) + systemCollectionLogger := systemCtx.Logger.With(). + Str("block_id", blockIdStr). + Uint64("height", blockHeader.Height). + Bool("system_chunk", true). + Bool("system_transaction", true). + Logger() + systemCollectionInfo := collectionInfo{ + blockId: blockId, + blockIdStr: blockIdStr, + collectionIndex: len(rawCollections), + CompleteCollection: &entity.CompleteCollection{ + Transactions: []*flow.TransactionBody{systemTxnBody}, + }, + isSystemTransaction: true, + } + + requestQueue <- newTransactionRequest( + systemCollectionInfo, + systemCtx, + systemCollectionLogger, + txnIndex, + systemTxnBody, + true) +} + +func numberOfTransactionsInBlock(collections []*entity.CompleteCollection) int { + numTxns := 1 // there's one system transaction per block + for _, collection := range collections { + numTxns += len(collection.Transactions) } - transactions = append( - transactions, - newTransactions( - collectionInfo{ - blockId: blockId, - blockIdStr: blockIdStr, - collectionIndex: len(rawCollections), - CompleteCollection: systemCollection, - isSystemTransaction: true, - }, - systemCtx, - startTxnIndex)...) - - return e.tracer.BlockRootSpan(blockId), transactions, nil + return numTxns } func (e *blockComputer) executeBlock( ctx context.Context, parentBlockExecutionResultID flow.Identifier, block *entity.ExecutableBlock, - baseSnapshot state.StorageSnapshot, + baseSnapshot snapshot.StorageSnapshot, derivedBlockData *derived.DerivedBlockData, ) ( *execution.ComputationResult, @@ -283,19 +307,28 @@ func (e *blockComputer) executeBlock( return nil, fmt.Errorf("executable block start state is not set") } - rootSpan, transactions, err := e.getRootSpanAndTransactions( - block, - derivedBlockData) - if err != nil { - return nil, err - } + blockId := block.ID() + blockIdStr := blockId.String() - blockSpan := e.tracer.StartSpanFromParent(rootSpan, trace.EXEComputeBlock) + rawCollections := block.Collections() + + blockSpan := e.tracer.StartSpanFromParent( + e.tracer.BlockRootSpan(blockId), + trace.EXEComputeBlock) blockSpan.SetAttributes( - attribute.String("block_id", block.ID().String()), - attribute.Int("collection_counts", len(block.CompleteCollections))) + attribute.String("block_id", blockIdStr), + attribute.Int("collection_counts", len(rawCollections))) defer blockSpan.End() + systemTxn, err := blueprints.SystemChunkTransaction(e.vmCtx.Chain) + if err != nil { + return nil, fmt.Errorf( + "could not get system chunk transaction: %w", + err) + } + + numTxns := numberOfTransactionsInBlock(rawCollections) + collector := newResultCollector( e.tracer, blockSpan, @@ -307,32 +340,43 @@ func (e *blockComputer) executeBlock( e.receiptHasher, parentBlockExecutionResultID, block, - len(transactions), + numTxns, e.colResCons) defer collector.Stop() - snapshotTree := storage.NewSnapshotTree(baseSnapshot) - for _, txn := range transactions { - txnExecutionSnapshot, output, err := e.executeTransaction( + requestQueue := make(chan TransactionRequest, numTxns) + + database := newTransactionCoordinator( + e.vm, + baseSnapshot, + derivedBlockData, + collector) + + e.queueTransactionRequests( + blockId, + blockIdStr, + block.Block.Header, + rawCollections, + systemTxn, + requestQueue) + close(requestQueue) + + wg := &sync.WaitGroup{} + wg.Add(e.maxConcurrency) + + for i := 0; i < e.maxConcurrency; i++ { + go e.executeTransactions( blockSpan, - txn, - snapshotTree, - collector) - if err != nil { - prefix := "" - if txn.isSystemTransaction { - prefix = "system " - } + database, + requestQueue, + wg) + } - return nil, fmt.Errorf( - "failed to execute %stransaction at txnIndex %v: %w", - prefix, - txn.txnIndex, - err) - } + wg.Wait() - collector.AddTransactionResult(txn, txnExecutionSnapshot, output) - snapshotTree = snapshotTree.Append(txnExecutionSnapshot) + err = database.Error() + if err != nil { + return nil, err } res, err := collector.Finalize(ctx) @@ -349,97 +393,145 @@ func (e *blockComputer) executeBlock( return res, nil } +func (e *blockComputer) executeTransactions( + blockSpan otelTrace.Span, + database *transactionCoordinator, + requestQueue chan TransactionRequest, + wg *sync.WaitGroup, +) { + defer wg.Done() + + for request := range requestQueue { + attempt := 0 + for { + request.ctx.Logger.Info(). + Int("attempt", attempt). + Msg("executing transaction") + + attempt += 1 + err := e.executeTransaction(blockSpan, database, request, attempt) + + if errors.IsRetryableConflictError(err) { + request.ctx.Logger.Info(). + Int("attempt", attempt). + Str("conflict_error", err.Error()). + Msg("conflict detected. retrying transaction") + continue + } + + if err != nil { + database.AbortAllOutstandingTransactions(err) + return + } + + break // process next transaction + } + } +} + func (e *blockComputer) executeTransaction( - parentSpan otelTrace.Span, - txn transaction, - storageSnapshot state.StorageSnapshot, - collector *resultCollector, + blockSpan otelTrace.Span, + database *transactionCoordinator, + request TransactionRequest, + attempt int, +) error { + txn, err := e.executeTransactionInternal( + blockSpan, + database, + request, + attempt) + if err != nil { + prefix := "" + if request.isSystemTransaction { + prefix = "system " + } + + snapshotTime := logical.Time(0) + if txn != nil { + snapshotTime = txn.SnapshotTime() + } + + return fmt.Errorf( + "failed to execute %stransaction %v (%d@%d) for block %s "+ + "at height %v: %w", + prefix, + request.txnIdStr, + request.txnIndex, + snapshotTime, + request.blockIdStr, + request.ctx.BlockHeader.Height, + err) + } + + return nil +} + +func (e *blockComputer) executeTransactionInternal( + blockSpan otelTrace.Span, + database *transactionCoordinator, + request TransactionRequest, + attempt int, ) ( - *state.ExecutionSnapshot, - fvm.ProcedureOutput, + *transaction, error, ) { - startedAt := time.Now() - memAllocBefore := debug.GetHeapAllocsBytes() - txSpan := e.tracer.StartSampledSpanFromParent( - parentSpan, - txn.txnId, + blockSpan, + request.txnId, trace.EXEComputeTransaction) txSpan.SetAttributes( - attribute.String("tx_id", txn.txnIdStr), - attribute.Int64("tx_index", int64(txn.txnIndex)), - attribute.Int("col_index", txn.collectionIndex), + attribute.String("tx_id", request.txnIdStr), + attribute.Int64("tx_index", int64(request.txnIndex)), + attribute.Int("col_index", request.collectionIndex), ) defer txSpan.End() - logger := e.log.With(). - Str("tx_id", txn.txnIdStr). - Uint32("tx_index", txn.txnIndex). - Str("block_id", txn.blockIdStr). - Uint64("height", txn.ctx.BlockHeader.Height). - Bool("system_chunk", txn.isSystemTransaction). - Bool("system_transaction", txn.isSystemTransaction). - Logger() - logger.Info().Msg("executing transaction in fvm") + request.ctx = fvm.NewContextFromParent(request.ctx, fvm.WithSpan(txSpan)) - txn.ctx = fvm.NewContextFromParent(txn.ctx, fvm.WithSpan(txSpan)) + txn, err := database.NewTransaction(request, attempt) + if err != nil { + return nil, err + } + defer txn.Cleanup() - executionSnapshot, output, err := e.vm.Run( - txn.ctx, - txn.TransactionProcedure, - storageSnapshot) + err = txn.Preprocess() if err != nil { - return nil, fvm.ProcedureOutput{}, fmt.Errorf( - "failed to execute transaction %v for block %s at height %v: %w", - txn.txnIdStr, - txn.blockIdStr, - txn.ctx.BlockHeader.Height, - err) + return txn, err } - postProcessSpan := e.tracer.StartSpanFromParent(txSpan, trace.EXEPostProcessTransaction) - defer postProcessSpan.End() + // Validating here gives us an opportunity to early abort/retry the + // transaction in case the conflict is detectable after preprocessing. + // This is strictly an optimization and hence we don't need to wait for + // updates (removing this validate call won't impact correctness). + err = txn.Validate() + if err != nil { + return txn, err + } - memAllocAfter := debug.GetHeapAllocsBytes() + err = txn.Execute() + if err != nil { + return txn, err + } - logger = logger.With(). - Uint64("computation_used", output.ComputationUsed). - Uint64("memory_used", output.MemoryEstimate). - Uint64("mem_alloc", memAllocAfter-memAllocBefore). - Int64("time_spent_in_ms", time.Since(startedAt).Milliseconds()). - Logger() + err = txn.Finalize() + if err != nil { + return txn, err + } - if output.Err != nil { - logger = logger.With(). - Str("error_message", output.Err.Error()). - Uint16("error_code", uint16(output.Err.Code())). - Logger() - logger.Info().Msg("transaction execution failed") - - if txn.isSystemTransaction { - // This log is used as the data source for an alert on grafana. - // The system_chunk_error field must not be changed without adding - // the corresponding changes in grafana. - // https://github.com/dapperlabs/flow-internal/issues/1546 - logger.Error(). - Bool("system_chunk_error", true). - Bool("system_transaction_error", true). - Bool("critical_error", true). - Msg("error executing system chunk transaction") + // Snapshot time smaller than execution time indicates there are outstanding + // transaction(s) that must be committed before this transaction can be + // committed. + for txn.SnapshotTime() < request.ExecutionTime() { + err = txn.WaitForUpdates() + if err != nil { + return txn, err + } + + err = txn.Validate() + if err != nil { + return txn, err } - } else { - logger.Info().Msg("transaction executed successfully") } - e.metrics.ExecutionTransactionExecuted( - time.Since(startedAt), - output.ComputationUsed, - output.MemoryEstimate, - memAllocAfter-memAllocBefore, - len(output.Events), - flow.EventsList(output.Events).ByteSize(), - output.Err != nil, - ) - return executionSnapshot, output, nil + return txn, txn.Commit() } diff --git a/engine/execution/computation/computer/computer_test.go b/engine/execution/computation/computer/computer_test.go index 4f5889a2853..3f025fa3646 100644 --- a/engine/execution/computation/computer/computer_test.go +++ b/engine/execution/computation/computer/computer_test.go @@ -4,10 +4,11 @@ import ( "context" "fmt" "math/rand" + "sync/atomic" "testing" "github.com/onflow/cadence" - "github.com/onflow/cadence/encoding/json" + "github.com/onflow/cadence/encoding/ccf" "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" @@ -26,16 +27,17 @@ import ( "github.com/onflow/flow-go/engine/execution/computation/committer" "github.com/onflow/flow-go/engine/execution/computation/computer" computermock "github.com/onflow/flow-go/engine/execution/computation/computer/mock" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/fvm/environment" fvmErrors "github.com/onflow/flow-go/fvm/errors" fvmmock "github.com/onflow/flow-go/fvm/mock" reusableRuntime "github.com/onflow/flow-go/fvm/runtime" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" @@ -51,6 +53,10 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) +const ( + testMaxConcurrency = 2 +) + func incStateCommitment(startState flow.StateCommitment) flow.StateCommitment { endState := flow.StateCommitment(startState) endState[0] += 1 @@ -62,7 +68,7 @@ type fakeCommitter struct { } func (committer *fakeCommitter) CommitView( - view *state.ExecutionSnapshot, + view *snapshot.ExecutionSnapshot, startState flow.StateCommitment, ) ( flow.StateCommitment, @@ -96,9 +102,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { t.Run("single collection", func(t *testing.T) { - execCtx := fvm.NewContext( - fvm.WithDerivedBlockData(derived.NewEmptyDerivedBlockData()), - ) + execCtx := fvm.NewContext() vm := &testVM{ t: t, @@ -124,9 +128,9 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { exemetrics.On("ExecutionTransactionExecuted", mock.Anything, // duration + mock.Anything, // conflict retry count mock.Anything, // computation used mock.Anything, // memory used - mock.Anything, // actual memory used mock.Anything, // number of events mock.Anything, // size of events false). // no failure @@ -167,7 +171,9 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer, me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) // create a block with 1 collection with 2 transactions @@ -179,9 +185,9 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { parentBlockExecutionResultID, block, nil, - derived.NewEmptyDerivedBlockData()) + derived.NewEmptyDerivedBlockData(0)) assert.NoError(t, err) - assert.Len(t, result.StateSnapshots, 1+1) // +1 system chunk + assert.Len(t, result.AllExecutionSnapshots(), 1+1) // +1 system chunk require.Equal(t, 2, committer.callCount) @@ -190,7 +196,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { expectedChunk1EndState := incStateCommitment(*block.StartState) expectedChunk2EndState := incStateCommitment(expectedChunk1EndState) - assert.Equal(t, expectedChunk2EndState, result.EndState) + assert.Equal(t, expectedChunk2EndState, result.CurrentEndState()) assertEventHashesMatch(t, 1+1, result) @@ -209,10 +215,11 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { chunk1 := receipt.Chunks[0] + eventCommits := result.AllEventCommitments() assert.Equal(t, block.ID(), chunk1.BlockID) assert.Equal(t, uint(0), chunk1.CollectionIndex) assert.Equal(t, uint64(2), chunk1.NumberOfTransactions) - assert.Equal(t, result.EventsHashes[0], chunk1.EventCollection) + assert.Equal(t, eventCommits[0], chunk1.EventCollection) assert.Equal(t, *block.StartState, chunk1.StartState) @@ -224,7 +231,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { assert.Equal(t, block.ID(), chunk2.BlockID) assert.Equal(t, uint(1), chunk2.CollectionIndex) assert.Equal(t, uint64(1), chunk2.NumberOfTransactions) - assert.Equal(t, result.EventsHashes[1], chunk2.EventCollection) + assert.Equal(t, eventCommits[1], chunk2.EventCollection) assert.Equal(t, expectedChunk1EndState, chunk2.StartState) @@ -235,16 +242,17 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { // Verify ChunkDataPacks - assert.Len(t, result.ChunkDataPacks, 1+1) // +1 system chunk + chunkDataPacks := result.AllChunkDataPacks() + assert.Len(t, chunkDataPacks, 1+1) // +1 system chunk - chunkDataPack1 := result.ChunkDataPacks[0] + chunkDataPack1 := chunkDataPacks[0] assert.Equal(t, chunk1.ID(), chunkDataPack1.ChunkID) assert.Equal(t, *block.StartState, chunkDataPack1.StartState) assert.Equal(t, []byte{1}, chunkDataPack1.Proof) assert.NotNil(t, chunkDataPack1.Collection) - chunkDataPack2 := result.ChunkDataPacks[1] + chunkDataPack2 := chunkDataPacks[1] assert.Equal(t, chunk2.ID(), chunkDataPack2.ChunkID) assert.Equal(t, chunk2.StartState, chunkDataPack2.StartState) @@ -268,7 +276,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { assert.NotNil(t, chunkExecutionData2.TrieUpdate) assert.Equal(t, byte(2), chunkExecutionData2.TrieUpdate.RootHash[0]) - assert.Equal(t, 3, vm.callCount) + assert.Equal(t, 3, vm.CallCount()) }) t.Run("empty block still computes system chunk", func(t *testing.T) { @@ -298,18 +306,17 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer, me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) // create an empty block block := generateBlock(0, 0, rag) - derivedBlockData := derived.NewEmptyDerivedBlockData() + derivedBlockData := derived.NewEmptyDerivedBlockData(0) - vm.On("Run", mock.Anything, mock.Anything, mock.Anything). - Return( - &state.ExecutionSnapshot{}, - fvm.ProcedureOutput{}, - nil). + vm.On("NewExecutor", mock.Anything, mock.Anything, mock.Anything). + Return(noOpExecutor{}). Once() // just system chunk committer.On("CommitView", mock.Anything, mock.Anything). @@ -323,8 +330,8 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { nil, derivedBlockData) assert.NoError(t, err) - assert.Len(t, result.StateSnapshots, 1) - assert.Len(t, result.TransactionResults, 1) + assert.Len(t, result.AllExecutionSnapshots(), 1) + assert.Len(t, result.AllTransactionResults(), 1) assert.Len(t, result.ChunkExecutionDatas, 1) assertEventHashesMatch(t, 1, result) @@ -353,7 +360,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { chain := flow.Localnet.Chain() vm := fvm.NewVirtualMachine() - derivedBlockData := derived.NewEmptyDerivedBlockData() + derivedBlockData := derived.NewEmptyDerivedBlockData(0) baseOpts := []fvm.Option{ fvm.WithChain(chain), fvm.WithDerivedBlockData(derivedBlockData), @@ -361,7 +368,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { opts := append(baseOpts, contextOptions...) ctx := fvm.NewContext(opts...) - snapshotTree := storage.NewSnapshotTree(nil) + snapshotTree := snapshot.NewSnapshotTree(nil) baseBootstrapOpts := []fvm.BootstrapProcedureOption{ fvm.WithInitialTokenSupply(unittest.GenesisTokenSupply), @@ -397,7 +404,9 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { comm, me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) // create an empty block @@ -412,13 +421,13 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { unittest.IdentifierFixture(), block, snapshotTree, - derivedBlockData) + derivedBlockData.NewChildDerivedBlockData()) assert.NoError(t, err) - assert.Len(t, result.StateSnapshots, 1) - assert.Len(t, result.TransactionResults, 1) + assert.Len(t, result.AllExecutionSnapshots(), 1) + assert.Len(t, result.AllTransactionResults(), 1) assert.Len(t, result.ChunkExecutionDatas, 1) - assert.Empty(t, result.TransactionResults[0].ErrorMessage) + assert.Empty(t, result.AllTransactionResults()[0].ErrorMessage) }) t.Run("multiple collections", func(t *testing.T) { @@ -455,7 +464,9 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer, me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) collectionCount := 2 @@ -466,7 +477,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { // create a block with 2 collections with 2 transactions each block := generateBlock(collectionCount, transactionsPerCollection, rag) - derivedBlockData := derived.NewEmptyDerivedBlockData() + derivedBlockData := derived.NewEmptyDerivedBlockData(0) committer.On("CommitView", mock.Anything, mock.Anything). Return(nil, nil, nil, nil). @@ -481,26 +492,24 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { assert.NoError(t, err) // chunk count should match collection count - assert.Len(t, result.StateSnapshots, collectionCount+1) // system chunk + assert.Equal(t, result.BlockExecutionResult.Size(), collectionCount+1) // system chunk // all events should have been collected - assert.Len(t, result.Events, collectionCount+1) - for i := 0; i < collectionCount; i++ { - assert.Len(t, result.Events[i], eventsPerCollection) + events := result.CollectionExecutionResultAt(i).Events() + assert.Len(t, events, eventsPerCollection) } - assert.Len(t, result.Events[len(result.Events)-1], eventsPerTransaction) + // system chunk + assert.Len(t, result.CollectionExecutionResultAt(collectionCount).Events(), eventsPerTransaction) + + events := result.AllEvents() // events should have been indexed by transaction and event k := 0 for expectedTxIndex := 0; expectedTxIndex < totalTransactionCount; expectedTxIndex++ { for expectedEventIndex := 0; expectedEventIndex < eventsPerTransaction; expectedEventIndex++ { - - chunkIndex := k / eventsPerCollection - eventIndex := k % eventsPerCollection - - e := result.Events[chunkIndex][eventIndex] + e := events[k] assert.EqualValues(t, expectedEventIndex, int(e.EventIndex)) assert.EqualValues(t, expectedTxIndex, e.TransactionIndex) k++ @@ -519,141 +528,203 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { expectedResults = append(expectedResults, txResult) } } - assert.ElementsMatch(t, expectedResults, result.TransactionResults[0:len(result.TransactionResults)-1]) // strip system chunk + txResults := result.AllTransactionResults() + assert.ElementsMatch(t, expectedResults, txResults[0:len(txResults)-1]) // strip system chunk assertEventHashesMatch(t, collectionCount+1, result) - assert.Equal(t, totalTransactionCount, vm.callCount) + assert.Equal(t, totalTransactionCount, vm.CallCount()) }) - t.Run("service events are emitted", func(t *testing.T) { - execCtx := fvm.NewContext( - fvm.WithServiceEventCollectionEnabled(), - fvm.WithAuthorizationChecksEnabled(false), - fvm.WithSequenceNumberCheckAndIncrementEnabled(false), - ) - - collectionCount := 2 - transactionsPerCollection := 2 + t.Run( + "service events are emitted", func(t *testing.T) { + execCtx := fvm.NewContext( + fvm.WithServiceEventCollectionEnabled(), + fvm.WithAuthorizationChecksEnabled(false), + fvm.WithSequenceNumberCheckAndIncrementEnabled(false), + ) - totalTransactionCount := (collectionCount * transactionsPerCollection) + 1 // +1 for system chunk + collectionCount := 2 + transactionsPerCollection := 2 - // create a block with 2 collections with 2 transactions each - block := generateBlock(collectionCount, transactionsPerCollection, rag) + // create a block with 2 collections with 2 transactions each + block := generateBlock(collectionCount, transactionsPerCollection, rag) - ordinaryEvent := cadence.Event{ - EventType: &cadence.EventType{ - Location: stdlib.FlowLocation{}, - QualifiedIdentifier: "what.ever", - }, - } + serviceEvents, err := systemcontracts.ServiceEventsForChain(execCtx.Chain.ChainID()) + require.NoError(t, err) - serviceEvents, err := systemcontracts.ServiceEventsForChain(execCtx.Chain.ChainID()) - require.NoError(t, err) + payload, err := ccf.Decode(nil, unittest.EpochSetupFixtureCCF) + require.NoError(t, err) - payload, err := json.Decode(nil, []byte(unittest.EpochSetupFixtureJSON)) - require.NoError(t, err) - - serviceEventA, ok := payload.(cadence.Event) - require.True(t, ok) + serviceEventA, ok := payload.(cadence.Event) + require.True(t, ok) - serviceEventA.EventType.Location = common.AddressLocation{ - Address: common.Address(serviceEvents.EpochSetup.Address), - } - serviceEventA.EventType.QualifiedIdentifier = serviceEvents.EpochSetup.QualifiedIdentifier() + serviceEventA.EventType.Location = common.AddressLocation{ + Address: common.Address(serviceEvents.EpochSetup.Address), + } + serviceEventA.EventType.QualifiedIdentifier = serviceEvents.EpochSetup.QualifiedIdentifier() - payload, err = json.Decode(nil, []byte(unittest.EpochCommitFixtureJSON)) - require.NoError(t, err) + payload, err = ccf.Decode(nil, unittest.EpochCommitFixtureCCF) + require.NoError(t, err) - serviceEventB, ok := payload.(cadence.Event) - require.True(t, ok) + serviceEventB, ok := payload.(cadence.Event) + require.True(t, ok) - serviceEventB.EventType.Location = common.AddressLocation{ - Address: common.Address(serviceEvents.EpochCommit.Address), - } - serviceEventB.EventType.QualifiedIdentifier = serviceEvents.EpochCommit.QualifiedIdentifier() - - // events to emit for each iteration/transaction - events := make([][]cadence.Event, totalTransactionCount) - events[0] = nil - events[1] = []cadence.Event{serviceEventA, ordinaryEvent} - events[2] = []cadence.Event{ordinaryEvent} - events[3] = nil - events[4] = []cadence.Event{serviceEventB} - - emittingRuntime := &testRuntime{ - executeTransaction: func(script runtime.Script, context runtime.Context) error { - for _, e := range events[0] { - err := context.Interface.EmitEvent(e) - if err != nil { - return err - } - } - events = events[1:] - return nil - }, - readStored: func(address common.Address, path cadence.Path, r runtime.Context) (cadence.Value, error) { - return nil, nil - }, - } + serviceEventB.EventType.Location = common.AddressLocation{ + Address: common.Address(serviceEvents.EpochCommit.Address), + } + serviceEventB.EventType.QualifiedIdentifier = serviceEvents.EpochCommit.QualifiedIdentifier() - execCtx = fvm.NewContextFromParent( - execCtx, - fvm.WithReusableCadenceRuntimePool( - reusableRuntime.NewCustomReusableCadenceRuntimePool( - 0, - runtime.Config{}, - func(_ runtime.Config) runtime.Runtime { - return emittingRuntime - }))) + payload, err = ccf.Decode(nil, unittest.VersionBeaconFixtureCCF) + require.NoError(t, err) - vm := fvm.NewVirtualMachine() + serviceEventC, ok := payload.(cadence.Event) + require.True(t, ok) - bservice := requesterunit.MockBlobService(blockstore.NewBlockstore(dssync.MutexWrap(datastore.NewMapDatastore()))) - trackerStorage := mocktracker.NewMockStorage() + serviceEventC.EventType.Location = common.AddressLocation{ + Address: common.Address(serviceEvents.VersionBeacon.Address), + } + serviceEventC.EventType.QualifiedIdentifier = serviceEvents.VersionBeacon.QualifiedIdentifier() - prov := provider.NewProvider( - zerolog.Nop(), - metrics.NewNoopCollector(), - execution_data.DefaultSerializer, - bservice, - trackerStorage, - ) + transactions := []*flow.TransactionBody{} + for _, col := range block.Collections() { + transactions = append(transactions, col.Transactions...) + } - exe, err := computer.NewBlockComputer( - vm, - execCtx, - metrics.NewNoopCollector(), - trace.NewNoopTracer(), - zerolog.Nop(), - committer.NewNoopViewCommitter(), - me, - prov, - nil) - require.NoError(t, err) + // events to emit for each iteration/transaction + events := map[common.Location][]cadence.Event{ + common.TransactionLocation(transactions[0].ID()): nil, + common.TransactionLocation(transactions[1].ID()): []cadence.Event{ + serviceEventA, + cadence.Event{ + EventType: &cadence.EventType{ + Location: stdlib.FlowLocation{}, + QualifiedIdentifier: "what.ever", + }, + }, + }, + common.TransactionLocation(transactions[2].ID()): []cadence.Event{ + cadence.Event{ + EventType: &cadence.EventType{ + Location: stdlib.FlowLocation{}, + QualifiedIdentifier: "what.ever", + }, + }, + }, + common.TransactionLocation(transactions[3].ID()): nil, + } - result, err := exe.ExecuteBlock( - context.Background(), - unittest.IdentifierFixture(), - block, - nil, - derived.NewEmptyDerivedBlockData()) - require.NoError(t, err) + systemTransactionEvents := []cadence.Event{ + serviceEventB, + serviceEventC, + } - // make sure event index sequence are valid - for _, eventsList := range result.Events { - unittest.EnsureEventsIndexSeq(t, eventsList, execCtx.Chain.ChainID()) - } + emittingRuntime := &testRuntime{ + executeTransaction: func( + script runtime.Script, + context runtime.Context, + ) error { + scriptEvents, ok := events[context.Location] + if !ok { + scriptEvents = systemTransactionEvents + } - // all events should have been collected - require.Len(t, result.ServiceEvents, 2) + for _, e := range scriptEvents { + err := context.Interface.EmitEvent(e) + if err != nil { + return err + } + } + return nil + }, + readStored: func( + address common.Address, + path cadence.Path, + r runtime.Context, + ) (cadence.Value, error) { + return nil, nil + }, + } - // events are ordered - require.Equal(t, serviceEventA.EventType.ID(), string(result.ServiceEvents[0].Type)) - require.Equal(t, serviceEventB.EventType.ID(), string(result.ServiceEvents[1].Type)) + execCtx = fvm.NewContextFromParent( + execCtx, + fvm.WithReusableCadenceRuntimePool( + reusableRuntime.NewCustomReusableCadenceRuntimePool( + 0, + runtime.Config{}, + func(_ runtime.Config) runtime.Runtime { + return emittingRuntime + }, + ), + ), + ) + + vm := fvm.NewVirtualMachine() + + bservice := requesterunit.MockBlobService(blockstore.NewBlockstore(dssync.MutexWrap(datastore.NewMapDatastore()))) + trackerStorage := mocktracker.NewMockStorage() + + prov := provider.NewProvider( + zerolog.Nop(), + metrics.NewNoopCollector(), + execution_data.DefaultSerializer, + bservice, + trackerStorage, + ) + + exe, err := computer.NewBlockComputer( + vm, + execCtx, + metrics.NewNoopCollector(), + trace.NewNoopTracer(), + zerolog.Nop(), + committer.NewNoopViewCommitter(), + me, + prov, + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) + require.NoError(t, err) + + result, err := exe.ExecuteBlock( + context.Background(), + unittest.IdentifierFixture(), + block, + nil, + derived.NewEmptyDerivedBlockData(0), + ) + require.NoError(t, err) + + // make sure event index sequence are valid + for i := 0; i < result.BlockExecutionResult.Size(); i++ { + collectionResult := result.CollectionExecutionResultAt(i) + unittest.EnsureEventsIndexSeq(t, collectionResult.Events(), execCtx.Chain.ChainID()) + } - assertEventHashesMatch(t, collectionCount+1, result) - }) + sEvents := result.AllServiceEvents() // all events should have been collected + require.Len(t, sEvents, 3) + + // events are ordered + require.Equal( + t, + serviceEventA.EventType.ID(), + string(sEvents[0].Type), + ) + require.Equal( + t, + serviceEventB.EventType.ID(), + string(sEvents[1].Type), + ) + + require.Equal( + t, + serviceEventC.EventType.ID(), + string(sEvents[2].Type), + ) + + assertEventHashesMatch(t, collectionCount+1, result) + }, + ) t.Run("succeeding transactions store programs", func(t *testing.T) { @@ -680,7 +751,11 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { return nil }, - readStored: func(address common.Address, path cadence.Path, r runtime.Context) (cadence.Value, error) { + readStored: func( + address common.Address, + path cadence.Path, + r runtime.Context, + ) (cadence.Value, error) { return nil, nil }, } @@ -717,7 +792,9 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) const collectionCount = 2 @@ -732,10 +809,10 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { context.Background(), unittest.IdentifierFixture(), block, - state.MapStorageSnapshot{key: value}, - derived.NewEmptyDerivedBlockData()) + snapshot.MapStorageSnapshot{key: value}, + derived.NewEmptyDerivedBlockData(0)) assert.NoError(t, err) - assert.Len(t, result.StateSnapshots, collectionCount+1) // +1 system chunk + assert.Len(t, result.AllExecutionSnapshots(), collectionCount+1) // +1 system chunk }) t.Run("failing transactions do not store programs", func(t *testing.T) { @@ -755,15 +832,21 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { const collectionCount = 2 const transactionCount = 2 + block := generateBlock(collectionCount, transactionCount, rag) - var executionCalls int + normalTransactions := map[common.Location]struct{}{} + for _, col := range block.Collections() { + for _, txn := range col.Transactions { + loc := common.TransactionLocation(txn.ID()) + normalTransactions[loc] = struct{}{} + } + } rt := &testRuntime{ executeTransaction: func(script runtime.Script, r runtime.Context) error { - executionCalls++ - - // NOTE: set a program and revert all transactions but the system chunk transaction + // NOTE: set a program and revert all transactions but the + // system chunk transaction _, err := r.Interface.GetOrLoadProgram( contractLocation, func() (*interpreter.Program, error) { @@ -772,15 +855,20 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { ) require.NoError(t, err) - if executionCalls > collectionCount*transactionCount { - return nil + _, ok := normalTransactions[r.Location] + if ok { + return runtime.Error{ + Err: fmt.Errorf("TX reverted"), + } } - return runtime.Error{ - Err: fmt.Errorf("TX reverted"), - } + return nil }, - readStored: func(address common.Address, path cadence.Path, r runtime.Context) (cadence.Value, error) { + readStored: func( + address common.Address, + path cadence.Path, + r runtime.Context, + ) (cadence.Value, error) { return nil, nil }, } @@ -817,11 +905,11 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) - block := generateBlock(collectionCount, transactionCount, rag) - key := flow.AccountStatusRegisterID( flow.BytesToAddress(address.Bytes())) value := environment.NewAccountStatus().ToBytes() @@ -830,23 +918,77 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { context.Background(), unittest.IdentifierFixture(), block, - state.MapStorageSnapshot{key: value}, - derived.NewEmptyDerivedBlockData()) + snapshot.MapStorageSnapshot{key: value}, + derived.NewEmptyDerivedBlockData(0)) require.NoError(t, err) - assert.Len(t, result.StateSnapshots, collectionCount+1) // +1 system chunk + assert.Len(t, result.AllExecutionSnapshots(), collectionCount+1) // +1 system chunk }) -} -func assertEventHashesMatch(t *testing.T, expectedNoOfChunks int, result *execution.ComputationResult) { + t.Run("internal error", func(t *testing.T) { + execCtx := fvm.NewContext() - require.Len(t, result.Events, expectedNoOfChunks) - require.Len(t, result.EventsHashes, expectedNoOfChunks) + committer := new(computermock.ViewCommitter) - for i := 0; i < expectedNoOfChunks; i++ { - calculatedHash, err := flow.EventsMerkleRootHash(result.Events[i]) + bservice := requesterunit.MockBlobService( + blockstore.NewBlockstore( + dssync.MutexWrap(datastore.NewMapDatastore()))) + trackerStorage := mocktracker.NewMockStorage() + + prov := provider.NewProvider( + zerolog.Nop(), + metrics.NewNoopCollector(), + execution_data.DefaultSerializer, + bservice, + trackerStorage) + + exe, err := computer.NewBlockComputer( + errorVM{errorAt: 5}, + execCtx, + metrics.NewNoopCollector(), + trace.NewNoopTracer(), + zerolog.Nop(), + committer, + me, + prov, + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) - require.Equal(t, calculatedHash, result.EventsHashes[i]) + collectionCount := 5 + transactionsPerCollection := 3 + block := generateBlock(collectionCount, transactionsPerCollection, rag) + + committer.On("CommitView", mock.Anything, mock.Anything). + Return(nil, nil, nil, nil). + Times(collectionCount + 1) + + _, err = exe.ExecuteBlock( + context.Background(), + unittest.IdentifierFixture(), + block, + nil, + derived.NewEmptyDerivedBlockData(0)) + assert.ErrorContains(t, err, "boom - internal error") + }) + +} + +func assertEventHashesMatch( + t *testing.T, + expectedNoOfChunks int, + result *execution.ComputationResult, +) { + execResSize := result.BlockExecutionResult.Size() + attestResSize := result.BlockAttestationResult.Size() + require.Equal(t, execResSize, expectedNoOfChunks) + require.Equal(t, execResSize, attestResSize) + + for i := 0; i < expectedNoOfChunks; i++ { + events := result.CollectionExecutionResultAt(i).Events() + calculatedHash, err := flow.EventsMerkleRootHash(events) + require.NoError(t, err) + require.Equal(t, calculatedHash, result.CollectionAttestationResultAt(i).EventCommitment()) } } @@ -873,7 +1015,10 @@ func (executor *testTransactionExecutor) Result() (cadence.Value, error) { type testRuntime struct { executeScript func(runtime.Script, runtime.Context) (cadence.Value, error) executeTransaction func(runtime.Script, runtime.Context) error - readStored func(common.Address, cadence.Path, runtime.Context) (cadence.Value, error) + readStored func(common.Address, cadence.Path, runtime.Context) ( + cadence.Value, + error, + ) } var _ runtime.Runtime = &testRuntime{} @@ -882,11 +1027,17 @@ func (e *testRuntime) Config() runtime.Config { panic("Config not expected") } -func (e *testRuntime) NewScriptExecutor(script runtime.Script, c runtime.Context) runtime.Executor { +func (e *testRuntime) NewScriptExecutor( + script runtime.Script, + c runtime.Context, +) runtime.Executor { panic("NewScriptExecutor not expected") } -func (e *testRuntime) NewTransactionExecutor(script runtime.Script, c runtime.Context) runtime.Executor { +func (e *testRuntime) NewTransactionExecutor( + script runtime.Script, + c runtime.Context, +) runtime.Executor { return &testTransactionExecutor{ executeTransaction: e.executeTransaction, script: script, @@ -894,7 +1045,13 @@ func (e *testRuntime) NewTransactionExecutor(script runtime.Script, c runtime.Co } } -func (e *testRuntime) NewContractFunctionExecutor(contractLocation common.AddressLocation, functionName string, arguments []cadence.Value, argumentTypes []sema.Type, context runtime.Context) runtime.Executor { +func (e *testRuntime) NewContractFunctionExecutor( + contractLocation common.AddressLocation, + functionName string, + arguments []cadence.Value, + argumentTypes []sema.Type, + context runtime.Context, +) runtime.Executor { panic("NewContractFunctionExecutor not expected") } @@ -910,19 +1067,34 @@ func (e *testRuntime) SetResourceOwnerChangeHandlerEnabled(_ bool) { panic("SetResourceOwnerChangeHandlerEnabled not expected") } -func (e *testRuntime) InvokeContractFunction(_ common.AddressLocation, _ string, _ []cadence.Value, _ []sema.Type, _ runtime.Context) (cadence.Value, error) { +func (e *testRuntime) InvokeContractFunction( + _ common.AddressLocation, + _ string, + _ []cadence.Value, + _ []sema.Type, + _ runtime.Context, +) (cadence.Value, error) { panic("InvokeContractFunction not expected") } -func (e *testRuntime) ExecuteScript(script runtime.Script, context runtime.Context) (cadence.Value, error) { +func (e *testRuntime) ExecuteScript( + script runtime.Script, + context runtime.Context, +) (cadence.Value, error) { return e.executeScript(script, context) } -func (e *testRuntime) ExecuteTransaction(script runtime.Script, context runtime.Context) error { +func (e *testRuntime) ExecuteTransaction( + script runtime.Script, + context runtime.Context, +) error { return e.executeTransaction(script, context) } -func (*testRuntime) ParseAndCheckProgram(_ []byte, _ runtime.Context) (*interpreter.Program, error) { +func (*testRuntime) ParseAndCheckProgram( + _ []byte, + _ runtime.Context, +) (*interpreter.Program, error) { panic("ParseAndCheckProgram not expected") } @@ -938,11 +1110,19 @@ func (*testRuntime) SetAtreeValidationEnabled(_ bool) { panic("SetAtreeValidationEnabled not expected") } -func (e *testRuntime) ReadStored(a common.Address, p cadence.Path, c runtime.Context) (cadence.Value, error) { +func (e *testRuntime) ReadStored( + a common.Address, + p cadence.Path, + c runtime.Context, +) (cadence.Value, error) { return e.readStored(a, p, c) } -func (*testRuntime) ReadLinked(_ common.Address, _ cadence.Path, _ runtime.Context) (cadence.Value, error) { +func (*testRuntime) ReadLinked( + _ common.Address, + _ cadence.Path, + _ runtime.Context, +) (cadence.Value, error) { panic("ReadLinked not expected") } @@ -968,7 +1148,11 @@ func (r *RandomAddressGenerator) AddressCount() uint64 { panic("not implemented") } -func (testRuntime) Storage(runtime.Context) (*runtime.Storage, *interpreter.Interpreter, error) { +func (testRuntime) Storage(runtime.Context) ( + *runtime.Storage, + *interpreter.Interpreter, + error, +) { panic("Storage not expected") } @@ -1012,8 +1196,8 @@ func Test_ExecutingSystemCollection(t *testing.T) { noopCollector := metrics.NewNoopCollector() - expectedNumberOfEvents := 2 - expectedEventSize := 911 + expectedNumberOfEvents := 3 + expectedEventSize := 1435 // bootstrapping does not cache programs expectedCachedPrograms := 0 @@ -1032,9 +1216,9 @@ func Test_ExecutingSystemCollection(t *testing.T) { metrics.On("ExecutionTransactionExecuted", mock.Anything, // duration + mock.Anything, // conflict retry count mock.Anything, // computation used mock.Anything, // memory used - mock.Anything, // actual memory used expectedNumberOfEvents, expectedEventSize, false). @@ -1054,6 +1238,12 @@ func Test_ExecutingSystemCollection(t *testing.T) { Return(nil). Times(1) // block + metrics.On( + "ExecutionBlockExecutionEffortVectorComponent", + mock.Anything, + mock.Anything). + Return(nil) + bservice := requesterunit.MockBlobService(blockstore.NewBlockstore(dssync.MutexWrap(datastore.NewMapDatastore()))) trackerStorage := mocktracker.NewMockStorage() @@ -1071,6 +1261,8 @@ func Test_ExecutingSystemCollection(t *testing.T) { me.On("SignFunc", mock.Anything, mock.Anything, mock.Anything). Return(nil, nil) + constRandomSource := make([]byte, 32) + exe, err := computer.NewBlockComputer( vm, execCtx, @@ -1080,7 +1272,9 @@ func Test_ExecutingSystemCollection(t *testing.T) { committer, me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(constRandomSource), + testMaxConcurrency) require.NoError(t, err) // create empty block, it will have system collection attached while executing @@ -1091,21 +1285,28 @@ func Test_ExecutingSystemCollection(t *testing.T) { unittest.IdentifierFixture(), block, ledger, - derived.NewEmptyDerivedBlockData()) + derived.NewEmptyDerivedBlockData(0)) assert.NoError(t, err) - assert.Len(t, result.StateSnapshots, 1) // +1 system chunk - assert.Len(t, result.TransactionResults, 1) + assert.Len(t, result.AllExecutionSnapshots(), 1) // +1 system chunk + assert.Len(t, result.AllTransactionResults(), 1) - assert.Empty(t, result.TransactionResults[0].ErrorMessage) + assert.Empty(t, result.AllTransactionResults()[0].ErrorMessage) committer.AssertExpectations(t) } -func generateBlock(collectionCount, transactionCount int, addressGenerator flow.AddressGenerator) *entity.ExecutableBlock { +func generateBlock( + collectionCount, transactionCount int, + addressGenerator flow.AddressGenerator, +) *entity.ExecutableBlock { return generateBlockWithVisitor(collectionCount, transactionCount, addressGenerator, nil) } -func generateBlockWithVisitor(collectionCount, transactionCount int, addressGenerator flow.AddressGenerator, visitor func(body *flow.TransactionBody)) *entity.ExecutableBlock { +func generateBlockWithVisitor( + collectionCount, transactionCount int, + addressGenerator flow.AddressGenerator, + visitor func(body *flow.TransactionBody), +) *entity.ExecutableBlock { collections := make([]*entity.CompleteCollection, collectionCount) guarantees := make([]*flow.CollectionGuarantee, collectionCount) completeCollections := make(map[flow.Identifier]*entity.CompleteCollection) @@ -1135,7 +1336,11 @@ func generateBlockWithVisitor(collectionCount, transactionCount int, addressGene } } -func generateCollection(transactionCount int, addressGenerator flow.AddressGenerator, visitor func(body *flow.TransactionBody)) *entity.CompleteCollection { +func generateCollection( + transactionCount int, + addressGenerator flow.AddressGenerator, + visitor func(body *flow.TransactionBody), +) *entity.CompleteCollection { transactions := make([]*flow.TransactionBody, transactionCount) for i := 0; i < transactionCount; i++ { @@ -1163,47 +1368,115 @@ func generateCollection(transactionCount int, addressGenerator flow.AddressGener } } +type noOpExecutor struct{} + +func (noOpExecutor) Cleanup() {} + +func (noOpExecutor) Preprocess() error { + return nil +} + +func (noOpExecutor) Execute() error { + return nil +} + +func (noOpExecutor) Output() fvm.ProcedureOutput { + return fvm.ProcedureOutput{} +} + type testVM struct { t *testing.T eventsPerTransaction int - callCount int + callCount int32 // atomic variable err fvmErrors.CodedError } +type testExecutor struct { + *testVM + + ctx fvm.Context + proc fvm.Procedure + txnState storage.TransactionPreparer +} + +func (testExecutor) Cleanup() { +} + +func (testExecutor) Preprocess() error { + return nil +} + +func (executor *testExecutor) Execute() error { + atomic.AddInt32(&executor.callCount, 1) + + getSetAProgram(executor.t, executor.txnState) + + return nil +} + +func (executor *testExecutor) Output() fvm.ProcedureOutput { + txn := executor.proc.(*fvm.TransactionProcedure) + + return fvm.ProcedureOutput{ + Events: generateEvents(executor.eventsPerTransaction, txn.TxIndex), + Err: executor.err, + } +} + +func (vm *testVM) NewExecutor( + ctx fvm.Context, + proc fvm.Procedure, + txnState storage.TransactionPreparer, +) fvm.ProcedureExecutor { + return &testExecutor{ + testVM: vm, + proc: proc, + ctx: ctx, + txnState: txnState, + } +} + +func (vm *testVM) CallCount() int { + return int(atomic.LoadInt32(&vm.callCount)) +} + func (vm *testVM) Run( ctx fvm.Context, proc fvm.Procedure, - storageSnapshot state.StorageSnapshot, + storageSnapshot snapshot.StorageSnapshot, ) ( - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, fvm.ProcedureOutput, error, ) { - vm.callCount += 1 + database := storage.NewBlockDatabase( + storageSnapshot, + proc.ExecutionTime(), + ctx.DerivedBlockData) - txn := proc.(*fvm.TransactionProcedure) + txn, err := database.NewTransaction( + proc.ExecutionTime(), + state.DefaultParameters()) + require.NoError(vm.t, err) - derivedTxnData, err := ctx.DerivedBlockData.NewDerivedTransactionData( - txn.ExecutionTime(), - txn.ExecutionTime()) + executor := vm.NewExecutor(ctx, proc, txn) + err = fvm.Run(executor) require.NoError(vm.t, err) - getSetAProgram(vm.t, storageSnapshot, derivedTxnData) + err = txn.Finalize() + require.NoError(vm.t, err) - snapshot := &state.ExecutionSnapshot{} - output := fvm.ProcedureOutput{ - Events: generateEvents(vm.eventsPerTransaction, txn.TxIndex), - Err: vm.err, - } + executionSnapshot, err := txn.Commit() + require.NoError(vm.t, err) - return snapshot, output, nil + return executionSnapshot, executor.Output(), nil } func (testVM) GetAccount( _ fvm.Context, _ flow.Address, - _ state.StorageSnapshot, + _ snapshot.StorageSnapshot, ) ( *flow.Account, error, @@ -1215,27 +1488,87 @@ func generateEvents(eventCount int, txIndex uint32) []flow.Event { events := make([]flow.Event, eventCount) for i := 0; i < eventCount; i++ { // creating some dummy event - event := flow.Event{Type: "whatever", EventIndex: uint32(i), TransactionIndex: txIndex} + event := flow.Event{ + Type: "whatever", + EventIndex: uint32(i), + TransactionIndex: txIndex, + } events[i] = event } return events } -func getSetAProgram( - t *testing.T, - storageSnapshot state.StorageSnapshot, - derivedTxnData derived.DerivedTransactionCommitter, +type errorVM struct { + errorAt logical.Time +} + +type errorExecutor struct { + err error +} + +func (errorExecutor) Cleanup() {} + +func (errorExecutor) Preprocess() error { + return nil +} + +func (e errorExecutor) Execute() error { + return e.err +} + +func (errorExecutor) Output() fvm.ProcedureOutput { + return fvm.ProcedureOutput{} +} + +func (vm errorVM) NewExecutor( + ctx fvm.Context, + proc fvm.Procedure, + txn storage.TransactionPreparer, +) fvm.ProcedureExecutor { + var err error + if proc.ExecutionTime() == vm.errorAt { + err = fmt.Errorf("boom - internal error") + } + + return errorExecutor{err: err} +} + +func (vm errorVM) Run( + ctx fvm.Context, + proc fvm.Procedure, + storageSnapshot snapshot.StorageSnapshot, +) ( + *snapshot.ExecutionSnapshot, + fvm.ProcedureOutput, + error, ) { + var err error + if proc.ExecutionTime() == vm.errorAt { + err = fmt.Errorf("boom - internal error") + } + return &snapshot.ExecutionSnapshot{}, fvm.ProcedureOutput{}, err +} - txnState := state.NewTransactionState( - delta.NewDeltaView(storageSnapshot), - state.DefaultParameters()) +func (errorVM) GetAccount( + ctx fvm.Context, + addr flow.Address, + storageSnapshot snapshot.StorageSnapshot, +) ( + *flow.Account, + error, +) { + panic("not implemented") +} +func getSetAProgram( + t *testing.T, + txnState storage.TransactionPreparer, +) { loc := common.AddressLocation{ Name: "SomeContract", Address: common.MustBytesToAddress([]byte{0x1}), } - _, err := derivedTxnData.GetOrComputeProgram( + _, err := txnState.GetOrComputeProgram( txnState, loc, &programLoader{ @@ -1245,9 +1578,6 @@ func getSetAProgram( }, ) require.NoError(t, err) - - err = derivedTxnData.Commit() - require.NoError(t, err) } type programLoader struct { @@ -1255,7 +1585,7 @@ type programLoader struct { } func (p *programLoader) Compute( - _ state.NestedTransaction, + _ state.NestedTransactionPreparer, _ common.AddressLocation, ) ( *derived.Program, diff --git a/engine/execution/computation/computer/mock/block_computer.go b/engine/execution/computation/computer/mock/block_computer.go index a60049b2227..7464c38e9b2 100644 --- a/engine/execution/computation/computer/mock/block_computer.go +++ b/engine/execution/computation/computer/mock/block_computer.go @@ -14,7 +14,7 @@ import ( mock "github.com/stretchr/testify/mock" - state "github.com/onflow/flow-go/fvm/state" + snapshot "github.com/onflow/flow-go/fvm/storage/snapshot" ) // BlockComputer is an autogenerated mock type for the BlockComputer type @@ -22,25 +22,25 @@ type BlockComputer struct { mock.Mock } -// ExecuteBlock provides a mock function with given fields: ctx, parentBlockExecutionResultID, block, snapshot, derivedBlockData -func (_m *BlockComputer) ExecuteBlock(ctx context.Context, parentBlockExecutionResultID flow.Identifier, block *entity.ExecutableBlock, snapshot state.StorageSnapshot, derivedBlockData *derived.DerivedBlockData) (*execution.ComputationResult, error) { - ret := _m.Called(ctx, parentBlockExecutionResultID, block, snapshot, derivedBlockData) +// ExecuteBlock provides a mock function with given fields: ctx, parentBlockExecutionResultID, block, _a3, derivedBlockData +func (_m *BlockComputer) ExecuteBlock(ctx context.Context, parentBlockExecutionResultID flow.Identifier, block *entity.ExecutableBlock, _a3 snapshot.StorageSnapshot, derivedBlockData *derived.DerivedBlockData) (*execution.ComputationResult, error) { + ret := _m.Called(ctx, parentBlockExecutionResultID, block, _a3, derivedBlockData) var r0 *execution.ComputationResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, state.StorageSnapshot, *derived.DerivedBlockData) (*execution.ComputationResult, error)); ok { - return rf(ctx, parentBlockExecutionResultID, block, snapshot, derivedBlockData) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, snapshot.StorageSnapshot, *derived.DerivedBlockData) (*execution.ComputationResult, error)); ok { + return rf(ctx, parentBlockExecutionResultID, block, _a3, derivedBlockData) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, state.StorageSnapshot, *derived.DerivedBlockData) *execution.ComputationResult); ok { - r0 = rf(ctx, parentBlockExecutionResultID, block, snapshot, derivedBlockData) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, snapshot.StorageSnapshot, *derived.DerivedBlockData) *execution.ComputationResult); ok { + r0 = rf(ctx, parentBlockExecutionResultID, block, _a3, derivedBlockData) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*execution.ComputationResult) } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, state.StorageSnapshot, *derived.DerivedBlockData) error); ok { - r1 = rf(ctx, parentBlockExecutionResultID, block, snapshot, derivedBlockData) + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, snapshot.StorageSnapshot, *derived.DerivedBlockData) error); ok { + r1 = rf(ctx, parentBlockExecutionResultID, block, _a3, derivedBlockData) } else { r1 = ret.Error(1) } diff --git a/engine/execution/computation/computer/mock/transaction_write_behind_logger.go b/engine/execution/computation/computer/mock/transaction_write_behind_logger.go new file mode 100644 index 00000000000..b9b42474d00 --- /dev/null +++ b/engine/execution/computation/computer/mock/transaction_write_behind_logger.go @@ -0,0 +1,39 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + computer "github.com/onflow/flow-go/engine/execution/computation/computer" + fvm "github.com/onflow/flow-go/fvm" + + mock "github.com/stretchr/testify/mock" + + snapshot "github.com/onflow/flow-go/fvm/storage/snapshot" + + time "time" +) + +// TransactionWriteBehindLogger is an autogenerated mock type for the TransactionWriteBehindLogger type +type TransactionWriteBehindLogger struct { + mock.Mock +} + +// AddTransactionResult provides a mock function with given fields: txn, _a1, output, timeSpent, numTxnConflictRetries +func (_m *TransactionWriteBehindLogger) AddTransactionResult(txn computer.TransactionRequest, _a1 *snapshot.ExecutionSnapshot, output fvm.ProcedureOutput, timeSpent time.Duration, numTxnConflictRetries int) { + _m.Called(txn, _a1, output, timeSpent, numTxnConflictRetries) +} + +type mockConstructorTestingTNewTransactionWriteBehindLogger interface { + mock.TestingT + Cleanup(func()) +} + +// NewTransactionWriteBehindLogger creates a new instance of TransactionWriteBehindLogger. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTransactionWriteBehindLogger(t mockConstructorTestingTNewTransactionWriteBehindLogger) *TransactionWriteBehindLogger { + mock := &TransactionWriteBehindLogger{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/execution/computation/computer/mock/view_committer.go b/engine/execution/computation/computer/mock/view_committer.go index a38657e3c66..dfcacb97c83 100644 --- a/engine/execution/computation/computer/mock/view_committer.go +++ b/engine/execution/computation/computer/mock/view_committer.go @@ -8,7 +8,7 @@ import ( mock "github.com/stretchr/testify/mock" - state "github.com/onflow/flow-go/fvm/state" + snapshot "github.com/onflow/flow-go/fvm/storage/snapshot" ) // ViewCommitter is an autogenerated mock type for the ViewCommitter type @@ -17,17 +17,17 @@ type ViewCommitter struct { } // CommitView provides a mock function with given fields: _a0, _a1 -func (_m *ViewCommitter) CommitView(_a0 *state.ExecutionSnapshot, _a1 flow.StateCommitment) (flow.StateCommitment, []byte, *ledger.TrieUpdate, error) { +func (_m *ViewCommitter) CommitView(_a0 *snapshot.ExecutionSnapshot, _a1 flow.StateCommitment) (flow.StateCommitment, []byte, *ledger.TrieUpdate, error) { ret := _m.Called(_a0, _a1) var r0 flow.StateCommitment var r1 []byte var r2 *ledger.TrieUpdate var r3 error - if rf, ok := ret.Get(0).(func(*state.ExecutionSnapshot, flow.StateCommitment) (flow.StateCommitment, []byte, *ledger.TrieUpdate, error)); ok { + if rf, ok := ret.Get(0).(func(*snapshot.ExecutionSnapshot, flow.StateCommitment) (flow.StateCommitment, []byte, *ledger.TrieUpdate, error)); ok { return rf(_a0, _a1) } - if rf, ok := ret.Get(0).(func(*state.ExecutionSnapshot, flow.StateCommitment) flow.StateCommitment); ok { + if rf, ok := ret.Get(0).(func(*snapshot.ExecutionSnapshot, flow.StateCommitment) flow.StateCommitment); ok { r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { @@ -35,7 +35,7 @@ func (_m *ViewCommitter) CommitView(_a0 *state.ExecutionSnapshot, _a1 flow.State } } - if rf, ok := ret.Get(1).(func(*state.ExecutionSnapshot, flow.StateCommitment) []byte); ok { + if rf, ok := ret.Get(1).(func(*snapshot.ExecutionSnapshot, flow.StateCommitment) []byte); ok { r1 = rf(_a0, _a1) } else { if ret.Get(1) != nil { @@ -43,7 +43,7 @@ func (_m *ViewCommitter) CommitView(_a0 *state.ExecutionSnapshot, _a1 flow.State } } - if rf, ok := ret.Get(2).(func(*state.ExecutionSnapshot, flow.StateCommitment) *ledger.TrieUpdate); ok { + if rf, ok := ret.Get(2).(func(*snapshot.ExecutionSnapshot, flow.StateCommitment) *ledger.TrieUpdate); ok { r2 = rf(_a0, _a1) } else { if ret.Get(2) != nil { @@ -51,7 +51,7 @@ func (_m *ViewCommitter) CommitView(_a0 *state.ExecutionSnapshot, _a1 flow.State } } - if rf, ok := ret.Get(3).(func(*state.ExecutionSnapshot, flow.StateCommitment) error); ok { + if rf, ok := ret.Get(3).(func(*snapshot.ExecutionSnapshot, flow.StateCommitment) error); ok { r3 = rf(_a0, _a1) } else { r3 = ret.Error(3) diff --git a/engine/execution/computation/computer/result_collector.go b/engine/execution/computation/computer/result_collector.go index 21927b6bf53..4915b1cc866 100644 --- a/engine/execution/computation/computer/result_collector.go +++ b/engine/execution/computation/computer/result_collector.go @@ -12,9 +12,10 @@ import ( "github.com/onflow/flow-go/crypto/hash" "github.com/onflow/flow-go/engine/execution" "github.com/onflow/flow-go/engine/execution/computation/result" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/meter" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -24,11 +25,12 @@ import ( "github.com/onflow/flow-go/module/trace" ) -// ViewCommitter commits views's deltas to the ledger and collects the proofs +// ViewCommitter commits execution snapshot to the ledger and collects +// the proofs type ViewCommitter interface { - // CommitView commits a views' register delta and collects proofs + // CommitView commits an execution snapshot and collects proofs CommitView( - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, flow.StateCommitment, ) ( flow.StateCommitment, @@ -39,9 +41,11 @@ type ViewCommitter interface { } type transactionResult struct { - transaction - *state.ExecutionSnapshot + TransactionRequest + *snapshot.ExecutionSnapshot fvm.ProcedureOutput + timeSpent time.Duration + numConflictRetries int } // TODO(ramtin): move committer and other folks to consumers layer @@ -69,15 +73,14 @@ type resultCollector struct { result *execution.ComputationResult consumers []result.ExecutedCollectionConsumer - chunks []*flow.Chunk - spockSignatures []crypto.Signature - convertedServiceEvents flow.ServiceEventList + spockSignatures []crypto.Signature blockStartTime time.Time blockStats module.ExecutionResultStats + blockMeter *meter.Meter currentCollectionStartTime time.Time - currentCollectionView state.View + currentCollectionState *state.ExecutionState currentCollectionStats module.ExecutionResultStats } @@ -111,11 +114,11 @@ func newResultCollector( parentBlockExecutionResultID: parentBlockExecutionResultID, result: execution.NewEmptyComputationResult(block), consumers: consumers, - chunks: make([]*flow.Chunk, 0, numCollections), spockSignatures: make([]crypto.Signature, 0, numCollections), blockStartTime: now, + blockMeter: meter.NewMeter(meter.DefaultParameters()), currentCollectionStartTime: now, - currentCollectionView: delta.NewDeltaView(nil), + currentCollectionState: state.NewExecutionState(nil, state.DefaultParameters()), currentCollectionStats: module.ExecutionResultStats{ NumberOfCollections: 1, }, @@ -129,13 +132,13 @@ func newResultCollector( func (collector *resultCollector) commitCollection( collection collectionInfo, startTime time.Time, - collectionExecutionSnapshot *state.ExecutionSnapshot, + collectionExecutionSnapshot *snapshot.ExecutionSnapshot, ) error { defer collector.tracer.StartSpanFromParent( collector.blockSpan, trace.EXECommitDelta).End() - startState := collector.result.EndState + startState := collector.result.CurrentEndState() endState, proof, trieUpdate, err := collector.committer.CommitView( collectionExecutionSnapshot, startState) @@ -143,65 +146,34 @@ func (collector *resultCollector) commitCollection( return fmt.Errorf("commit view failed: %w", err) } - events := collector.result.Events[collection.collectionIndex] + execColRes := collector.result.CollectionExecutionResultAt(collection.collectionIndex) + execColRes.UpdateExecutionSnapshot(collectionExecutionSnapshot) + + events := execColRes.Events() eventsHash, err := flow.EventsMerkleRootHash(events) if err != nil { return fmt.Errorf("hash events failed: %w", err) } - collector.result.EventsHashes = append( - collector.result.EventsHashes, - eventsHash) + col := collection.Collection() + chunkExecData := &execution_data.ChunkExecutionData{ + Collection: &col, + Events: events, + TrieUpdate: trieUpdate, + } - chunk := flow.NewChunk( - collection.blockId, - collection.collectionIndex, + collector.result.AppendCollectionAttestationResult( startState, - len(collection.Transactions), + endState, + proof, eventsHash, - endState) - collector.chunks = append(collector.chunks, chunk) - - collectionStruct := collection.Collection() - - // Note: There's some inconsistency in how chunk execution data and - // chunk data pack populate their collection fields when the collection - // is the system collection. - executionCollection := &collectionStruct - dataPackCollection := executionCollection - if collection.isSystemTransaction { - dataPackCollection = nil - } - - collector.result.ChunkDataPacks = append( - collector.result.ChunkDataPacks, - flow.NewChunkDataPack( - chunk.ID(), - startState, - proof, - dataPackCollection)) - - collector.result.ChunkExecutionDatas = append( - collector.result.ChunkExecutionDatas, - &execution_data.ChunkExecutionData{ - Collection: executionCollection, - Events: collector.result.Events[collection.collectionIndex], - TrieUpdate: trieUpdate, - }) + chunkExecData, + ) collector.metrics.ExecutionChunkDataPackGenerated( len(proof), len(collection.Transactions)) - collector.result.EndState = endState - - collector.result.TransactionResultIndex = append( - collector.result.TransactionResultIndex, - len(collector.result.TransactionResults)) - collector.result.StateSnapshots = append( - collector.result.StateSnapshots, - collectionExecutionSnapshot) - spock, err := collector.signer.SignFunc( collectionExecutionSnapshot.SpockSecret, collector.spockHasher, @@ -226,15 +198,16 @@ func (collector *resultCollector) commitCollection( collector.currentCollectionStats) collector.blockStats.Merge(collector.currentCollectionStats) + collector.blockMeter.MergeMeter(collectionExecutionSnapshot.Meter) collector.currentCollectionStartTime = time.Now() - collector.currentCollectionView = delta.NewDeltaView(nil) + collector.currentCollectionState = state.NewExecutionState(nil, state.DefaultParameters()) collector.currentCollectionStats = module.ExecutionResultStats{ NumberOfCollections: 1, } for _, consumer := range collector.consumers { - err = consumer.OnExecutedCollection(collector.result.CollectionResult(collection.collectionIndex)) + err = consumer.OnExecutedCollection(collector.result.CollectionExecutionResultAt(collection.collectionIndex)) if err != nil { return fmt.Errorf("consumer failed: %w", err) } @@ -244,20 +217,49 @@ func (collector *resultCollector) commitCollection( } func (collector *resultCollector) processTransactionResult( - txn transaction, - txnExecutionSnapshot *state.ExecutionSnapshot, + txn TransactionRequest, + txnExecutionSnapshot *snapshot.ExecutionSnapshot, output fvm.ProcedureOutput, + timeSpent time.Duration, + numConflictRetries int, ) error { - collector.convertedServiceEvents = append( - collector.convertedServiceEvents, - output.ConvertedServiceEvents...) + logger := txn.ctx.Logger.With(). + Uint64("computation_used", output.ComputationUsed). + Uint64("memory_used", output.MemoryEstimate). + Int64("time_spent_in_ms", timeSpent.Milliseconds()). + Logger() + + if output.Err != nil { + logger = logger.With(). + Str("error_message", output.Err.Error()). + Uint16("error_code", uint16(output.Err.Code())). + Logger() + logger.Info().Msg("transaction execution failed") + + if txn.isSystemTransaction { + // This log is used as the data source for an alert on grafana. + // The system_chunk_error field must not be changed without adding + // the corresponding changes in grafana. + // https://github.com/dapperlabs/flow-internal/issues/1546 + logger.Error(). + Bool("system_chunk_error", true). + Bool("system_transaction_error", true). + Bool("critical_error", true). + Msg("error executing system chunk transaction") + } + } else { + logger.Info().Msg("transaction executed successfully") + } - collector.result.Events[txn.collectionIndex] = append( - collector.result.Events[txn.collectionIndex], - output.Events...) - collector.result.ServiceEvents = append( - collector.result.ServiceEvents, - output.ServiceEvents...) + collector.metrics.ExecutionTransactionExecuted( + timeSpent, + numConflictRetries, + output.ComputationUsed, + output.MemoryEstimate, + len(output.Events), + flow.EventsList(output.Events).ByteSize(), + output.Err != nil, + ) txnResult := flow.TransactionResult{ TransactionID: txn.ID, @@ -268,15 +270,16 @@ func (collector *resultCollector) processTransactionResult( txnResult.ErrorMessage = output.Err.Error() } - collector.result.TransactionResults = append( - collector.result.TransactionResults, - txnResult) + collector.result. + CollectionExecutionResultAt(txn.collectionIndex). + AppendTransactionResults( + output.Events, + output.ServiceEvents, + output.ConvertedServiceEvents, + txnResult, + ) - for computationKind, intensity := range output.ComputationIntensities { - collector.result.ComputationIntensities[computationKind] += intensity - } - - err := collector.currentCollectionView.Merge(txnExecutionSnapshot) + err := collector.currentCollectionState.Merge(txnExecutionSnapshot) if err != nil { return fmt.Errorf("failed to merge into collection view: %w", err) } @@ -292,18 +295,22 @@ func (collector *resultCollector) processTransactionResult( return collector.commitCollection( txn.collectionInfo, collector.currentCollectionStartTime, - collector.currentCollectionView.Finalize()) + collector.currentCollectionState.Finalize()) } func (collector *resultCollector) AddTransactionResult( - txn transaction, - snapshot *state.ExecutionSnapshot, + request TransactionRequest, + snapshot *snapshot.ExecutionSnapshot, output fvm.ProcedureOutput, + timeSpent time.Duration, + numConflictRetries int, ) { result := transactionResult{ - transaction: txn, - ExecutionSnapshot: snapshot, - ProcedureOutput: output, + TransactionRequest: request, + ExecutionSnapshot: snapshot, + ProcedureOutput: output, + timeSpent: timeSpent, + numConflictRetries: numConflictRetries, } select { @@ -319,9 +326,11 @@ func (collector *resultCollector) runResultProcessor() { for result := range collector.processorInputChan { err := collector.processTransactionResult( - result.transaction, + result.TransactionRequest, result.ExecutionSnapshot, - result.ProcedureOutput) + result.ProcedureOutput, + result.timeSpent, + result.numConflictRetries) if err != nil { collector.processorError = err return @@ -360,8 +369,8 @@ func (collector *resultCollector) Finalize( executionResult := flow.NewExecutionResult( collector.parentBlockExecutionResultID, collector.result.ExecutableBlock.ID(), - collector.chunks, - collector.convertedServiceEvents, + collector.result.AllChunks(), + collector.result.AllConvertedServiceEvents(), executionDataID) executionReceipt, err := GenerateExecutionReceipt( @@ -379,6 +388,12 @@ func (collector *resultCollector) Finalize( time.Since(collector.blockStartTime), collector.blockStats) + for kind, intensity := range collector.blockMeter.ComputationIntensities() { + collector.metrics.ExecutionBlockExecutionEffortVectorComponent( + kind.String(), + intensity) + } + return collector.result, nil } diff --git a/engine/execution/computation/computer/transaction_coordinator.go b/engine/execution/computation/computer/transaction_coordinator.go new file mode 100644 index 00000000000..6ce2cb3757c --- /dev/null +++ b/engine/execution/computation/computer/transaction_coordinator.go @@ -0,0 +1,193 @@ +package computer + +import ( + "sync" + "time" + + "github.com/onflow/flow-go/fvm" + "github.com/onflow/flow-go/fvm/storage" + "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" +) + +type TransactionWriteBehindLogger interface { + AddTransactionResult( + txn TransactionRequest, + snapshot *snapshot.ExecutionSnapshot, + output fvm.ProcedureOutput, + timeSpent time.Duration, + numTxnConflictRetries int, + ) +} + +// transactionCoordinator provides synchronization functionality for driving +// transaction execution. +type transactionCoordinator struct { + vm fvm.VM + + mutex *sync.Mutex + cond *sync.Cond + + snapshotTime logical.Time // guarded by mutex, cond broadcast on updates. + abortErr error // guarded by mutex, cond broadcast on updates. + + // Note: database commit and result logging must occur within the same + // critical section (guraded by mutex). + database *storage.BlockDatabase + writeBehindLog TransactionWriteBehindLogger +} + +type transaction struct { + request TransactionRequest + numConflictRetries int + + coordinator *transactionCoordinator + + startedAt time.Time + storage.Transaction + fvm.ProcedureExecutor +} + +func newTransactionCoordinator( + vm fvm.VM, + storageSnapshot snapshot.StorageSnapshot, + cachedDerivedBlockData *derived.DerivedBlockData, + writeBehindLog TransactionWriteBehindLogger, +) *transactionCoordinator { + mutex := &sync.Mutex{} + cond := sync.NewCond(mutex) + + database := storage.NewBlockDatabase( + storageSnapshot, + 0, + cachedDerivedBlockData) + + return &transactionCoordinator{ + vm: vm, + mutex: mutex, + cond: cond, + snapshotTime: 0, + abortErr: nil, + database: database, + writeBehindLog: writeBehindLog, + } +} + +func (coordinator *transactionCoordinator) SnapshotTime() logical.Time { + coordinator.mutex.Lock() + defer coordinator.mutex.Unlock() + + return coordinator.snapshotTime +} + +func (coordinator *transactionCoordinator) Error() error { + coordinator.mutex.Lock() + defer coordinator.mutex.Unlock() + + return coordinator.abortErr +} + +func (coordinator *transactionCoordinator) AbortAllOutstandingTransactions( + err error, +) { + coordinator.mutex.Lock() + defer coordinator.mutex.Unlock() + + if coordinator.abortErr != nil { // Transactions are already aborting. + return + } + + coordinator.abortErr = err + coordinator.cond.Broadcast() +} + +func (coordinator *transactionCoordinator) NewTransaction( + request TransactionRequest, + attempt int, +) ( + *transaction, + error, +) { + err := coordinator.Error() + if err != nil { + return nil, err + } + + txn, err := coordinator.database.NewTransaction( + request.ExecutionTime(), + fvm.ProcedureStateParameters(request.ctx, request)) + if err != nil { + return nil, err + } + + return &transaction{ + request: request, + coordinator: coordinator, + numConflictRetries: attempt, + startedAt: time.Now(), + Transaction: txn, + ProcedureExecutor: coordinator.vm.NewExecutor( + request.ctx, + request.TransactionProcedure, + txn), + }, nil +} + +func (coordinator *transactionCoordinator) commit(txn *transaction) error { + coordinator.mutex.Lock() + defer coordinator.mutex.Unlock() + + if coordinator.abortErr != nil { + return coordinator.abortErr + } + + executionSnapshot, err := txn.Transaction.Commit() + if err != nil { + return err + } + + coordinator.writeBehindLog.AddTransactionResult( + txn.request, + executionSnapshot, + txn.Output(), + time.Since(txn.startedAt), + txn.numConflictRetries) + + // Commit advances the database's snapshot. + coordinator.snapshotTime += 1 + coordinator.cond.Broadcast() + + return nil +} + +func (txn *transaction) Commit() error { + return txn.coordinator.commit(txn) +} + +func (coordinator *transactionCoordinator) waitForUpdatesNewerThan( + snapshotTime logical.Time, +) ( + logical.Time, + error, + logical.Time, + error, +) { + coordinator.mutex.Lock() + defer coordinator.mutex.Unlock() + + startTime := coordinator.snapshotTime + startErr := coordinator.abortErr + for coordinator.snapshotTime <= snapshotTime && coordinator.abortErr == nil { + coordinator.cond.Wait() + } + + return startTime, startErr, coordinator.snapshotTime, coordinator.abortErr +} + +func (txn *transaction) WaitForUpdates() error { + // Note: the frist three returned values are only used by tests to ensure + // the function correctly waited. + _, _, _, err := txn.coordinator.waitForUpdatesNewerThan(txn.SnapshotTime()) + return err +} diff --git a/engine/execution/computation/computer/transaction_coordinator_test.go b/engine/execution/computation/computer/transaction_coordinator_test.go new file mode 100644 index 00000000000..08d2717a66f --- /dev/null +++ b/engine/execution/computation/computer/transaction_coordinator_test.go @@ -0,0 +1,357 @@ +package computer + +import ( + "fmt" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/fvm" + "github.com/onflow/flow-go/fvm/storage" + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/model/flow" +) + +type testCoordinatorVM struct{} + +func (testCoordinatorVM) NewExecutor( + ctx fvm.Context, + proc fvm.Procedure, + txnState storage.TransactionPreparer, +) fvm.ProcedureExecutor { + return testCoordinatorExecutor{ + executionTime: proc.ExecutionTime(), + } +} + +func (testCoordinatorVM) Run( + ctx fvm.Context, + proc fvm.Procedure, + storageSnapshot snapshot.StorageSnapshot, +) ( + *snapshot.ExecutionSnapshot, + fvm.ProcedureOutput, + error, +) { + panic("not implemented") +} + +func (testCoordinatorVM) GetAccount( + _ fvm.Context, + _ flow.Address, + _ snapshot.StorageSnapshot, +) ( + *flow.Account, + error, +) { + panic("not implemented") +} + +type testCoordinatorExecutor struct { + executionTime logical.Time +} + +func (testCoordinatorExecutor) Cleanup() {} + +func (testCoordinatorExecutor) Preprocess() error { + return nil +} + +func (testCoordinatorExecutor) Execute() error { + return nil +} + +func (executor testCoordinatorExecutor) Output() fvm.ProcedureOutput { + return fvm.ProcedureOutput{ + ComputationUsed: uint64(executor.executionTime), + } +} + +type testCoordinator struct { + *transactionCoordinator + committed []uint64 +} + +func newTestCoordinator(t *testing.T) *testCoordinator { + db := &testCoordinator{} + db.transactionCoordinator = newTransactionCoordinator( + testCoordinatorVM{}, + nil, + nil, + db) + + require.Equal(t, db.SnapshotTime(), logical.Time(0)) + + // commit a transaction to increment the snapshot time + setupTxn, err := db.newTransaction(0) + require.NoError(t, err) + + err = setupTxn.Finalize() + require.NoError(t, err) + + err = setupTxn.Commit() + require.NoError(t, err) + + require.Equal(t, db.SnapshotTime(), logical.Time(1)) + + return db + +} + +func (db *testCoordinator) AddTransactionResult( + txn TransactionRequest, + snapshot *snapshot.ExecutionSnapshot, + output fvm.ProcedureOutput, + timeSpent time.Duration, + numConflictRetries int, +) { + db.committed = append(db.committed, output.ComputationUsed) +} + +func (db *testCoordinator) newTransaction(txnIndex uint32) ( + *transaction, + error, +) { + return db.NewTransaction( + newTransactionRequest( + collectionInfo{}, + fvm.NewContext(), + zerolog.Nop(), + txnIndex, + &flow.TransactionBody{}, + false), + 0) +} + +type testWaitValues struct { + startTime logical.Time + startErr error + snapshotTime logical.Time + abortErr error +} + +func (db *testCoordinator) setupWait(txn *transaction) chan testWaitValues { + ret := make(chan testWaitValues, 1) + go func() { + startTime, startErr, snapshotTime, abortErr := db.waitForUpdatesNewerThan( + txn.SnapshotTime()) + ret <- testWaitValues{ + startTime: startTime, + startErr: startErr, + snapshotTime: snapshotTime, + abortErr: abortErr, + } + }() + + // Sleep a bit to ensure goroutine is running before returning the channel. + time.Sleep(10 * time.Millisecond) + return ret +} + +func TestTransactionCoordinatorBasicCommit(t *testing.T) { + db := newTestCoordinator(t) + + txns := []*transaction{} + for i := uint32(1); i < 6; i++ { + txn, err := db.newTransaction(i) + require.NoError(t, err) + + txns = append(txns, txn) + } + + for i, txn := range txns { + executionTime := logical.Time(1 + i) + + require.Equal(t, txn.SnapshotTime(), logical.Time(1)) + + err := txn.Finalize() + require.NoError(t, err) + + err = txn.Validate() + require.NoError(t, err) + + require.Equal(t, txn.SnapshotTime(), executionTime) + + err = txn.Commit() + require.NoError(t, err) + + require.Equal(t, db.SnapshotTime(), executionTime+1) + } + + require.Equal(t, db.committed, []uint64{0, 1, 2, 3, 4, 5}) +} + +func TestTransactionCoordinatorBlockingWaitForCommit(t *testing.T) { + db := newTestCoordinator(t) + + testTxn, err := db.newTransaction(6) + require.NoError(t, err) + + require.Equal(t, db.SnapshotTime(), logical.Time(1)) + require.Equal(t, testTxn.SnapshotTime(), logical.Time(1)) + + ret := db.setupWait(testTxn) + + setupTxn, err := db.newTransaction(1) + require.NoError(t, err) + + err = setupTxn.Finalize() + require.NoError(t, err) + + err = setupTxn.Commit() + require.NoError(t, err) + + require.Equal(t, db.SnapshotTime(), logical.Time(2)) + + select { + case val := <-ret: + require.Equal( + t, + val, + testWaitValues{ + startTime: 1, + startErr: nil, + snapshotTime: 2, + abortErr: nil, + }) + case <-time.After(time.Second): + require.Fail(t, "Failed to return result") + } + + require.Equal(t, testTxn.SnapshotTime(), logical.Time(1)) + + err = testTxn.Validate() + require.NoError(t, err) + + require.Equal(t, testTxn.SnapshotTime(), logical.Time(2)) + +} + +func TestTransactionCoordinatorNonblockingWaitForCommit(t *testing.T) { + db := newTestCoordinator(t) + + testTxn, err := db.newTransaction(6) + require.NoError(t, err) + + setupTxn, err := db.newTransaction(1) + require.NoError(t, err) + + err = setupTxn.Finalize() + require.NoError(t, err) + + err = setupTxn.Commit() + require.NoError(t, err) + + require.Equal(t, db.SnapshotTime(), logical.Time(2)) + require.Equal(t, testTxn.SnapshotTime(), logical.Time(1)) + + ret := db.setupWait(testTxn) + + select { + case val := <-ret: + require.Equal( + t, + val, + testWaitValues{ + startTime: 2, + startErr: nil, + snapshotTime: 2, + abortErr: nil, + }) + case <-time.After(time.Second): + require.Fail(t, "Failed to return result") + } +} + +func TestTransactionCoordinatorBasicAbort(t *testing.T) { + db := newTestCoordinator(t) + + txn, err := db.newTransaction(1) + require.NoError(t, err) + + abortErr := fmt.Errorf("abort") + db.AbortAllOutstandingTransactions(abortErr) + + err = txn.Finalize() + require.NoError(t, err) + + err = txn.Commit() + require.Equal(t, err, abortErr) + + txn, err = db.newTransaction(2) + require.Equal(t, err, abortErr) + require.Nil(t, txn) +} + +func TestTransactionCoordinatorBlockingWaitForAbort(t *testing.T) { + db := newTestCoordinator(t) + + testTxn, err := db.newTransaction(6) + require.NoError(t, err) + + // start waiting before aborting. + require.Equal(t, testTxn.SnapshotTime(), logical.Time(1)) + ret := db.setupWait(testTxn) + + abortErr := fmt.Errorf("abort") + db.AbortAllOutstandingTransactions(abortErr) + + select { + case val := <-ret: + require.Equal( + t, + val, + testWaitValues{ + startTime: 1, + startErr: nil, + snapshotTime: 1, + abortErr: abortErr, + }) + case <-time.After(time.Second): + require.Fail(t, "Failed to return result") + } + + err = testTxn.Finalize() + require.NoError(t, err) + + err = testTxn.Commit() + require.Equal(t, err, abortErr) +} + +func TestTransactionCoordinatorNonblockingWaitForAbort(t *testing.T) { + db := newTestCoordinator(t) + + testTxn, err := db.newTransaction(6) + require.NoError(t, err) + + // start aborting before waiting. + abortErr := fmt.Errorf("abort") + db.AbortAllOutstandingTransactions(abortErr) + + require.Equal(t, testTxn.SnapshotTime(), logical.Time(1)) + ret := db.setupWait(testTxn) + + select { + case val := <-ret: + require.Equal( + t, + val, + testWaitValues{ + startTime: 1, + startErr: abortErr, + snapshotTime: 1, + abortErr: abortErr, + }) + case <-time.After(time.Second): + require.Fail(t, "Failed to return result") + } + + err = testTxn.Finalize() + require.NoError(t, err) + + err = testTxn.Commit() + require.Equal(t, err, abortErr) +} diff --git a/engine/execution/computation/execution_verification_test.go b/engine/execution/computation/execution_verification_test.go index 0ab9b1a3f11..9b5f53641c1 100644 --- a/engine/execution/computation/execution_verification_test.go +++ b/engine/execution/computation/execution_verification_test.go @@ -45,6 +45,12 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) +const ( + // TODO: enable parallel execution once cadence type equivalence check issue + // is resolved. + testVerifyMaxConcurrency = 1 +) + var chain = flow.Emulator.Chain() // In the following tests the system transaction is expected to fail, as the epoch related things are not set up properly. @@ -92,11 +98,14 @@ func Test_ExecutionMatchesVerification(t *testing.T) { }, }, fvm.BootstrapProcedureFeeParameters{}, fvm.DefaultMinimumStorageReservation) + colResult := cr.CollectionExecutionResultAt(0) + txResults := colResult.TransactionResults() + events := colResult.Events() // ensure event is emitted - require.Empty(t, cr.TransactionResults[0].ErrorMessage) - require.Empty(t, cr.TransactionResults[1].ErrorMessage) - require.Len(t, cr.Events[0], 2) - require.Equal(t, flow.EventType(fmt.Sprintf("A.%s.Foo.FooEvent", chain.ServiceAddress())), cr.Events[0][1].Type) + require.Empty(t, txResults[0].ErrorMessage) + require.Empty(t, txResults[1].ErrorMessage) + require.Len(t, events, 2) + require.Equal(t, flow.EventType(fmt.Sprintf("A.%s.Foo.FooEvent", chain.ServiceAddress())), events[1].Type) }) t.Run("multiple collections events", func(t *testing.T) { @@ -147,13 +156,38 @@ func Test_ExecutionMatchesVerification(t *testing.T) { }, }, fvm.BootstrapProcedureFeeParameters{}, fvm.DefaultMinimumStorageReservation) - // ensure event is emitted - require.Empty(t, cr.TransactionResults[0].ErrorMessage) - require.Empty(t, cr.TransactionResults[1].ErrorMessage) - require.Empty(t, cr.TransactionResults[2].ErrorMessage) - require.Empty(t, cr.TransactionResults[3].ErrorMessage) - require.Len(t, cr.Events[0], 2) - require.Equal(t, flow.EventType(fmt.Sprintf("A.%s.Foo.FooEvent", chain.ServiceAddress())), cr.Events[0][1].Type) + verifyTxResults := func(t *testing.T, colIndex, expResCount int) { + colResult := cr.CollectionExecutionResultAt(colIndex) + txResults := colResult.TransactionResults() + require.Len(t, txResults, expResCount) + for i := 0; i < expResCount; i++ { + require.Empty(t, txResults[i].ErrorMessage) + } + } + + verifyEvents := func(t *testing.T, colIndex int, eventTypes []flow.EventType) { + colResult := cr.CollectionExecutionResultAt(colIndex) + events := colResult.Events() + require.Len(t, events, len(eventTypes)) + for i, event := range events { + require.Equal(t, event.Type, eventTypes[i]) + } + } + + expEventType1 := flow.EventType("flow.AccountContractAdded") + expEventType2 := flow.EventType(fmt.Sprintf("A.%s.Foo.FooEvent", chain.ServiceAddress())) + + // first collection + verifyTxResults(t, 0, 2) + verifyEvents(t, 0, []flow.EventType{expEventType1, expEventType2}) + + // second collection + verifyTxResults(t, 1, 1) + verifyEvents(t, 1, []flow.EventType{expEventType2}) + + // 3rd collection + verifyTxResults(t, 2, 1) + verifyEvents(t, 2, []flow.EventType{expEventType2}) }) t.Run("with failed storage limit", func(t *testing.T) { @@ -183,14 +217,21 @@ func Test_ExecutionMatchesVerification(t *testing.T) { }, }, fvm.DefaultTransactionFees, minimumStorage) + colResult := cr.CollectionExecutionResultAt(0) + txResults := colResult.TransactionResults() // storage limit error - assert.Equal(t, cr.TransactionResults[0].ErrorMessage, "") + assert.Len(t, txResults, 1) + assert.Equal(t, txResults[0].ErrorMessage, "") // ensure events from the first transaction is emitted - require.Len(t, cr.Events[0], 10) - // ensure fee deduction events are emitted even though tx fails - require.Len(t, cr.Events[1], 3) + require.Len(t, colResult.Events(), 10) + + colResult = cr.CollectionExecutionResultAt(1) + txResults = colResult.TransactionResults() + assert.Len(t, txResults, 1) // storage limit error - assert.Contains(t, cr.TransactionResults[1].ErrorMessage, errors.ErrCodeStorageCapacityExceeded.String()) + assert.Contains(t, txResults[0].ErrorMessage, errors.ErrCodeStorageCapacityExceeded.String()) + // ensure fee deduction events are emitted even though tx fails + require.Len(t, colResult.Events(), 3) }) t.Run("with failed transaction fee deduction", func(t *testing.T) { @@ -248,24 +289,28 @@ func Test_ExecutionMatchesVerification(t *testing.T) { fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW), }) + colResult := cr.CollectionExecutionResultAt(0) + txResults := colResult.TransactionResults() + events := colResult.Events() + // no error - assert.Equal(t, cr.TransactionResults[0].ErrorMessage, "") + assert.Equal(t, txResults[0].ErrorMessage, "") // ensure events from the first transaction is emitted. Since transactions are in the same block, get all events from Events[0] transactionEvents := 0 - for _, event := range cr.Events[0] { - if event.TransactionID == cr.TransactionResults[0].TransactionID { + for _, event := range events { + if event.TransactionID == txResults[0].TransactionID { transactionEvents += 1 } } require.Equal(t, 10, transactionEvents) - assert.Contains(t, cr.TransactionResults[1].ErrorMessage, errors.ErrCodeStorageCapacityExceeded.String()) + assert.Contains(t, txResults[1].ErrorMessage, errors.ErrCodeStorageCapacityExceeded.String()) // ensure tx fee deduction events are emitted even though tx failed transactionEvents = 0 - for _, event := range cr.Events[0] { - if event.TransactionID == cr.TransactionResults[1].TransactionID { + for _, event := range events { + if event.TransactionID == txResults[1].TransactionID { transactionEvents += 1 } } @@ -293,14 +338,18 @@ func TestTransactionFeeDeduction(t *testing.T) { fundWith: fundingAmount, tryToTransfer: 0, checkResult: func(t *testing.T, cr *execution.ComputationResult) { - require.Empty(t, cr.TransactionResults[0].ErrorMessage) - require.Empty(t, cr.TransactionResults[1].ErrorMessage) - require.Empty(t, cr.TransactionResults[2].ErrorMessage) + txResults := cr.AllTransactionResults() + + require.Empty(t, txResults[0].ErrorMessage) + require.Empty(t, txResults[1].ErrorMessage) + require.Empty(t, txResults[2].ErrorMessage) var deposits []flow.Event var withdraws []flow.Event - for _, e := range cr.Events[2] { + // events of the first collection + events := cr.CollectionExecutionResultAt(2).Events() + for _, e := range events { if string(e.Type) == fmt.Sprintf("A.%s.FlowToken.TokensDeposited", fvm.FlowTokenAddress(chain)) { deposits = append(deposits, e) } @@ -318,14 +367,18 @@ func TestTransactionFeeDeduction(t *testing.T) { fundWith: txFees + transferAmount, tryToTransfer: transferAmount, checkResult: func(t *testing.T, cr *execution.ComputationResult) { - require.Empty(t, cr.TransactionResults[0].ErrorMessage) - require.Empty(t, cr.TransactionResults[1].ErrorMessage) - require.Empty(t, cr.TransactionResults[2].ErrorMessage) + txResults := cr.AllTransactionResults() + + require.Empty(t, txResults[0].ErrorMessage) + require.Empty(t, txResults[1].ErrorMessage) + require.Empty(t, txResults[2].ErrorMessage) var deposits []flow.Event var withdraws []flow.Event - for _, e := range cr.Events[2] { + // events of the last collection + events := cr.CollectionExecutionResultAt(2).Events() + for _, e := range events { if string(e.Type) == fmt.Sprintf("A.%s.FlowToken.TokensDeposited", fvm.FlowTokenAddress(chain)) { deposits = append(deposits, e) } @@ -345,14 +398,18 @@ func TestTransactionFeeDeduction(t *testing.T) { fundWith: txFees, tryToTransfer: 1, checkResult: func(t *testing.T, cr *execution.ComputationResult) { - require.Empty(t, cr.TransactionResults[0].ErrorMessage) - require.Empty(t, cr.TransactionResults[1].ErrorMessage) - require.Empty(t, cr.TransactionResults[2].ErrorMessage) + txResults := cr.AllTransactionResults() + + require.Empty(t, txResults[0].ErrorMessage) + require.Empty(t, txResults[1].ErrorMessage) + require.Empty(t, txResults[2].ErrorMessage) var deposits []flow.Event var withdraws []flow.Event - for _, e := range cr.Events[2] { + // events of the last collection + events := cr.CollectionExecutionResultAt(2).Events() + for _, e := range events { if string(e.Type) == fmt.Sprintf("A.%s.FlowToken.TokensDeposited", fvm.FlowTokenAddress(chain)) { deposits = append(deposits, e) } @@ -370,14 +427,18 @@ func TestTransactionFeeDeduction(t *testing.T) { fundWith: fundingAmount, tryToTransfer: 2 * fundingAmount, checkResult: func(t *testing.T, cr *execution.ComputationResult) { - require.Empty(t, cr.TransactionResults[0].ErrorMessage) - require.Empty(t, cr.TransactionResults[1].ErrorMessage) - require.Contains(t, cr.TransactionResults[2].ErrorMessage, "Error Code: 1101") + txResults := cr.AllTransactionResults() + + require.Empty(t, txResults[0].ErrorMessage) + require.Empty(t, txResults[1].ErrorMessage) + require.Contains(t, txResults[2].ErrorMessage, "Error Code: 1101") var deposits []flow.Event var withdraws []flow.Event - for _, e := range cr.Events[2] { + // events of the last collection + events := cr.CollectionExecutionResultAt(2).Events() + for _, e := range events { if string(e.Type) == fmt.Sprintf("A.%s.FlowToken.TokensDeposited", fvm.FlowTokenAddress(chain)) { deposits = append(deposits, e) } @@ -398,14 +459,18 @@ func TestTransactionFeeDeduction(t *testing.T) { fundWith: fundingAmount, tryToTransfer: 0, checkResult: func(t *testing.T, cr *execution.ComputationResult) { - require.Empty(t, cr.TransactionResults[0].ErrorMessage) - require.Empty(t, cr.TransactionResults[1].ErrorMessage) - require.Empty(t, cr.TransactionResults[2].ErrorMessage) + txResults := cr.AllTransactionResults() + + require.Empty(t, txResults[0].ErrorMessage) + require.Empty(t, txResults[1].ErrorMessage) + require.Empty(t, txResults[2].ErrorMessage) var deposits []flow.Event var withdraws []flow.Event - for _, e := range cr.Events[2] { + // events of the last collection + events := cr.CollectionExecutionResultAt(2).Events() + for _, e := range events { if string(e.Type) == fmt.Sprintf("A.%s.FlowToken.TokensDeposited", fvm.FlowTokenAddress(chain)) { deposits = append(deposits, e) } @@ -423,14 +488,18 @@ func TestTransactionFeeDeduction(t *testing.T) { fundWith: txFees + transferAmount, tryToTransfer: transferAmount, checkResult: func(t *testing.T, cr *execution.ComputationResult) { - require.Empty(t, cr.TransactionResults[0].ErrorMessage) - require.Empty(t, cr.TransactionResults[1].ErrorMessage) - require.Empty(t, cr.TransactionResults[2].ErrorMessage) + txResults := cr.AllTransactionResults() + + require.Empty(t, txResults[0].ErrorMessage) + require.Empty(t, txResults[1].ErrorMessage) + require.Empty(t, txResults[2].ErrorMessage) var deposits []flow.Event var withdraws []flow.Event - for _, e := range cr.Events[2] { + // events of the last collection + events := cr.CollectionExecutionResultAt(2).Events() + for _, e := range events { if string(e.Type) == fmt.Sprintf("A.%s.FlowToken.TokensDeposited", fvm.FlowTokenAddress(chain)) { deposits = append(deposits, e) } @@ -448,14 +517,18 @@ func TestTransactionFeeDeduction(t *testing.T) { fundWith: fundingAmount, tryToTransfer: 2 * fundingAmount, checkResult: func(t *testing.T, cr *execution.ComputationResult) { - require.Empty(t, cr.TransactionResults[0].ErrorMessage) - require.Empty(t, cr.TransactionResults[1].ErrorMessage) - require.Contains(t, cr.TransactionResults[2].ErrorMessage, "Error Code: 1101") + txResults := cr.AllTransactionResults() + + require.Empty(t, txResults[0].ErrorMessage) + require.Empty(t, txResults[1].ErrorMessage) + require.Contains(t, txResults[2].ErrorMessage, "Error Code: 1101") var deposits []flow.Event var withdraws []flow.Event - for _, e := range cr.Events[2] { + // events of the last collection + events := cr.CollectionExecutionResultAt(2).Events() + for _, e := range events { if string(e.Type) == fmt.Sprintf("A.%s.FlowToken.TokensDeposited", fvm.FlowTokenAddress(chain)) { deposits = append(deposits, e) } @@ -473,14 +546,18 @@ func TestTransactionFeeDeduction(t *testing.T) { fundWith: 0, tryToTransfer: 0, checkResult: func(t *testing.T, cr *execution.ComputationResult) { - require.Empty(t, cr.TransactionResults[0].ErrorMessage) - require.Empty(t, cr.TransactionResults[1].ErrorMessage) - require.Contains(t, cr.TransactionResults[2].ErrorMessage, errors.ErrCodeStorageCapacityExceeded.String()) + txResults := cr.AllTransactionResults() + + require.Empty(t, txResults[0].ErrorMessage) + require.Empty(t, txResults[1].ErrorMessage) + require.Contains(t, txResults[2].ErrorMessage, errors.ErrCodeStorageCapacityExceeded.String()) var deposits []flow.Event var withdraws []flow.Event - for _, e := range cr.Events[2] { + // events of the last collection + events := cr.CollectionExecutionResultAt(2).Events() + for _, e := range events { if string(e.Type) == fmt.Sprintf("A.%s.FlowToken.TokensDeposited", fvm.FlowTokenAddress(chain)) { deposits = append(deposits, e) } @@ -703,7 +780,9 @@ func executeBlockAndVerifyWithParameters(t *testing.T, ledgerCommiter, me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testVerifyMaxConcurrency) require.NoError(t, err) executableBlock := unittest.ExecutableBlockFromTransactions(chain.ChainID(), txs) @@ -717,11 +796,14 @@ func executeBlockAndVerifyWithParameters(t *testing.T, state.NewLedgerStorageSnapshot( ledger, initialCommit), - derived.NewEmptyDerivedBlockData()) + derived.NewEmptyDerivedBlockData(0)) require.NoError(t, err) spockHasher := utils.NewSPOCKHasher() - for i, snapshot := range computationResult.StateSnapshots { + + for i := 0; i < computationResult.BlockExecutionResult.Size(); i++ { + res := computationResult.CollectionExecutionResultAt(i) + snapshot := res.ExecutionSnapshot() valid, err := crypto.SPOCKVerifyAgainstData( myIdentity.StakingPubKey, computationResult.Spocks[i], @@ -741,9 +823,9 @@ func executeBlockAndVerifyWithParameters(t *testing.T, require.NoError(t, err) require.True(t, valid) - require.Equal(t, len(computationResult.ChunkDataPacks), len(receipt.Spocks)) + chdps := computationResult.AllChunkDataPacks() + require.Equal(t, len(chdps), len(receipt.Spocks)) - chdps := computationResult.ChunkDataPacks er := &computationResult.ExecutionResult verifier := chunks.NewChunkVerifier(vm, fvmContext, logger) diff --git a/engine/execution/computation/manager.go b/engine/execution/computation/manager.go index 896faa68dff..907e17cd8e7 100644 --- a/engine/execution/computation/manager.go +++ b/engine/execution/computation/manager.go @@ -12,8 +12,8 @@ import ( "github.com/onflow/flow-go/engine/execution/computation/query" "github.com/onflow/flow-go/fvm" reusableRuntime "github.com/onflow/flow-go/fvm/runtime" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/executiondatasync/provider" @@ -32,7 +32,7 @@ type ComputationManager interface { script []byte, arguments [][]byte, blockHeader *flow.Header, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) ( []byte, error, @@ -42,7 +42,7 @@ type ComputationManager interface { ctx context.Context, parentBlockExecutionResultID flow.Identifier, block *entity.ExecutableBlock, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) ( *execution.ComputationResult, error, @@ -52,7 +52,7 @@ type ComputationManager interface { ctx context.Context, addr flow.Address, header *flow.Header, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) ( *flow.Account, error, @@ -64,6 +64,7 @@ type ComputationConfig struct { CadenceTracing bool ExtensiveTracing bool DerivedDataCacheSize uint + MaxConcurrency int // When NewCustomVirtualMachine is nil, the manager will create a standard // fvm virtual machine via fvm.NewVirtualMachine. Otherwise, the manager @@ -115,9 +116,10 @@ func New( AccountLinkingEnabled: true, // Attachments are enabled everywhere except for Mainnet AttachmentsEnabled: chainID != flow.Mainnet, + // Capability Controllers are enabled everywhere except for Mainnet + CapabilityControllersEnabled: chainID != flow.Mainnet, }, - ), - ), + )), } if params.ExtensiveTracing { options = append(options, fvm.WithExtensiveTracing()) @@ -135,6 +137,8 @@ func New( me, executionDataProvider, nil, // TODO(ramtin): update me with proper consumers + protoState, + params.MaxConcurrency, ) if err != nil { @@ -174,7 +178,7 @@ func (e *Manager) ComputeBlock( ctx context.Context, parentBlockExecutionResultID flow.Identifier, block *entity.ExecutableBlock, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) (*execution.ComputationResult, error) { e.log.Debug(). @@ -211,7 +215,7 @@ func (e *Manager) ExecuteScript( code []byte, arguments [][]byte, blockHeader *flow.Header, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) ([]byte, error) { return e.queryExecutor.ExecuteScript(ctx, code, @@ -224,7 +228,7 @@ func (e *Manager) GetAccount( ctx context.Context, address flow.Address, blockHeader *flow.Header, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) ( *flow.Account, error, diff --git a/engine/execution/computation/manager_benchmark_test.go b/engine/execution/computation/manager_benchmark_test.go index b54b57e0afa..d5d55a50691 100644 --- a/engine/execution/computation/manager_benchmark_test.go +++ b/engine/execution/computation/manager_benchmark_test.go @@ -20,8 +20,8 @@ import ( "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/fvm" reusableRuntime "github.com/onflow/flow-go/fvm/runtime" - "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/executiondatasync/execution_data" exedataprovider "github.com/onflow/flow-go/module/executiondatasync/provider" @@ -47,10 +47,10 @@ type testAccounts struct { func createAccounts( b *testing.B, vm fvm.VM, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, num int, ) ( - storage.SnapshotTree, + snapshot.SnapshotTree, *testAccounts, ) { privateKeys, err := testutil.GenerateAccountPrivateKeys(num) @@ -78,10 +78,10 @@ func createAccounts( func mustFundAccounts( b *testing.B, vm fvm.VM, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, execCtx fvm.Context, accs *testAccounts, -) storage.SnapshotTree { +) snapshot.SnapshotTree { var err error for _, acc := range accs.accounts { transferTx := testutil.CreateTokenTransferTransaction(chain, 1_000_000, acc.address, chain.ServiceAddress()) @@ -103,7 +103,48 @@ func mustFundAccounts( func BenchmarkComputeBlock(b *testing.B) { b.StopTimer() + b.SetParallelism(1) + + type benchmarkCase struct { + numCollections int + numTransactionsPerCollection int + maxConcurrency int + } + for _, benchCase := range []benchmarkCase{ + { + numCollections: 16, + numTransactionsPerCollection: 128, + maxConcurrency: 1, + }, + { + numCollections: 16, + numTransactionsPerCollection: 128, + maxConcurrency: 2, + }, + } { + b.Run( + fmt.Sprintf( + "%d/cols/%d/txes/%d/max-concurrency", + benchCase.numCollections, + benchCase.numTransactionsPerCollection, + benchCase.maxConcurrency), + func(b *testing.B) { + benchmarkComputeBlock( + b, + benchCase.numCollections, + benchCase.numTransactionsPerCollection, + benchCase.maxConcurrency) + }) + } +} + +func benchmarkComputeBlock( + b *testing.B, + numCollections int, + numTransactionsPerCollection int, + maxConcurrency int, +) { tracer, err := trace.NewTracer(zerolog.Nop(), "", "", 4) require.NoError(b, err) @@ -159,7 +200,9 @@ func BenchmarkComputeBlock(b *testing.B) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + maxConcurrency) require.NoError(b, err) derivedChainData, err := derived.NewDerivedChainData( @@ -171,53 +214,49 @@ func BenchmarkComputeBlock(b *testing.B) { derivedChainData: derivedChainData, } - b.SetParallelism(1) - parentBlock := &flow.Block{ Header: &flow.Header{}, Payload: &flow.Payload{}, } - const ( - cols = 16 - txes = 128 - ) - - b.Run(fmt.Sprintf("%d/cols/%d/txes", cols, txes), func(b *testing.B) { + b.StopTimer() + b.ResetTimer() + + var elapsed time.Duration + for i := 0; i < b.N; i++ { + executableBlock := createBlock( + b, + parentBlock, + accs, + numCollections, + numTransactionsPerCollection) + parentBlock = executableBlock.Block + + b.StartTimer() + start := time.Now() + res, err := engine.ComputeBlock( + context.Background(), + unittest.IdentifierFixture(), + executableBlock, + snapshotTree) + elapsed += time.Since(start) b.StopTimer() - b.ResetTimer() - - var elapsed time.Duration - for i := 0; i < b.N; i++ { - executableBlock := createBlock(b, parentBlock, accs, cols, txes) - parentBlock = executableBlock.Block - - b.StartTimer() - start := time.Now() - res, err := engine.ComputeBlock( - context.Background(), - unittest.IdentifierFixture(), - executableBlock, - snapshotTree) - elapsed += time.Since(start) - b.StopTimer() - - for _, snapshot := range res.StateSnapshots { - snapshotTree = snapshotTree.Append(snapshot) - } - require.NoError(b, err) - for j, r := range res.TransactionResults { - // skip system transactions - if j >= cols*txes { - break - } - require.Emptyf(b, r.ErrorMessage, "Transaction %d failed", j) + require.NoError(b, err) + for _, snapshot := range res.AllExecutionSnapshots() { + snapshotTree = snapshotTree.Append(snapshot) + } + + for j, r := range res.AllTransactionResults() { + // skip system transactions + if j >= numCollections*numTransactionsPerCollection { + break } + require.Emptyf(b, r.ErrorMessage, "Transaction %d failed", j) } - totalTxes := int64(cols) * int64(txes) * int64(b.N) - b.ReportMetric(float64(elapsed.Nanoseconds()/totalTxes/int64(time.Microsecond)), "us/tx") - }) + } + totalTxes := int64(numCollections) * int64(numTransactionsPerCollection) * int64(b.N) + b.ReportMetric(float64(elapsed.Nanoseconds()/totalTxes/int64(time.Microsecond)), "us/tx") } func createBlock(b *testing.B, parentBlock *flow.Block, accs *testAccounts, colNum int, txNum int) *entity.ExecutableBlock { diff --git a/engine/execution/computation/manager_test.go b/engine/execution/computation/manager_test.go index 2ab899a4979..3dbfcb4e527 100644 --- a/engine/execution/computation/manager_test.go +++ b/engine/execution/computation/manager_test.go @@ -30,8 +30,9 @@ import ( "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/fvm/environment" fvmErrors "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger/complete" "github.com/onflow/flow-go/ledger/complete/wal/fixtures" "github.com/onflow/flow-go/model/flow" @@ -141,7 +142,9 @@ func TestComputeBlockWithStorage(t *testing.T) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) derivedChainData, err := derived.NewDerivedChainData(10) @@ -160,15 +163,15 @@ func TestComputeBlockWithStorage(t *testing.T) { require.NoError(t, err) hasUpdates := false - for _, snapshot := range returnedComputationResult.StateSnapshots { + for _, snapshot := range returnedComputationResult.AllExecutionSnapshots() { if len(snapshot.WriteSet) > 0 { hasUpdates = true break } } require.True(t, hasUpdates) - require.Len(t, returnedComputationResult.StateSnapshots, 1+1) // 1 coll + 1 system chunk - assert.NotEmpty(t, returnedComputationResult.StateSnapshots[0].UpdatedRegisters()) + require.Equal(t, returnedComputationResult.BlockExecutionResult.Size(), 1+1) // 1 coll + 1 system chunk + assert.NotEmpty(t, returnedComputationResult.AllExecutionSnapshots()[0].UpdatedRegisters()) } func TestComputeBlock_Uploader(t *testing.T) { @@ -267,6 +270,7 @@ func TestExecuteScript(t *testing.T) { ComputationConfig{ QueryConfig: query.NewDefaultConfig(), DerivedDataCacheSize: derived.DefaultDerivedDataCacheSize, + MaxConcurrency: 1, }, ) require.NoError(t, err) @@ -295,7 +299,7 @@ func TestExecuteScript_BalanceScriptFailsIfViewIsEmpty(t *testing.T) { me.On("SignFunc", mock.Anything, mock.Anything, mock.Anything). Return(nil, nil) - snapshot := state.NewReadFuncStorageSnapshot( + snapshot := snapshot.NewReadFuncStorageSnapshot( func(id flow.RegisterID) (flow.RegisterValue, error) { return nil, fmt.Errorf("error getting register") }) @@ -331,6 +335,7 @@ func TestExecuteScript_BalanceScriptFailsIfViewIsEmpty(t *testing.T) { ComputationConfig{ QueryConfig: query.NewDefaultConfig(), DerivedDataCacheSize: derived.DefaultDerivedDataCacheSize, + MaxConcurrency: 1, }, ) require.NoError(t, err) @@ -376,6 +381,7 @@ func TestExecuteScripPanicsAreHandled(t *testing.T) { ComputationConfig{ QueryConfig: query.NewDefaultConfig(), DerivedDataCacheSize: derived.DefaultDerivedDataCacheSize, + MaxConcurrency: 1, NewCustomVirtualMachine: func() fvm.VM { return &PanickingVM{} }, @@ -429,6 +435,7 @@ func TestExecuteScript_LongScriptsAreLogged(t *testing.T) { ExecutionTimeLimit: query.DefaultExecutionTimeLimit, }, DerivedDataCacheSize: 10, + MaxConcurrency: 1, NewCustomVirtualMachine: func() fvm.VM { return &LongRunningVM{duration: 2 * time.Millisecond} }, @@ -482,6 +489,7 @@ func TestExecuteScript_ShortScriptsAreNotLogged(t *testing.T) { ExecutionTimeLimit: query.DefaultExecutionTimeLimit, }, DerivedDataCacheSize: derived.DefaultDerivedDataCacheSize, + MaxConcurrency: 1, NewCustomVirtualMachine: func() fvm.VM { return &LongRunningVM{duration: 0} }, @@ -501,14 +509,38 @@ func TestExecuteScript_ShortScriptsAreNotLogged(t *testing.T) { require.NotContains(t, buffer.String(), "exceeded threshold") } +type PanickingExecutor struct{} + +func (PanickingExecutor) Cleanup() {} + +func (PanickingExecutor) Preprocess() error { + return nil +} + +func (PanickingExecutor) Execute() error { + panic("panic, but expected with sentinel for test: Verunsicherung ") +} + +func (PanickingExecutor) Output() fvm.ProcedureOutput { + return fvm.ProcedureOutput{} +} + type PanickingVM struct{} +func (p *PanickingVM) NewExecutor( + f fvm.Context, + procedure fvm.Procedure, + txn storage.TransactionPreparer, +) fvm.ProcedureExecutor { + return PanickingExecutor{} +} + func (p *PanickingVM) Run( f fvm.Context, procedure fvm.Procedure, - storageSnapshot state.StorageSnapshot, + storageSnapshot snapshot.StorageSnapshot, ) ( - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, fvm.ProcedureOutput, error, ) { @@ -518,7 +550,7 @@ func (p *PanickingVM) Run( func (p *PanickingVM) GetAccount( ctx fvm.Context, address flow.Address, - storageSnapshot state.StorageSnapshot, + storageSnapshot snapshot.StorageSnapshot, ) ( *flow.Account, error, @@ -526,22 +558,53 @@ func (p *PanickingVM) GetAccount( panic("not expected") } +type LongRunningExecutor struct { + duration time.Duration +} + +func (LongRunningExecutor) Cleanup() {} + +func (LongRunningExecutor) Preprocess() error { + return nil +} + +func (l LongRunningExecutor) Execute() error { + time.Sleep(l.duration) + return nil +} + +func (LongRunningExecutor) Output() fvm.ProcedureOutput { + return fvm.ProcedureOutput{ + Value: cadence.NewVoid(), + } +} + type LongRunningVM struct { duration time.Duration } +func (l *LongRunningVM) NewExecutor( + f fvm.Context, + procedure fvm.Procedure, + txn storage.TransactionPreparer, +) fvm.ProcedureExecutor { + return LongRunningExecutor{ + duration: l.duration, + } +} + func (l *LongRunningVM) Run( f fvm.Context, procedure fvm.Procedure, - storageSnapshot state.StorageSnapshot, + storageSnapshot snapshot.StorageSnapshot, ) ( - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, fvm.ProcedureOutput, error, ) { time.Sleep(l.duration) - snapshot := &state.ExecutionSnapshot{} + snapshot := &snapshot.ExecutionSnapshot{} output := fvm.ProcedureOutput{ Value: cadence.NewVoid(), } @@ -551,7 +614,7 @@ func (l *LongRunningVM) Run( func (l *LongRunningVM) GetAccount( ctx fvm.Context, address flow.Address, - storageSnapshot state.StorageSnapshot, + storageSnapshot snapshot.StorageSnapshot, ) ( *flow.Account, error, @@ -567,7 +630,7 @@ func (f *FakeBlockComputer) ExecuteBlock( context.Context, flow.Identifier, *entity.ExecutableBlock, - state.StorageSnapshot, + snapshot.StorageSnapshot, *derived.DerivedBlockData, ) ( *execution.ComputationResult, @@ -594,6 +657,7 @@ func TestExecuteScriptTimeout(t *testing.T) { ExecutionTimeLimit: timeout, }, DerivedDataCacheSize: derived.DefaultDerivedDataCacheSize, + MaxConcurrency: 1, }, ) @@ -640,6 +704,7 @@ func TestExecuteScriptCancelled(t *testing.T) { ExecutionTimeLimit: timeout, }, DerivedDataCacheSize: derived.DefaultDerivedDataCacheSize, + MaxConcurrency: 1, }, ) @@ -771,7 +836,8 @@ func Test_EventEncodingFailsOnlyTxAndCarriesOn(t *testing.T) { me, prov, nil, - ) + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) derivedChainData, err := derived.NewDerivedChainData(10) @@ -791,19 +857,23 @@ func Test_EventEncodingFailsOnlyTxAndCarriesOn(t *testing.T) { snapshotTree) require.NoError(t, err) - require.Len(t, returnedComputationResult.Events, 2) // 1 collection + 1 system chunk - require.Len(t, returnedComputationResult.TransactionResults, 4) // 2 txs + 1 system tx + txResults := returnedComputationResult.AllTransactionResults() + require.Len(t, txResults, 4) // 2 txs + 1 system tx + + require.Empty(t, txResults[0].ErrorMessage) + require.Contains(t, txResults[1].ErrorMessage, "I failed encoding") + require.Empty(t, txResults[2].ErrorMessage) - require.Empty(t, returnedComputationResult.TransactionResults[0].ErrorMessage) - require.Contains(t, returnedComputationResult.TransactionResults[1].ErrorMessage, "I failed encoding") - require.Empty(t, returnedComputationResult.TransactionResults[2].ErrorMessage) + colRes := returnedComputationResult.CollectionExecutionResultAt(0) + events := colRes.Events() + require.Len(t, events, 2) // 1 collection + 1 system chunk // first event should be contract deployed - assert.EqualValues(t, "flow.AccountContractAdded", returnedComputationResult.Events[0][0].Type) + assert.EqualValues(t, "flow.AccountContractAdded", events[0].Type) // second event should come from tx3 (index 2) as tx2 (index 1) should fail encoding - hasValidEventValue(t, returnedComputationResult.Events[0][1], 1) - assert.Equal(t, returnedComputationResult.Events[0][1].TransactionIndex, uint32(2)) + hasValidEventValue(t, events[1], 1) + assert.Equal(t, events[1].TransactionIndex, uint32(2)) } type testingEventEncoder struct { @@ -845,6 +915,7 @@ func TestScriptStorageMutationsDiscarded(t *testing.T) { ExecutionTimeLimit: timeout, }, DerivedDataCacheSize: derived.DefaultDerivedDataCacheSize, + MaxConcurrency: 1, }, ) vm := manager.vm @@ -889,11 +960,10 @@ func TestScriptStorageMutationsDiscarded(t *testing.T) { rt := env.BorrowCadenceRuntime() defer env.ReturnCadenceRuntime(rt) - v, err := rt.ReadStored( - commonAddress, - cadence.NewPath("storage", "x"), - ) + path, err := cadence.NewPath(common.PathDomainStorage, "x") + require.NoError(t, err) + v, err := rt.ReadStored(commonAddress, path) // the save should not update account storage by writing the updates // back to the snapshotTree require.NoError(t, err) diff --git a/engine/execution/computation/mock/computation_manager.go b/engine/execution/computation/mock/computation_manager.go index 9f2f3840b60..f019caf61bd 100644 --- a/engine/execution/computation/mock/computation_manager.go +++ b/engine/execution/computation/mock/computation_manager.go @@ -12,7 +12,7 @@ import ( mock "github.com/stretchr/testify/mock" - state "github.com/onflow/flow-go/fvm/state" + snapshot "github.com/onflow/flow-go/fvm/storage/snapshot" ) // ComputationManager is an autogenerated mock type for the ComputationManager type @@ -20,25 +20,25 @@ type ComputationManager struct { mock.Mock } -// ComputeBlock provides a mock function with given fields: ctx, parentBlockExecutionResultID, block, snapshot -func (_m *ComputationManager) ComputeBlock(ctx context.Context, parentBlockExecutionResultID flow.Identifier, block *entity.ExecutableBlock, snapshot state.StorageSnapshot) (*execution.ComputationResult, error) { - ret := _m.Called(ctx, parentBlockExecutionResultID, block, snapshot) +// ComputeBlock provides a mock function with given fields: ctx, parentBlockExecutionResultID, block, _a3 +func (_m *ComputationManager) ComputeBlock(ctx context.Context, parentBlockExecutionResultID flow.Identifier, block *entity.ExecutableBlock, _a3 snapshot.StorageSnapshot) (*execution.ComputationResult, error) { + ret := _m.Called(ctx, parentBlockExecutionResultID, block, _a3) var r0 *execution.ComputationResult var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, state.StorageSnapshot) (*execution.ComputationResult, error)); ok { - return rf(ctx, parentBlockExecutionResultID, block, snapshot) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, snapshot.StorageSnapshot) (*execution.ComputationResult, error)); ok { + return rf(ctx, parentBlockExecutionResultID, block, _a3) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, state.StorageSnapshot) *execution.ComputationResult); ok { - r0 = rf(ctx, parentBlockExecutionResultID, block, snapshot) + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, snapshot.StorageSnapshot) *execution.ComputationResult); ok { + r0 = rf(ctx, parentBlockExecutionResultID, block, _a3) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*execution.ComputationResult) } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, state.StorageSnapshot) error); ok { - r1 = rf(ctx, parentBlockExecutionResultID, block, snapshot) + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, *entity.ExecutableBlock, snapshot.StorageSnapshot) error); ok { + r1 = rf(ctx, parentBlockExecutionResultID, block, _a3) } else { r1 = ret.Error(1) } @@ -46,25 +46,25 @@ func (_m *ComputationManager) ComputeBlock(ctx context.Context, parentBlockExecu return r0, r1 } -// ExecuteScript provides a mock function with given fields: ctx, script, arguments, blockHeader, snapshot -func (_m *ComputationManager) ExecuteScript(ctx context.Context, script []byte, arguments [][]byte, blockHeader *flow.Header, snapshot state.StorageSnapshot) ([]byte, error) { - ret := _m.Called(ctx, script, arguments, blockHeader, snapshot) +// ExecuteScript provides a mock function with given fields: ctx, script, arguments, blockHeader, _a4 +func (_m *ComputationManager) ExecuteScript(ctx context.Context, script []byte, arguments [][]byte, blockHeader *flow.Header, _a4 snapshot.StorageSnapshot) ([]byte, error) { + ret := _m.Called(ctx, script, arguments, blockHeader, _a4) var r0 []byte var r1 error - if rf, ok := ret.Get(0).(func(context.Context, []byte, [][]byte, *flow.Header, state.StorageSnapshot) ([]byte, error)); ok { - return rf(ctx, script, arguments, blockHeader, snapshot) + if rf, ok := ret.Get(0).(func(context.Context, []byte, [][]byte, *flow.Header, snapshot.StorageSnapshot) ([]byte, error)); ok { + return rf(ctx, script, arguments, blockHeader, _a4) } - if rf, ok := ret.Get(0).(func(context.Context, []byte, [][]byte, *flow.Header, state.StorageSnapshot) []byte); ok { - r0 = rf(ctx, script, arguments, blockHeader, snapshot) + if rf, ok := ret.Get(0).(func(context.Context, []byte, [][]byte, *flow.Header, snapshot.StorageSnapshot) []byte); ok { + r0 = rf(ctx, script, arguments, blockHeader, _a4) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]byte) } } - if rf, ok := ret.Get(1).(func(context.Context, []byte, [][]byte, *flow.Header, state.StorageSnapshot) error); ok { - r1 = rf(ctx, script, arguments, blockHeader, snapshot) + if rf, ok := ret.Get(1).(func(context.Context, []byte, [][]byte, *flow.Header, snapshot.StorageSnapshot) error); ok { + r1 = rf(ctx, script, arguments, blockHeader, _a4) } else { r1 = ret.Error(1) } @@ -72,25 +72,25 @@ func (_m *ComputationManager) ExecuteScript(ctx context.Context, script []byte, return r0, r1 } -// GetAccount provides a mock function with given fields: ctx, addr, header, snapshot -func (_m *ComputationManager) GetAccount(ctx context.Context, addr flow.Address, header *flow.Header, snapshot state.StorageSnapshot) (*flow.Account, error) { - ret := _m.Called(ctx, addr, header, snapshot) +// GetAccount provides a mock function with given fields: ctx, addr, header, _a3 +func (_m *ComputationManager) GetAccount(ctx context.Context, addr flow.Address, header *flow.Header, _a3 snapshot.StorageSnapshot) (*flow.Account, error) { + ret := _m.Called(ctx, addr, header, _a3) var r0 *flow.Account var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Address, *flow.Header, state.StorageSnapshot) (*flow.Account, error)); ok { - return rf(ctx, addr, header, snapshot) + if rf, ok := ret.Get(0).(func(context.Context, flow.Address, *flow.Header, snapshot.StorageSnapshot) (*flow.Account, error)); ok { + return rf(ctx, addr, header, _a3) } - if rf, ok := ret.Get(0).(func(context.Context, flow.Address, *flow.Header, state.StorageSnapshot) *flow.Account); ok { - r0 = rf(ctx, addr, header, snapshot) + if rf, ok := ret.Get(0).(func(context.Context, flow.Address, *flow.Header, snapshot.StorageSnapshot) *flow.Account); ok { + r0 = rf(ctx, addr, header, _a3) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*flow.Account) } } - if rf, ok := ret.Get(1).(func(context.Context, flow.Address, *flow.Header, state.StorageSnapshot) error); ok { - r1 = rf(ctx, addr, header, snapshot) + if rf, ok := ret.Get(1).(func(context.Context, flow.Address, *flow.Header, snapshot.StorageSnapshot) error); ok { + r1 = rf(ctx, addr, header, _a3) } else { r1 = ret.Error(1) } diff --git a/engine/execution/computation/programs_test.go b/engine/execution/computation/programs_test.go index 07b94ad5364..8fe46e8ea6e 100644 --- a/engine/execution/computation/programs_test.go +++ b/engine/execution/computation/programs_test.go @@ -10,7 +10,7 @@ import ( dssync "github.com/ipfs/go-datastore/sync" blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/onflow/cadence" - jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/cadence/encoding/ccf" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -21,8 +21,8 @@ import ( "github.com/onflow/flow-go/engine/execution/computation/computer" "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/fvm" - "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/module/executiondatasync/provider" @@ -35,6 +35,10 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) +const ( + testMaxConcurrency = 2 +) + func TestPrograms_TestContractUpdates(t *testing.T) { chain := flow.Mainnet.Chain() vm := fvm.NewVirtualMachine() @@ -133,7 +137,9 @@ func TestPrograms_TestContractUpdates(t *testing.T) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) derivedChainData, err := derived.NewDerivedChainData(10) @@ -151,22 +157,22 @@ func TestPrograms_TestContractUpdates(t *testing.T) { snapshotTree) require.NoError(t, err) - require.Len(t, returnedComputationResult.Events, 2) // 1 collection + 1 system chunk + events := returnedComputationResult.AllEvents() // first event should be contract deployed - assert.EqualValues(t, "flow.AccountContractAdded", returnedComputationResult.Events[0][0].Type) + assert.EqualValues(t, "flow.AccountContractAdded", events[0].Type) // second event should have a value of 1 (since is calling version 1 of contract) - hasValidEventValue(t, returnedComputationResult.Events[0][1], 1) + hasValidEventValue(t, events[1], 1) // third event should be contract updated - assert.EqualValues(t, "flow.AccountContractUpdated", returnedComputationResult.Events[0][2].Type) + assert.EqualValues(t, "flow.AccountContractUpdated", events[2].Type) // 4th event should have a value of 2 (since is calling version 2 of contract) - hasValidEventValue(t, returnedComputationResult.Events[0][3], 2) + hasValidEventValue(t, events[3], 2) // 5th event should have a value of 2 (since is calling version 2 of contract) - hasValidEventValue(t, returnedComputationResult.Events[0][4], 2) + hasValidEventValue(t, events[4], 2) } type blockProvider struct { @@ -243,7 +249,9 @@ func TestPrograms_TestBlockForks(t *testing.T) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(nil), + testMaxConcurrency) require.NoError(t, err) derivedChainData, err := derived.NewDerivedChainData(10) @@ -261,7 +269,7 @@ func TestPrograms_TestBlockForks(t *testing.T) { block1111, block12, block121, block1211 *flow.Block block1Snapshot, block11Snapshot, block111Snapshot, block112Snapshot, - block12Snapshot, block121Snapshot storage.SnapshotTree + block12Snapshot, block121Snapshot snapshot.SnapshotTree ) t.Run("executing block1 (no collection)", func(t *testing.T) { @@ -301,7 +309,8 @@ func TestPrograms_TestBlockForks(t *testing.T) { // cache should include value for this block require.NotNil(t, derivedChainData.Get(block11.ID())) // 1st event should be contract deployed - assert.EqualValues(t, "flow.AccountContractAdded", res.Events[0][0].Type) + + assert.EqualValues(t, "flow.AccountContractAdded", res.AllEvents()[0].Type) }) t.Run("executing block111 (emit event (expected v1), update contract to v3)", func(t *testing.T) { @@ -324,12 +333,13 @@ func TestPrograms_TestBlockForks(t *testing.T) { // cache should include a program for this block require.NotNil(t, derivedChainData.Get(block111.ID())) - require.Len(t, res.Events, 2) + events := res.AllEvents() + require.Equal(t, res.BlockExecutionResult.Size(), 2) // 1st event - hasValidEventValue(t, res.Events[0][0], block111ExpectedValue) + hasValidEventValue(t, events[0], block111ExpectedValue) // second event should be contract deployed - assert.EqualValues(t, "flow.AccountContractUpdated", res.Events[0][1].Type) + assert.EqualValues(t, "flow.AccountContractUpdated", events[1].Type) }) t.Run("executing block1111 (emit event (expected v3))", func(t *testing.T) { @@ -347,10 +357,11 @@ func TestPrograms_TestBlockForks(t *testing.T) { // cache should include a program for this block require.NotNil(t, derivedChainData.Get(block1111.ID())) - require.Len(t, res.Events, 2) + events := res.AllEvents() + require.Equal(t, res.BlockExecutionResult.Size(), 2) // 1st event - hasValidEventValue(t, res.Events[0][0], block1111ExpectedValue) + hasValidEventValue(t, events[0], block1111ExpectedValue) }) t.Run("executing block112 (emit event (expected v1))", func(t *testing.T) { @@ -372,12 +383,13 @@ func TestPrograms_TestBlockForks(t *testing.T) { // cache should include a program for this block require.NotNil(t, derivedChainData.Get(block112.ID())) - require.Len(t, res.Events, 2) + events := res.AllEvents() + require.Equal(t, res.BlockExecutionResult.Size(), 2) // 1st event - hasValidEventValue(t, res.Events[0][0], block112ExpectedValue) + hasValidEventValue(t, events[0], block112ExpectedValue) // second event should be contract deployed - assert.EqualValues(t, "flow.AccountContractUpdated", res.Events[0][1].Type) + assert.EqualValues(t, "flow.AccountContractUpdated", events[1].Type) }) t.Run("executing block1121 (emit event (expected v4))", func(t *testing.T) { @@ -395,10 +407,11 @@ func TestPrograms_TestBlockForks(t *testing.T) { // cache should include a program for this block require.NotNil(t, derivedChainData.Get(block1121.ID())) - require.Len(t, res.Events, 2) + events := res.AllEvents() + require.Equal(t, res.BlockExecutionResult.Size(), 2) // 1st event - hasValidEventValue(t, res.Events[0][0], block1121ExpectedValue) + hasValidEventValue(t, events[0], block1121ExpectedValue) }) t.Run("executing block12 (deploys contract V2)", func(t *testing.T) { @@ -416,9 +429,10 @@ func TestPrograms_TestBlockForks(t *testing.T) { // cache should include a program for this block require.NotNil(t, derivedChainData.Get(block12.ID())) - require.Len(t, res.Events, 2) + events := res.AllEvents() + require.Equal(t, res.BlockExecutionResult.Size(), 2) - assert.EqualValues(t, "flow.AccountContractAdded", res.Events[0][0].Type) + assert.EqualValues(t, "flow.AccountContractAdded", events[0].Type) }) t.Run("executing block121 (emit event (expected V2)", func(t *testing.T) { block121ExpectedValue := 2 @@ -435,10 +449,11 @@ func TestPrograms_TestBlockForks(t *testing.T) { // cache should include a program for this block require.NotNil(t, derivedChainData.Get(block121.ID())) - require.Len(t, res.Events, 2) + events := res.AllEvents() + require.Equal(t, res.BlockExecutionResult.Size(), 2) // 1st event - hasValidEventValue(t, res.Events[0][0], block121ExpectedValue) + hasValidEventValue(t, events[0], block121ExpectedValue) }) t.Run("executing Block1211 (emit event (expected V2)", func(t *testing.T) { block1211ExpectedValue := 2 @@ -457,10 +472,11 @@ func TestPrograms_TestBlockForks(t *testing.T) { // had no change so cache should be equal to parent require.Equal(t, derivedChainData.Get(block121.ID()), derivedChainData.Get(block1211.ID())) - require.Len(t, res.Events, 2) + events := res.AllEvents() + require.Equal(t, res.BlockExecutionResult.Size(), 2) // 1st event - hasValidEventValue(t, res.Events[0][0], block1211ExpectedValue) + hasValidEventValue(t, events[0], block1211ExpectedValue) }) } @@ -470,11 +486,11 @@ func createTestBlockAndRun( engine *Manager, parentBlock *flow.Block, col flow.Collection, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) ( *flow.Block, *execution.ComputationResult, - storage.SnapshotTree, + snapshot.SnapshotTree, ) { guarantee := flow.CollectionGuarantee{ CollectionID: col.ID(), @@ -509,11 +525,11 @@ func createTestBlockAndRun( snapshotTree) require.NoError(t, err) - for _, txResult := range returnedComputationResult.TransactionResults { + for _, txResult := range returnedComputationResult.AllTransactionResults() { require.Empty(t, txResult.ErrorMessage) } - for _, snapshot := range returnedComputationResult.StateSnapshots { + for _, snapshot := range returnedComputationResult.AllExecutionSnapshots() { snapshotTree = snapshotTree.Append(snapshot) } @@ -535,7 +551,7 @@ func prepareTx(t *testing.T, } func hasValidEventValue(t *testing.T, event flow.Event, value int) { - data, err := jsoncdc.Decode(nil, event.Payload) + data, err := ccf.Decode(nil, event.Payload) require.NoError(t, err) assert.Equal(t, int16(value), data.(cadence.Event).Fields[0].ToGoValue()) } diff --git a/engine/execution/computation/query/executor.go b/engine/execution/computation/query/executor.go index 9ac77f030ba..7b5a7eb4b35 100644 --- a/engine/execution/computation/query/executor.go +++ b/engine/execution/computation/query/executor.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "fmt" - "math/rand" "strings" "sync" "time" @@ -13,11 +12,12 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/fvm" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/utils/debug" + "github.com/onflow/flow-go/utils/rand" ) const ( @@ -32,7 +32,7 @@ type Executor interface { script []byte, arguments [][]byte, blockHeader *flow.Header, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) ( []byte, error, @@ -42,7 +42,7 @@ type Executor interface { ctx context.Context, addr flow.Address, header *flow.Header, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) ( *flow.Account, error, @@ -71,7 +71,6 @@ type QueryExecutor struct { vmCtx fvm.Context derivedChainData *derived.DerivedChainData rngLock *sync.Mutex - rng *rand.Rand } var _ Executor = &QueryExecutor{} @@ -92,7 +91,6 @@ func NewQueryExecutor( vmCtx: vmCtx, derivedChainData: derivedChainData, rngLock: &sync.Mutex{}, - rng: rand.New(rand.NewSource(time.Now().UnixNano())), } } @@ -101,7 +99,7 @@ func (e *QueryExecutor) ExecuteScript( script []byte, arguments [][]byte, blockHeader *flow.Header, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) ( encodedValue []byte, err error, @@ -115,8 +113,11 @@ func (e *QueryExecutor) ExecuteScript( // TODO: this is a temporary measure, we could remove this in the future if e.logger.Debug().Enabled() { e.rngLock.Lock() - trackerID := e.rng.Uint32() - e.rngLock.Unlock() + defer e.rngLock.Unlock() + trackerID, err := rand.Uint32() + if err != nil { + return nil, fmt.Errorf("failed to generate trackerID: %w", err) + } trackedLogger := e.logger.With().Hex("script_hex", script).Uint32("trackerID", trackerID).Logger() trackedLogger.Debug().Msg("script is sent for execution") @@ -207,7 +208,7 @@ func (e *QueryExecutor) GetAccount( ctx context.Context, address flow.Address, blockHeader *flow.Header, - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, ) ( *flow.Account, error, diff --git a/engine/execution/computation/result/consumer.go b/engine/execution/computation/result/consumer.go index 685d3a31430..b7218577f10 100644 --- a/engine/execution/computation/result/consumer.go +++ b/engine/execution/computation/result/consumer.go @@ -1,31 +1,96 @@ package result import ( + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" ) -// ExecutedCollection holds results of a collection execution -type ExecutedCollection interface { +type ExecutableCollection interface { // BlockHeader returns the block header in which collection was included BlockHeader() *flow.Header // Collection returns the content of the collection Collection() *flow.Collection - // RegisterUpdates returns all registers that were updated during collection execution - UpdatedRegisters() flow.RegisterEntries + // CollectionIndex returns the index of collection in the block + CollectionIndex() int + + // IsSystemCollection returns true if the collection is the last collection of the block + IsSystemCollection() bool +} + +// ExecutedCollection holds results of a collection execution +type ExecutedCollection interface { + + // Events returns a list of all the events emitted during collection execution + Events() flow.EventsList - // ReadRegisterIDs returns all registers that has been read during collection execution - ReadRegisterIDs() flow.RegisterIDs + // ServiceEventList returns a list of only service events emitted during this collection + ServiceEventList() flow.EventsList - // EmittedEvents returns a list of events emitted during collection execution - EmittedEvents() flow.EventsList + // ConvertedServiceEvents returns a list of converted service events + ConvertedServiceEvents() flow.ServiceEventList // TransactionResults returns a list of transaction results TransactionResults() flow.TransactionResults + + // ExecutionSnapshot returns the execution snapshot + ExecutionSnapshot() *snapshot.ExecutionSnapshot } // ExecutedCollectionConsumer consumes ExecutedCollections type ExecutedCollectionConsumer interface { - OnExecutedCollection(ec ExecutedCollection) error + module.ReadyDoneAware + OnExecutedCollection(res ExecutedCollection) error +} + +// AttestedCollection holds results of a collection attestation +type AttestedCollection interface { + ExecutedCollection + + // StartStateCommitment returns a commitment to the state before collection execution + StartStateCommitment() flow.StateCommitment + + // EndStateCommitment returns a commitment to the state after collection execution + EndStateCommitment() flow.StateCommitment + + // StateProof returns state proofs that could be used to build a partial trie + StateProof() flow.StorageProof + + // TODO(ramtin): unlock these + // // StateDeltaCommitment returns a commitment over the state delta + // StateDeltaCommitment() flow.Identifier + + // // TxResultListCommitment returns a commitment over the list of transaction results + // TxResultListCommitment() flow.Identifier + + // EventCommitment returns commitment over eventList + EventListCommitment() flow.Identifier +} + +// AttestedCollectionConsumer consumes AttestedCollection +type AttestedCollectionConsumer interface { + module.ReadyDoneAware + OnAttestedCollection(ac AttestedCollection) error +} + +type ExecutedBlock interface { + // BlockHeader returns the block header in which collection was included + BlockHeader() *flow.Header + + // Receipt returns the execution receipt + Receipt() *flow.ExecutionReceipt + + // AttestedCollections returns attested collections + // + // TODO(ramtin): this could be reduced, currently we need this + // to store chunk data packs, trie updates package used by access nodes, + AttestedCollections() []AttestedCollection +} + +// ExecutedBlockConsumer consumes ExecutedBlock +type ExecutedBlockConsumer interface { + module.ReadyDoneAware + OnExecutedBlock(eb ExecutedBlock) error } diff --git a/engine/execution/ingestion/ingest_rpc.go b/engine/execution/engines.go similarity index 78% rename from engine/execution/ingestion/ingest_rpc.go rename to engine/execution/engines.go index a0c71c51db4..66d4dbccd57 100644 --- a/engine/execution/ingestion/ingest_rpc.go +++ b/engine/execution/engines.go @@ -1,4 +1,4 @@ -package ingestion +package execution import ( "context" @@ -6,8 +6,8 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// IngestRPC represents the RPC calls that the execution ingest engine exposes to support the Access Node API calls -type IngestRPC interface { +// ScriptExecutor represents the RPC calls that the execution script engine exposes to support the Access Node API calls +type ScriptExecutor interface { // ExecuteScriptAtBlockID executes a script at the given Block id ExecuteScriptAtBlockID(ctx context.Context, script []byte, arguments [][]byte, blockID flow.Identifier) ([]byte, error) diff --git a/engine/execution/ingestion/engine.go b/engine/execution/ingestion/engine.go index 81b34401c84..53ed58c99c6 100644 --- a/engine/execution/ingestion/engine.go +++ b/engine/execution/ingestion/engine.go @@ -2,10 +2,9 @@ package ingestion import ( "context" - "encoding/hex" "errors" "fmt" - "strings" + "regexp" "sync" "time" @@ -15,6 +14,7 @@ import ( "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/execution" "github.com/onflow/flow-go/engine/execution/computation" + "github.com/onflow/flow-go/engine/execution/ingestion/stop" "github.com/onflow/flow-go/engine/execution/ingestion/uploader" "github.com/onflow/flow-go/engine/execution/provider" "github.com/onflow/flow-go/engine/execution/state" @@ -43,6 +43,7 @@ type Engine struct { me module.Local request module.Requester // used to request collections state protocol.State + headers storage.Headers // see comments on getHeaderByHeight for why we need it blocks storage.Blocks collections storage.Collections events storage.Events @@ -59,15 +60,25 @@ type Engine struct { checkAuthorizedAtBlock func(blockID flow.Identifier) (bool, error) executionDataPruner *pruner.Pruner uploader *uploader.Manager - stopControl *StopControl + stopControl *stop.StopControl + + // This is included to temporarily work around an issue observed on a small number of ENs. + // It works around an issue where some collection nodes are not configured with enough + // this works around an issue where some collection nodes are not configured with enough + // file descriptors causing connection failures. + onflowOnlyLNs bool } +var onlyOnflowRegex = regexp.MustCompile(`.*\.onflow\.org:3569$`) + func New( + unit *engine.Unit, logger zerolog.Logger, net network.Network, me module.Local, request module.Requester, state protocol.State, + headers storage.Headers, blocks storage.Blocks, collections storage.Collections, events storage.Events, @@ -82,18 +93,20 @@ func New( checkAuthorizedAtBlock func(blockID flow.Identifier) (bool, error), pruner *pruner.Pruner, uploader *uploader.Manager, - stopControl *StopControl, + stopControl *stop.StopControl, + onflowOnlyLNs bool, ) (*Engine, error) { log := logger.With().Str("engine", "ingestion").Logger() mempool := newMempool() eng := Engine{ - unit: engine.NewUnit(), + unit: unit, log: log, me: me, request: request, state: state, + headers: headers, blocks: blocks, collections: collections, events: events, @@ -111,6 +124,7 @@ func New( executionDataPruner: pruner, uploader: uploader, stopControl: stopControl, + onflowOnlyLNs: onflowOnlyLNs, } return &eng, nil @@ -119,15 +133,17 @@ func New( // Ready returns a channel that will close when the engine has // successfully started. func (e *Engine) Ready() <-chan struct{} { - if !e.stopControl.IsPaused() { - if err := e.uploader.RetryUploads(); err != nil { - e.log.Warn().Msg("failed to re-upload all ComputationResults") - } + if e.stopControl.IsExecutionStopped() { + return e.unit.Ready() + } - err := e.reloadUnexecutedBlocks() - if err != nil { - e.log.Fatal().Err(err).Msg("failed to load all unexecuted blocks") - } + if err := e.uploader.RetryUploads(); err != nil { + e.log.Warn().Msg("failed to re-upload all ComputationResults") + } + + err := e.reloadUnexecutedBlocks() + if err != nil { + e.log.Fatal().Err(err).Msg("failed to load all unexecuted blocks") } return e.unit.Ready() @@ -152,7 +168,11 @@ func (e *Engine) SubmitLocal(event interface{}) { // Submit submits the given event from the node with the given origin ID // for processing in a non-blocking manner. It returns instantly and logs // a potential processing error internally when done. -func (e *Engine) Submit(channel channels.Channel, originID flow.Identifier, event interface{}) { +func (e *Engine) Submit( + channel channels.Channel, + originID flow.Identifier, + event interface{}, +) { e.unit.Launch(func() { err := e.process(originID, event) if err != nil { @@ -166,7 +186,11 @@ func (e *Engine) ProcessLocal(event interface{}) error { return fmt.Errorf("ingestion error does not process local events") } -func (e *Engine) Process(channel channels.Channel, originID flow.Identifier, event interface{}) error { +func (e *Engine) Process( + channel channels.Channel, + originID flow.Identifier, + event interface{}, +) error { return e.unit.Do(func() error { return e.process(originID, event) }) @@ -176,7 +200,10 @@ func (e *Engine) process(originID flow.Identifier, event interface{}) error { return nil } -func (e *Engine) finalizedUnexecutedBlocks(finalized protocol.Snapshot) ([]flow.Identifier, error) { +func (e *Engine) finalizedUnexecutedBlocks(finalized protocol.Snapshot) ( + []flow.Identifier, + error, +) { // get finalized height final, err := finalized.Head() if err != nil { @@ -193,13 +220,15 @@ func (e *Engine) finalizedUnexecutedBlocks(finalized protocol.Snapshot) ([]flow. // blocks. lastExecuted := final.Height - rootBlock, err := e.state.Params().Root() + // dynamically bootstrapped execution node will reload blocks from + // [sealedRoot.Height + 1, finalizedRoot.Height] and execute them on startup. + rootBlock, err := e.state.Params().SealedRoot() if err != nil { return nil, fmt.Errorf("failed to retrieve root block: %w", err) } for ; lastExecuted > rootBlock.Height; lastExecuted-- { - header, err := e.state.AtHeight(lastExecuted).Head() + header, err := e.getHeaderByHeight(lastExecuted) if err != nil { return nil, fmt.Errorf("could not get header at height: %v, %w", lastExecuted, err) } @@ -216,14 +245,12 @@ func (e *Engine) finalizedUnexecutedBlocks(finalized protocol.Snapshot) ([]flow. firstUnexecuted := lastExecuted + 1 - e.log.Info().Msgf("last finalized and executed height: %v", lastExecuted) - unexecuted := make([]flow.Identifier, 0) // starting from the first unexecuted block, go through each unexecuted and finalized block // reload its block to execution queues for height := firstUnexecuted; height <= final.Height; height++ { - header, err := e.state.AtHeight(height).Head() + header, err := e.getHeaderByHeight(height) if err != nil { return nil, fmt.Errorf("could not get header at height: %v, %w", height, err) } @@ -231,10 +258,22 @@ func (e *Engine) finalizedUnexecutedBlocks(finalized protocol.Snapshot) ([]flow. unexecuted = append(unexecuted, header.ID()) } + e.log.Info(). + Uint64("last_finalized", final.Height). + Uint64("last_finalized_executed", lastExecuted). + Uint64("sealed_root_height", rootBlock.Height). + Hex("sealed_root_id", logging.Entity(rootBlock)). + Uint64("first_unexecuted", firstUnexecuted). + Int("total_finalized_unexecuted", len(unexecuted)). + Msgf("finalized unexecuted blocks") + return unexecuted, nil } -func (e *Engine) pendingUnexecutedBlocks(finalized protocol.Snapshot) ([]flow.Identifier, error) { +func (e *Engine) pendingUnexecutedBlocks(finalized protocol.Snapshot) ( + []flow.Identifier, + error, +) { pendings, err := finalized.Descendants() if err != nil { return nil, fmt.Errorf("could not get pending blocks: %w", err) @@ -256,7 +295,11 @@ func (e *Engine) pendingUnexecutedBlocks(finalized protocol.Snapshot) ([]flow.Id return unexecuted, nil } -func (e *Engine) unexecutedBlocks() (finalized []flow.Identifier, pending []flow.Identifier, err error) { +func (e *Engine) unexecutedBlocks() ( + finalized []flow.Identifier, + pending []flow.Identifier, + err error, +) { // pin the snapshot so that finalizedUnexecutedBlocks and pendingUnexecutedBlocks are based // on the same snapshot. snapshot := e.state.Final() @@ -286,7 +329,8 @@ func (e *Engine) reloadUnexecutedBlocks() error { // is called before reloading is finished, it will be blocked, which will avoid that edge case. return e.mempool.Run(func( blockByCollection *stdmap.BlockByCollectionBackdata, - executionQueues *stdmap.QueuesBackdata) error { + executionQueues *stdmap.QueuesBackdata, + ) error { // saving an executed block is currently not transactional, so it's possible // the block is marked as executed but the receipt might not be saved during a crash. @@ -302,13 +346,13 @@ func (e *Engine) reloadUnexecutedBlocks() error { return fmt.Errorf("could not get last executed: %w", err) } - last, err := e.state.AtBlockID(lastExecutedID).Head() + last, err := e.headers.ByBlockID(lastExecutedID) if err != nil { return fmt.Errorf("could not get last executed final by ID: %w", err) } // don't reload root block - rootBlock, err := e.state.Params().Root() + rootBlock, err := e.state.Params().SealedRoot() if err != nil { return fmt.Errorf("failed to retrieve root block: %w", err) } @@ -367,7 +411,8 @@ func (e *Engine) reloadUnexecutedBlocks() error { func (e *Engine) reloadBlock( blockByCollection *stdmap.BlockByCollectionBackdata, executionQueues *stdmap.QueuesBackdata, - blockID flow.Identifier) error { + blockID flow.Identifier, +) error { block, err := e.blocks.ByID(blockID) if err != nil { return fmt.Errorf("could not get block by ID: %v %w", blockID, err) @@ -404,8 +449,10 @@ func (e *Engine) reloadBlock( // NOTE: Ready calls reloadUnexecutedBlocks during initialization, which handles dropped protocol events. func (e *Engine) BlockProcessable(b *flow.Header, _ *flow.QuorumCertificate) { + // TODO: this should not be blocking: https://github.com/onflow/flow-go/issues/4400 + // skip if stopControl tells to skip - if !e.stopControl.blockProcessable(b) { + if !e.stopControl.ShouldExecuteBlock(b) { return } @@ -425,12 +472,6 @@ func (e *Engine) BlockProcessable(b *flow.Header, _ *flow.QuorumCertificate) { } } -// BlockFinalized implements part of state.protocol.Consumer interface. -// Method gets called for every finalized block -func (e *Engine) BlockFinalized(h *flow.Header) { - e.stopControl.blockFinalized(e.unit.Ctx(), e.execState, h) -} - // Main handling // handle block will process the incoming block. @@ -479,7 +520,8 @@ func (e *Engine) enqueueBlockAndCheckExecutable( blockByCollection *stdmap.BlockByCollectionBackdata, executionQueues *stdmap.QueuesBackdata, block *flow.Block, - checkStateSync bool) ([]*flow.CollectionGuarantee, error) { + checkStateSync bool, +) ([]*flow.CollectionGuarantee, error) { executableBlock := &entity.ExecutableBlock{ Block: block, CompleteCollections: make(map[flow.Identifier]*entity.CompleteCollection), @@ -574,8 +616,6 @@ func (e *Engine) executeBlock( startedAt := time.Now() - e.stopControl.executingBlockHeight(executableBlock.Block.Header.Height) - span, ctx := e.tracer.StartSpanFromContext(ctx, trace.EXEExecuteBlock) defer span.End() @@ -648,11 +688,12 @@ func (e *Engine) executeBlock( } } + finalEndState := computationResult.CurrentEndState() lg.Info(). Hex("parent_block", executableBlock.Block.Header.ParentID[:]). Int("collections", len(executableBlock.Block.Payload.Guarantees)). Hex("start_state", executableBlock.StartState[:]). - Hex("final_state", computationResult.EndState[:]). + Hex("final_state", finalEndState[:]). Hex("receipt_id", logging.Entity(receipt)). Hex("result_id", logging.Entity(receipt.ExecutionResult)). Hex("execution_data_id", receipt.ExecutionResult.ExecutionDataID[:]). @@ -661,11 +702,7 @@ func (e *Engine) executeBlock( Int64("timeSpentInMS", time.Since(startedAt).Milliseconds()). Msg("block executed") - for computationKind, intensity := range computationResult.ComputationIntensities { - e.metrics.ExecutionBlockExecutionEffortVectorComponent(computationKind.String(), intensity) - } - - err = e.onBlockExecuted(executableBlock, computationResult.EndState) + err = e.onBlockExecuted(executableBlock, finalEndState) if err != nil { lg.Err(err).Msg("failed in process block's children") } @@ -674,9 +711,10 @@ func (e *Engine) executeBlock( e.executionDataPruner.NotifyFulfilledHeight(executableBlock.Height()) } + e.stopControl.OnBlockExecuted(executableBlock.Block.Header) + e.unit.Ctx() - e.stopControl.blockExecuted(executableBlock.Block.Header) } // we've executed the block, now we need to check: @@ -695,7 +733,10 @@ func (e *Engine) executeBlock( // 13 // 14 <- 15 <- 16 -func (e *Engine) onBlockExecuted(executed *entity.ExecutableBlock, finalState flow.StateCommitment) error { +func (e *Engine) onBlockExecuted( + executed *entity.ExecutableBlock, + finalState flow.StateCommitment, +) error { e.metrics.ExecutionStorageStateCommitment(int64(len(finalState))) e.metrics.ExecutionLastExecutedBlockHeight(executed.Block.Header.Height) @@ -711,6 +752,7 @@ func (e *Engine) onBlockExecuted(executed *entity.ExecutableBlock, finalState fl // find the block that was just executed executionQueue, exists := executionQueues.ByID(executed.ID()) if !exists { + logQueueState(e.log, executionQueues, executed.ID()) // when the block no longer exists in the queue, it means there was a race condition that // two onBlockExecuted was called for the same block, and one process has already removed the // block from the queue, so we will print an error here @@ -766,8 +808,9 @@ func (e *Engine) onBlockExecuted(executed *entity.ExecutableBlock, finalState fl }) if err != nil { - e.log.Err(err). + e.log.Fatal().Err(err). Hex("block", logging.Entity(executed)). + Uint64("height", executed.Block.Header.Height). Msg("error while requeueing blocks after execution") } @@ -833,7 +876,10 @@ func (e *Engine) OnCollection(originID flow.Identifier, entity flow.Entity) { // find all the blocks that are needing this collection, and then // check if any of these block becomes executable and execute it if // is. -func (e *Engine) handleCollection(originID flow.Identifier, collection *flow.Collection) error { +func (e *Engine) handleCollection( + originID flow.Identifier, + collection *flow.Collection, +) error { collID := collection.ID() span, _ := e.tracer.StartCollectionSpan(context.Background(), collID, trace.EXEHandleCollection) @@ -859,7 +905,10 @@ func (e *Engine) handleCollection(originID flow.Identifier, collection *flow.Col ) } -func (e *Engine) addCollectionToMempool(collection *flow.Collection, backdata *stdmap.BlockByCollectionBackdata) error { +func (e *Engine) addCollectionToMempool( + collection *flow.Collection, + backdata *stdmap.BlockByCollectionBackdata, +) error { collID := collection.ID() blockByCollectionID, exists := backdata.ByID(collID) @@ -910,7 +959,10 @@ func (e *Engine) addCollectionToMempool(collection *flow.Collection, backdata *s return nil } -func newQueue(blockify queue.Blockify, queues *stdmap.QueuesBackdata) (*queue.Queue, bool) { +func newQueue(blockify queue.Blockify, queues *stdmap.QueuesBackdata) ( + *queue.Queue, + bool, +) { q := queue.NewQueue(blockify) qID := q.ID() return q, queues.Add(qID, q) @@ -940,7 +992,11 @@ func newQueue(blockify queue.Blockify, queues *stdmap.QueuesBackdata) (*queue.Qu // A <- B <- C // ^- D <- E // G -func enqueue(blockify queue.Blockify, queues *stdmap.QueuesBackdata) (*queue.Queue, bool, bool) { +func enqueue(blockify queue.Blockify, queues *stdmap.QueuesBackdata) ( + *queue.Queue, + bool, + bool, +) { for _, queue := range queues.All() { if stored, isNew := queue.TryAdd(blockify); stored { return queue, isNew, false @@ -1004,92 +1060,6 @@ func (e *Engine) matchAndFindMissingCollections( return missingCollections, nil } -func (e *Engine) ExecuteScriptAtBlockID(ctx context.Context, script []byte, arguments [][]byte, blockID flow.Identifier) ([]byte, error) { - - stateCommit, err := e.execState.StateCommitmentByBlockID(ctx, blockID) - if err != nil { - return nil, fmt.Errorf("failed to get state commitment for block (%s): %w", blockID, err) - } - - // return early if state with the given state commitment is not in memory - // and already purged. This reduces allocations for scripts targeting old blocks. - if !e.execState.HasState(stateCommit) { - return nil, fmt.Errorf("failed to execute script at block (%s): state commitment not found (%s). this error usually happens if the reference block for this script is not set to a recent block", blockID.String(), hex.EncodeToString(stateCommit[:])) - } - - block, err := e.state.AtBlockID(blockID).Head() - if err != nil { - return nil, fmt.Errorf("failed to get block (%s): %w", blockID, err) - } - - blockSnapshot := e.execState.NewStorageSnapshot(stateCommit) - - if e.extensiveLogging { - args := make([]string, 0) - for _, a := range arguments { - args = append(args, hex.EncodeToString(a)) - } - e.log.Debug(). - Hex("block_id", logging.ID(blockID)). - Uint64("block_height", block.Height). - Hex("state_commitment", stateCommit[:]). - Hex("script_hex", script). - Str("args", strings.Join(args[:], ",")). - Msg("extensive log: executed script content") - } - return e.computationManager.ExecuteScript( - ctx, - script, - arguments, - block, - blockSnapshot) -} - -func (e *Engine) GetRegisterAtBlockID(ctx context.Context, owner, key []byte, blockID flow.Identifier) ([]byte, error) { - - stateCommit, err := e.execState.StateCommitmentByBlockID(ctx, blockID) - if err != nil { - return nil, fmt.Errorf("failed to get state commitment for block (%s): %w", blockID, err) - } - - blockSnapshot := e.execState.NewStorageSnapshot(stateCommit) - - id := flow.NewRegisterID(string(owner), string(key)) - data, err := blockSnapshot.Get(id) - if err != nil { - return nil, fmt.Errorf("failed to get the register (%s): %w", id, err) - } - - return data, nil -} - -func (e *Engine) GetAccount(ctx context.Context, addr flow.Address, blockID flow.Identifier) (*flow.Account, error) { - stateCommit, err := e.execState.StateCommitmentByBlockID(ctx, blockID) - if err != nil { - return nil, fmt.Errorf("failed to get state commitment for block (%s): %w", blockID, err) - } - - // return early if state with the given state commitment is not in memory - // and already purged. This reduces allocations for get accounts targeting old blocks. - if !e.execState.HasState(stateCommit) { - return nil, fmt.Errorf( - "failed to get account at block (%s): state commitment not "+ - "found (%s). this error usually happens if the reference "+ - "block for this script is not set to a recent block.", - blockID.String(), - hex.EncodeToString(stateCommit[:])) - } - - block, err := e.state.AtBlockID(blockID).Head() - if err != nil { - return nil, fmt.Errorf("failed to get block (%s): %w", blockID, err) - } - - blockSnapshot := e.execState.NewStorageSnapshot(stateCommit) - - return e.computationManager.GetAccount(ctx, addr, block, blockSnapshot) -} - // save the execution result of a block func (e *Engine) saveExecutionResults( ctx context.Context, @@ -1106,7 +1076,7 @@ func (e *Engine) saveExecutionResults( e.log.Info(). Uint64("block_height", result.ExecutableBlock.Height()). Hex("block_id", logging.Entity(result.ExecutableBlock)). - Str("event_type", event.Type). + Str("event_type", event.Type.String()). Msg("service event emitted") } @@ -1115,10 +1085,11 @@ func (e *Engine) saveExecutionResults( return fmt.Errorf("cannot persist execution state: %w", err) } + finalEndState := result.CurrentEndState() e.log.Debug(). Hex("block_id", logging.Entity(result.ExecutableBlock)). Hex("start_state", result.ExecutableBlock.StartState[:]). - Hex("final_state", result.EndState[:]). + Hex("final_state", finalEndState[:]). Msg("saved computation results") return nil @@ -1157,7 +1128,11 @@ func (e *Engine) logExecutableBlock(eb *entity.ExecutableBlock) { // addOrFetch checks if there are stored collections for the given guarantees, if there is, // forward them to mempool to process the collection, otherwise fetch the collections. // any error returned are exception -func (e *Engine) addOrFetch(blockID flow.Identifier, height uint64, guarantees []*flow.CollectionGuarantee) error { +func (e *Engine) addOrFetch( + blockID flow.Identifier, + height uint64, + guarantees []*flow.CollectionGuarantee, +) error { return e.fetchAndHandleCollection(blockID, height, guarantees, func(collection *flow.Collection) error { err := e.mempool.BlockByCollection.Run( func(backdata *stdmap.BlockByCollectionBackdata) error { @@ -1219,7 +1194,11 @@ func (e *Engine) fetchAndHandleCollection( // fetchCollection takes a guarantee and forwards to requester engine for fetching the collection // any error returned are fatal error -func (e *Engine) fetchCollection(blockID flow.Identifier, height uint64, guarantee *flow.CollectionGuarantee) error { +func (e *Engine) fetchCollection( + blockID flow.Identifier, + height uint64, + guarantee *flow.CollectionGuarantee, +) error { e.log.Debug(). Hex("block", blockID[:]). Hex("collection_id", logging.ID(guarantee.ID())). @@ -1238,10 +1217,45 @@ func (e *Engine) fetchCollection(blockID flow.Identifier, height uint64, guarant ) return fmt.Errorf("could not find guarantors: %w", err) } + + filters := []flow.IdentityFilter{ + filter.HasNodeID(guarantors...), + } + + // This is included to temporarily work around an issue observed on a small number of ENs. + // It works around an issue where some collection nodes are not configured with enough + // file descriptors causing connection failures. This will be removed once a + // proper fix is in place. + if e.onflowOnlyLNs { + // func(Identity("verification-049.mainnet20.nodes.onflow.org:3569")) => true + // func(Identity("verification-049.hello.org:3569")) => false + filters = append(filters, func(identity *flow.Identity) bool { + return onlyOnflowRegex.MatchString(identity.Address) + }) + } + // queue the collection to be requested from one of the guarantors e.request.EntityByID(guarantee.ID(), filter.And( - filter.HasNodeID(guarantors...), + filters..., )) return nil } + +// if the EN is dynamically bootstrapped, the finalized blocks at height range: +// [ sealedRoot.Height, finalizedRoot.Height - 1] can not be retrieved from +// protocol state, but only from headers +func (e *Engine) getHeaderByHeight(height uint64) (*flow.Header, error) { + // we don't use protocol state because for dynamic boostrapped execution node + // the last executed and sealed block is below the finalized root block + return e.headers.ByHeight(height) +} + +func logQueueState(log zerolog.Logger, queues *stdmap.QueuesBackdata, blockID flow.Identifier) { + all := queues.All() + + log.With().Hex("queue_state__executed_block_id", blockID[:]).Int("count", len(all)).Logger() + for i, queue := range all { + log.Error().Msgf("%v-th queue state: %v", i, queue.String()) + } +} diff --git a/engine/execution/ingestion/engine_test.go b/engine/execution/ingestion/engine_test.go index 0adb344e801..a9afec0edee 100644 --- a/engine/execution/ingestion/engine_test.go +++ b/engine/execution/ingestion/engine_test.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "fmt" mathRand "math/rand" - "strings" "sync" "testing" "time" @@ -19,8 +18,10 @@ import ( "github.com/onflow/flow-go/crypto" + enginePkg "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/execution" computation "github.com/onflow/flow-go/engine/execution/computation/mock" + "github.com/onflow/flow-go/engine/execution/ingestion/stop" "github.com/onflow/flow-go/engine/execution/ingestion/uploader" uploadermock "github.com/onflow/flow-go/engine/execution/ingestion/uploader/mock" provider "github.com/onflow/flow-go/engine/execution/provider/mock" @@ -108,6 +109,7 @@ func (es *mockExecutionState) ExecuteBlock(t *testing.T, block *flow.Block) { type testingContext struct { t *testing.T engine *Engine + headers *storage.MockHeaders blocks *storage.MockBlocks collections *storage.MockCollections state *protocol.State @@ -121,7 +123,7 @@ type testingContext struct { broadcastedReceipts map[flow.Identifier]*flow.ExecutionReceipt collectionRequester *module.MockRequester identities flow.IdentityList - stopControl *StopControl + stopControl *stop.StopControl uploadMgr *uploader.Manager mu *sync.Mutex @@ -148,6 +150,7 @@ func runWithEngine(t *testing.T, f func(testingContext)) { myIdentity.StakingPubKey = sk.PublicKey() me := mocklocal.NewMockLocal(sk, myIdentity.ID(), t) + headers := storage.NewMockHeaders(ctrl) blocks := storage.NewMockBlocks(ctrl) payloads := storage.NewMockPayloads(ctrl) collections := storage.NewMockCollections(ctrl) @@ -200,16 +203,30 @@ func runWithEngine(t *testing.T, f func(testingContext)) { return stateProtocol.IsNodeAuthorizedAt(protocolState.AtBlockID(blockID), myIdentity.NodeID) } - stopControl := NewStopControl(zerolog.Nop(), false, 0) + unit := enginePkg.NewUnit() + stopControl := stop.NewStopControl( + unit, + time.Second, + zerolog.Nop(), + executionState, + headers, + nil, + nil, + &flow.Header{Height: 1}, + false, + false, + ) uploadMgr := uploader.NewManager(trace.NewNoopTracer()) engine, err = New( + unit, log, net, me, request, protocolState, + headers, blocks, collections, events, @@ -225,12 +242,14 @@ func runWithEngine(t *testing.T, f func(testingContext)) { nil, uploadMgr, stopControl, + false, ) require.NoError(t, err) f(testingContext{ t: t, engine: engine, + headers: headers, blocks: blocks, collections: collections, state: protocolState, @@ -296,7 +315,7 @@ func (ctx *testingContext) assertSuccessfulBlockComputation( func(args mock.Arguments) { result := args[1].(*execution.ComputationResult) blockID := result.ExecutableBlock.Block.Header.ID() - commit := result.EndState + commit := result.CurrentEndState() ctx.mu.Lock() commits[blockID] = commit @@ -315,6 +334,11 @@ func (ctx *testingContext) assertSuccessfulBlockComputation( Run(func(args mock.Arguments) { receipt := args[1].(*flow.ExecutionReceipt) + assert.Equal(ctx.t, + len(computationResult.ServiceEvents), + len(receipt.ExecutionResult.ServiceEvents), + ) + ctx.mu.Lock() ctx.broadcastedReceipts[receipt.ExecutionResult.BlockID] = receipt ctx.mu.Unlock() @@ -419,8 +443,7 @@ func TestExecuteOneBlock(t *testing.T) { // A <- B blockA := unittest.BlockHeaderFixture() - blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA) - blockB.StartState = unittest.StateCommitmentPointerFixture() + blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA, unittest.StateCommitmentPointerFixture()) ctx.mockHasWeightAtBlockID(blockA.ID(), true) ctx.mockHasWeightAtBlockID(blockB.ID(), true) @@ -453,7 +476,7 @@ func TestExecuteOneBlock(t *testing.T) { unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second) - _, more := <-ctx.engine.Done() //wait for all the blocks to be processed + _, more := <-ctx.engine.Done() // wait for all the blocks to be processed require.False(t, more) _, ok := commits[blockB.ID()] @@ -487,17 +510,14 @@ func Test_OnlyHeadOfTheQueueIsExecuted(t *testing.T) { }) // last executed block - it will be re-queued regardless of state commit - blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA) - blockB.StartState = unittest.StateCommitmentPointerFixture() + blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA, unittest.StateCommitmentPointerFixture()) // finalized block - it can be executed in parallel, as blockB has been executed // and this should be fixed - blockC := unittest.ExecutableBlockFixtureWithParent(nil, blockB.Block.Header) - blockC.StartState = blockB.StartState + blockC := unittest.ExecutableBlockFixtureWithParent(nil, blockB.Block.Header, blockB.StartState) // expected to be executed afterwards - blockD := unittest.ExecutableBlockFixtureWithParent(nil, blockC.Block.Header) - blockD.StartState = blockC.StartState + blockD := unittest.ExecutableBlockFixtureWithParent(nil, blockC.Block.Header, blockC.StartState) logBlocks(map[string]*entity.ExecutableBlock{ "B": blockB, @@ -508,7 +528,6 @@ func Test_OnlyHeadOfTheQueueIsExecuted(t *testing.T) { commits := make(map[flow.Identifier]flow.StateCommitment) commits[blockB.Block.Header.ParentID] = *blockB.StartState commits[blockC.Block.Header.ParentID] = *blockC.StartState - //ctx.mockStateCommitsWithMap(commits) wg := sync.WaitGroup{} @@ -610,7 +629,7 @@ func Test_OnlyHeadOfTheQueueIsExecuted(t *testing.T) { ctx.state.On("AtHeight", blockC.Height()).Return(blockCSnapshot) ctx.state.On("Params").Return(params) - params.On("Root").Return(&blockA, nil) + params.On("FinalizedRoot").Return(&blockA, nil) <-ctx.engine.Ready() @@ -620,7 +639,7 @@ func Test_OnlyHeadOfTheQueueIsExecuted(t *testing.T) { unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second) - _, more := <-ctx.engine.Done() //wait for all the blocks to be processed + _, more := <-ctx.engine.Done() // wait for all the blocks to be processed require.False(t, more) _, ok := commits[blockB.ID()] @@ -643,13 +662,10 @@ func TestBlocksArentExecutedMultipleTimes_multipleBlockEnqueue(t *testing.T) { // A <- B <- C blockA := unittest.BlockHeaderFixture() - blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA) - blockB.StartState = unittest.StateCommitmentPointerFixture() - - //blockCstartState := unittest.StateCommitmentFixture() + blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA, unittest.StateCommitmentPointerFixture()) - blockC := unittest.ExecutableBlockFixtureWithParent([][]flow.Identifier{{colSigner}}, blockB.Block.Header) - blockC.StartState = blockB.StartState //blocks are empty, so no state change is expected + // blocks are empty, so no state change is expected + blockC := unittest.ExecutableBlockFixtureWithParent([][]flow.Identifier{{colSigner}}, blockB.Block.Header, blockB.StartState) logBlocks(map[string]*entity.ExecutableBlock{ "B": blockB, @@ -738,7 +754,7 @@ func TestBlocksArentExecutedMultipleTimes_multipleBlockEnqueue(t *testing.T) { unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second) - _, more := <-ctx.engine.Done() //wait for all the blocks to be processed + _, more := <-ctx.engine.Done() // wait for all the blocks to be processed require.False(t, more) _, ok := commits[blockB.ID()] @@ -762,13 +778,12 @@ func TestBlocksArentExecutedMultipleTimes_collectionArrival(t *testing.T) { // A (0 collection) <- B (0 collection) <- C (0 collection) <- D (1 collection) blockA := unittest.BlockHeaderFixture() - blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA) - blockB.StartState = unittest.StateCommitmentPointerFixture() + blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA, unittest.StateCommitmentPointerFixture()) collectionIdentities := ctx.identities.Filter(filter.HasRole(flow.RoleCollection)) colSigner := collectionIdentities[0].ID() - blockC := unittest.ExecutableBlockFixtureWithParent([][]flow.Identifier{{colSigner}}, blockB.Block.Header) - blockC.StartState = blockB.StartState //blocks are empty, so no state change is expected + // blocks are empty, so no state change is expected + blockC := unittest.ExecutableBlockFixtureWithParent([][]flow.Identifier{{colSigner}}, blockB.Block.Header, blockB.StartState) // the default fixture uses a 10 collectors committee, but in this test case, there are only 4, // so we need to update the signer indices. // set the first identity as signer @@ -780,8 +795,7 @@ func TestBlocksArentExecutedMultipleTimes_collectionArrival(t *testing.T) { blockC.Block.Payload.Guarantees[0].SignerIndices = indices // block D to make sure execution resumes after block C multiple execution has been prevented - blockD := unittest.ExecutableBlockFixtureWithParent(nil, blockC.Block.Header) - blockD.StartState = blockC.StartState + blockD := unittest.ExecutableBlockFixtureWithParent(nil, blockC.Block.Header, blockC.StartState) logBlocks(map[string]*entity.ExecutableBlock{ "B": blockB, @@ -890,7 +904,7 @@ func TestBlocksArentExecutedMultipleTimes_collectionArrival(t *testing.T) { unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second) - _, more := <-ctx.engine.Done() //wait for all the blocks to be processed + _, more := <-ctx.engine.Done() // wait for all the blocks to be processed require.False(t, more) _, ok := commits[blockB.ID()] @@ -921,21 +935,16 @@ func TestExecuteBlockInOrder(t *testing.T) { blockSealed := unittest.BlockHeaderFixture() blocks := make(map[string]*entity.ExecutableBlock) - blocks["A"] = unittest.ExecutableBlockFixtureWithParent(nil, blockSealed) - blocks["A"].StartState = unittest.StateCommitmentPointerFixture() + blocks["A"] = unittest.ExecutableBlockFixtureWithParent(nil, blockSealed, unittest.StateCommitmentPointerFixture()) - blocks["B"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["A"].Block.Header) - blocks["C"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["A"].Block.Header) - blocks["D"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["C"].Block.Header) + // none of the blocks has any collection, so state is essentially the same + blocks["B"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["A"].Block.Header, blocks["A"].StartState) + blocks["C"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["A"].Block.Header, blocks["A"].StartState) + blocks["D"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["C"].Block.Header, blocks["C"].StartState) // log the blocks, so that we can link the block ID in the log with the blocks in tests logBlocks(blocks) - // none of the blocks has any collection, so state is essentially the same - blocks["C"].StartState = blocks["A"].StartState - blocks["B"].StartState = blocks["A"].StartState - blocks["D"].StartState = blocks["C"].StartState - commits := make(map[flow.Identifier]flow.StateCommitment) commits[blocks["A"].Block.Header.ParentID] = *blocks["A"].StartState @@ -1011,7 +1020,7 @@ func TestExecuteBlockInOrder(t *testing.T) { // wait until all 4 blocks have been executed unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second) - _, more := <-ctx.engine.Done() //wait for all the blocks to be processed + _, more := <-ctx.engine.Done() // wait for all the blocks to be processed assert.False(t, more) var ok bool @@ -1025,8 +1034,8 @@ func TestExecuteBlockInOrder(t *testing.T) { require.True(t, ok) // make sure no stopping has been engaged, as it was not set - stopState := ctx.stopControl.GetState() - require.Equal(t, stopState, StopControlOff) + require.False(t, ctx.stopControl.IsExecutionStopped()) + require.False(t, ctx.stopControl.GetStopParameters().Set()) }) } @@ -1036,25 +1045,22 @@ func TestStopAtHeight(t *testing.T) { blockSealed := unittest.BlockHeaderFixture() blocks := make(map[string]*entity.ExecutableBlock) - blocks["A"] = unittest.ExecutableBlockFixtureWithParent(nil, blockSealed) - blocks["A"].StartState = unittest.StateCommitmentPointerFixture() + blocks["A"] = unittest.ExecutableBlockFixtureWithParent(nil, blockSealed, unittest.StateCommitmentPointerFixture()) - blocks["B"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["A"].Block.Header) - blocks["C"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["B"].Block.Header) - blocks["D"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["C"].Block.Header) + // none of the blocks has any collection, so state is essentially the same + blocks["B"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["A"].Block.Header, blocks["A"].StartState) + blocks["C"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["B"].Block.Header, blocks["A"].StartState) + blocks["D"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["C"].Block.Header, blocks["A"].StartState) // stop at block C - _, _, err := ctx.stopControl.SetStopHeight(blockSealed.Height+3, false) + err := ctx.stopControl.SetStopParameters(stop.StopParameters{ + StopBeforeHeight: blockSealed.Height + 3, + }) require.NoError(t, err) // log the blocks, so that we can link the block ID in the log with the blocks in tests logBlocks(blocks) - // none of the blocks has any collection, so state is essentially the same - blocks["B"].StartState = blocks["A"].StartState - blocks["C"].StartState = blocks["A"].StartState - blocks["D"].StartState = blocks["A"].StartState - commits := make(map[flow.Identifier]flow.StateCommitment) commits[blocks["A"].Block.Header.ParentID] = *blocks["A"].StartState @@ -1095,7 +1101,7 @@ func TestStopAtHeight(t *testing.T) { *blocks["B"].StartState, nil) - assert.False(t, ctx.stopControl.IsPaused()) + assert.False(t, ctx.stopControl.IsExecutionStopped()) wg.Add(1) ctx.engine.BlockProcessable(blocks["A"].Block.Header, nil) @@ -1109,18 +1115,18 @@ func TestStopAtHeight(t *testing.T) { unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second) // we don't pause until a block has been finalized - assert.False(t, ctx.stopControl.IsPaused()) + assert.False(t, ctx.stopControl.IsExecutionStopped()) - ctx.engine.BlockFinalized(blocks["A"].Block.Header) - ctx.engine.BlockFinalized(blocks["B"].Block.Header) + ctx.stopControl.BlockFinalizedForTesting(blocks["A"].Block.Header) + ctx.stopControl.BlockFinalizedForTesting(blocks["B"].Block.Header) - assert.False(t, ctx.stopControl.IsPaused()) - ctx.engine.BlockFinalized(blocks["C"].Block.Header) - assert.True(t, ctx.stopControl.IsPaused()) + assert.False(t, ctx.stopControl.IsExecutionStopped()) + ctx.stopControl.BlockFinalizedForTesting(blocks["C"].Block.Header) + assert.True(t, ctx.stopControl.IsExecutionStopped()) - ctx.engine.BlockFinalized(blocks["D"].Block.Header) + ctx.stopControl.BlockFinalizedForTesting(blocks["D"].Block.Header) - _, more := <-ctx.engine.Done() //wait for all the blocks to be processed + _, more := <-ctx.engine.Done() // wait for all the blocks to be processed assert.False(t, more) var ok bool @@ -1169,14 +1175,14 @@ func TestStopAtHeightRaceFinalization(t *testing.T) { blockSealed := unittest.BlockHeaderFixture() blocks := make(map[string]*entity.ExecutableBlock) - blocks["A"] = unittest.ExecutableBlockFixtureWithParent(nil, blockSealed) - blocks["A"].StartState = unittest.StateCommitmentPointerFixture() - - blocks["B"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["A"].Block.Header) - blocks["C"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["B"].Block.Header) + blocks["A"] = unittest.ExecutableBlockFixtureWithParent(nil, blockSealed, unittest.StateCommitmentPointerFixture()) + blocks["B"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["A"].Block.Header, nil) + blocks["C"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["B"].Block.Header, nil) // stop at block B, so B-1 (A) will be last executed - _, _, err := ctx.stopControl.SetStopHeight(blocks["B"].Height(), false) + err := ctx.stopControl.SetStopParameters(stop.StopParameters{ + StopBeforeHeight: blocks["B"].Height(), + }) require.NoError(t, err) // log the blocks, so that we can link the block ID in the log with the blocks in tests @@ -1226,24 +1232,24 @@ func TestStopAtHeightRaceFinalization(t *testing.T) { *blocks["A"].StartState, nil) - assert.False(t, ctx.stopControl.IsPaused()) + assert.False(t, ctx.stopControl.IsExecutionStopped()) executionWg.Add(1) ctx.engine.BlockProcessable(blocks["A"].Block.Header, nil) ctx.engine.BlockProcessable(blocks["B"].Block.Header, nil) - assert.False(t, ctx.stopControl.IsPaused()) + assert.False(t, ctx.stopControl.IsExecutionStopped()) finalizationWg.Add(1) - ctx.engine.BlockFinalized(blocks["B"].Block.Header) + ctx.stopControl.BlockFinalizedForTesting(blocks["B"].Block.Header) finalizationWg.Wait() executionWg.Wait() - _, more := <-ctx.engine.Done() //wait for all the blocks to be processed + _, more := <-ctx.engine.Done() // wait for all the blocks to be processed assert.False(t, more) - assert.True(t, ctx.stopControl.IsPaused()) + assert.True(t, ctx.stopControl.IsExecutionStopped()) var ok bool @@ -1284,15 +1290,18 @@ func TestExecutionGenerationResultsAreChained(t *testing.T) { ctrl := gomock.NewController(t) me := module.NewMockLocal(ctrl) - executableBlock := unittest.ExecutableBlockFixture([][]flow.Identifier{{collection1Identity.NodeID}, {collection1Identity.NodeID}}) + startState := unittest.StateCommitmentFixture() + executableBlock := unittest.ExecutableBlockFixture( + [][]flow.Identifier{{collection1Identity.NodeID}, + {collection1Identity.NodeID}}, + &startState, + ) previousExecutionResultID := unittest.IdentifierFixture() cr := executionUnittest.ComputationResultFixture( previousExecutionResultID, nil) cr.ExecutableBlock = executableBlock - startState := unittest.StateCommitmentFixture() - cr.ExecutableBlock.StartState = &startState execState. On("SaveExecutionResults", mock.Anything, cr). @@ -1311,75 +1320,6 @@ func TestExecutionGenerationResultsAreChained(t *testing.T) { execState.AssertExpectations(t) } -func TestExecuteScriptAtBlockID(t *testing.T) { - t.Run("happy path", func(t *testing.T) { - runWithEngine(t, func(ctx testingContext) { - // Meaningless script - script := []byte{1, 1, 2, 3, 5, 8, 11} - scriptResult := []byte{1} - - // Ensure block we're about to query against is executable - blockA := unittest.ExecutableBlockFixture(nil) - blockA.StartState = unittest.StateCommitmentPointerFixture() - - snapshot := new(protocol.Snapshot) - snapshot.On("Head").Return(blockA.Block.Header, nil) - - commits := make(map[flow.Identifier]flow.StateCommitment) - commits[blockA.ID()] = *blockA.StartState - - ctx.stateCommitmentExist(blockA.ID(), *blockA.StartState) - - ctx.state.On("AtBlockID", blockA.Block.ID()).Return(snapshot) - ctx.executionState.On("NewStorageSnapshot", *blockA.StartState).Return(nil) - - ctx.executionState.On("HasState", *blockA.StartState).Return(true) - - // Successful call to computation manager - ctx.computationManager. - On("ExecuteScript", mock.Anything, script, [][]byte(nil), blockA.Block.Header, nil). - Return(scriptResult, nil) - - // Execute our script and expect no error - res, err := ctx.engine.ExecuteScriptAtBlockID(context.Background(), script, nil, blockA.Block.ID()) - assert.NoError(t, err) - assert.Equal(t, scriptResult, res) - - // Assert other components were called as expected - ctx.computationManager.AssertExpectations(t) - ctx.executionState.AssertExpectations(t) - ctx.state.AssertExpectations(t) - }) - }) - - t.Run("return early when state commitment not exist", func(t *testing.T) { - runWithEngine(t, func(ctx testingContext) { - // Meaningless script - script := []byte{1, 1, 2, 3, 5, 8, 11} - - // Ensure block we're about to query against is executable - blockA := unittest.ExecutableBlockFixture(nil) - blockA.StartState = unittest.StateCommitmentPointerFixture() - - // make sure blockID to state commitment mapping exist - ctx.executionState.On("StateCommitmentByBlockID", mock.Anything, blockA.ID()).Return(*blockA.StartState, nil) - - // but the state commitment does not exist (e.g. purged) - ctx.executionState.On("HasState", *blockA.StartState).Return(false) - - // Execute our script and expect no error - _, err := ctx.engine.ExecuteScriptAtBlockID(context.Background(), script, nil, blockA.Block.ID()) - assert.Error(t, err) - assert.True(t, strings.Contains(err.Error(), "state commitment not found")) - - // Assert other components were called as expected - ctx.executionState.AssertExpectations(t) - ctx.state.AssertExpectations(t) - }) - }) - -} - func TestUnauthorizedNodeDoesNotBroadcastReceipts(t *testing.T) { runWithEngine(t, func(ctx testingContext) { @@ -1388,21 +1328,16 @@ func TestUnauthorizedNodeDoesNotBroadcastReceipts(t *testing.T) { blockSealed := unittest.BlockHeaderFixture() blocks := make(map[string]*entity.ExecutableBlock) - blocks["A"] = unittest.ExecutableBlockFixtureWithParent(nil, blockSealed) - blocks["A"].StartState = unittest.StateCommitmentPointerFixture() + blocks["A"] = unittest.ExecutableBlockFixtureWithParent(nil, blockSealed, unittest.StateCommitmentPointerFixture()) - blocks["B"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["A"].Block.Header) - blocks["C"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["B"].Block.Header) - blocks["D"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["C"].Block.Header) + // none of the blocks has any collection, so state is essentially the same + blocks["B"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["A"].Block.Header, blocks["A"].StartState) + blocks["C"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["B"].Block.Header, blocks["B"].StartState) + blocks["D"] = unittest.ExecutableBlockFixtureWithParent(nil, blocks["C"].Block.Header, blocks["C"].StartState) // log the blocks, so that we can link the block ID in the log with the blocks in tests logBlocks(blocks) - // none of the blocks has any collection, so state is essentially the same - blocks["B"].StartState = blocks["A"].StartState - blocks["C"].StartState = blocks["B"].StartState - blocks["D"].StartState = blocks["C"].StartState - commits := make(map[flow.Identifier]flow.StateCommitment) commits[blocks["A"].Block.Header.ParentID] = *blocks["A"].StartState @@ -1484,9 +1419,9 @@ func TestUnauthorizedNodeDoesNotBroadcastReceipts(t *testing.T) { err = ctx.engine.handleBlock(context.Background(), blocks["D"].Block) require.NoError(t, err) - //// wait until all 4 blocks have been executed + // // wait until all 4 blocks have been executed unittest.AssertReturnsBefore(t, wg.Wait, 15*time.Second) - _, more := <-ctx.engine.Done() //wait for all the blocks to be processed + _, more := <-ctx.engine.Done() // wait for all the blocks to be processed assert.False(t, more) require.Len(t, ctx.broadcastedReceipts, 2) @@ -1529,7 +1464,7 @@ func TestUnauthorizedNodeDoesNotBroadcastReceipts(t *testing.T) { // require.True(t, shouldTriggerStateSync(20, 29, 10)) // } -func newIngestionEngine(t *testing.T, ps *mocks.ProtocolState, es *mockExecutionState) *Engine { +func newIngestionEngine(t *testing.T, ps *mocks.ProtocolState, es *mockExecutionState) (*Engine, *storage.MockHeaders) { log := unittest.Logger() metrics := metrics.NewNoopCollector() tracer, err := trace.NewTracer(log, "test", "test", trace.SensitivityCaptureAll) @@ -1549,6 +1484,7 @@ func newIngestionEngine(t *testing.T, ps *mocks.ProtocolState, es *mockExecution myIdentity.StakingPubKey = sk.PublicKey() me := mocklocal.NewMockLocal(sk, myIdentity.ID(), t) + headers := storage.NewMockHeaders(ctrl) blocks := storage.NewMockBlocks(ctrl) collections := storage.NewMockCollections(ctrl) events := storage.NewMockEvents(ctrl) @@ -1562,12 +1498,15 @@ func newIngestionEngine(t *testing.T, ps *mocks.ProtocolState, es *mockExecution return stateProtocol.IsNodeAuthorizedAt(ps.AtBlockID(blockID), myIdentity.NodeID) } + unit := enginePkg.NewUnit() engine, err = New( + unit, log, net, me, request, ps, + headers, blocks, collections, events, @@ -1582,11 +1521,23 @@ func newIngestionEngine(t *testing.T, ps *mocks.ProtocolState, es *mockExecution checkAuthorizedAtBlock, nil, nil, - NewStopControl(zerolog.Nop(), false, 0), + stop.NewStopControl( + unit, + time.Second, + zerolog.Nop(), + nil, + headers, + nil, + nil, + &flow.Header{Height: 1}, + false, + false, + ), + false, ) require.NoError(t, err) - return engine + return engine, headers } func logChain(chain []*flow.Block) { @@ -1608,7 +1559,7 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Bootstrap(genesis, result, seal)) es := newMockExecutionState(seal) - engine := newIngestionEngine(t, ps, es) + engine, _ := newIngestionEngine(t, ps, es) finalized, pending, err := engine.unexecutedBlocks() require.NoError(t, err) @@ -1633,7 +1584,7 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Extend(blockD)) es := newMockExecutionState(seal) - engine := newIngestionEngine(t, ps, es) + engine, _ := newIngestionEngine(t, ps, es) finalized, pending, err := engine.unexecutedBlocks() require.NoError(t, err) @@ -1658,7 +1609,7 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Extend(blockD)) es := newMockExecutionState(seal) - engine := newIngestionEngine(t, ps, es) + engine, _ := newIngestionEngine(t, ps, es) es.ExecuteBlock(t, blockA) es.ExecuteBlock(t, blockB) @@ -1688,7 +1639,10 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Finalize(blockC.ID())) es := newMockExecutionState(seal) - engine := newIngestionEngine(t, ps, es) + engine, headers := newIngestionEngine(t, ps, es) + + // block C is the only finalized block, index its header by its height + headers.EXPECT().ByHeight(blockC.Header.Height).Return(blockC.Header, nil) es.ExecuteBlock(t, blockA) es.ExecuteBlock(t, blockB) @@ -1719,7 +1673,10 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Finalize(blockC.ID())) es := newMockExecutionState(seal) - engine := newIngestionEngine(t, ps, es) + engine, headers := newIngestionEngine(t, ps, es) + + // block C is finalized, index its header by its height + headers.EXPECT().ByHeight(blockC.Header.Height).Return(blockC.Header, nil) es.ExecuteBlock(t, blockA) es.ExecuteBlock(t, blockB) @@ -1749,7 +1706,10 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Finalize(blockA.ID())) es := newMockExecutionState(seal) - engine := newIngestionEngine(t, ps, es) + engine, headers := newIngestionEngine(t, ps, es) + + // block A is finalized, index its header by its height + headers.EXPECT().ByHeight(blockA.Header.Height).Return(blockA.Header, nil) es.ExecuteBlock(t, blockA) es.ExecuteBlock(t, blockB) @@ -1805,7 +1765,10 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { es := newMockExecutionState(seal) - engine := newIngestionEngine(t, ps, es) + engine, headers := newIngestionEngine(t, ps, es) + + // block C is finalized, index its header by its height + headers.EXPECT().ByHeight(blockC.Header.Height).Return(blockC.Header, nil) es.ExecuteBlock(t, blockA) es.ExecuteBlock(t, blockB) @@ -1835,8 +1798,7 @@ func TestExecutedBlockIsUploaded(t *testing.T) { // A <- B blockA := unittest.BlockHeaderFixture() - blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA) - blockB.StartState = unittest.StateCommitmentPointerFixture() + blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA, unittest.StateCommitmentPointerFixture()) ctx.mockHasWeightAtBlockID(blockA.ID(), true) ctx.mockHasWeightAtBlockID(blockB.ID(), true) @@ -1879,7 +1841,7 @@ func TestExecutedBlockIsUploaded(t *testing.T) { unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second) - _, more := <-ctx.engine.Done() //wait for all the blocks to be processed + _, more := <-ctx.engine.Done() // wait for all the blocks to be processed require.False(t, more) _, ok := commits[blockB.ID()] @@ -1895,8 +1857,7 @@ func TestExecutedBlockUploadedFailureDoesntBlock(t *testing.T) { // A <- B blockA := unittest.BlockHeaderFixture() - blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA) - blockB.StartState = unittest.StateCommitmentPointerFixture() + blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA, unittest.StateCommitmentPointerFixture()) ctx.mockHasWeightAtBlockID(blockA.ID(), true) ctx.mockHasWeightAtBlockID(blockB.ID(), true) @@ -1940,7 +1901,7 @@ func TestExecutedBlockUploadedFailureDoesntBlock(t *testing.T) { unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second) - _, more := <-ctx.engine.Done() //wait for all the blocks to be processed + _, more := <-ctx.engine.Done() // wait for all the blocks to be processed require.False(t, more) _, ok := commits[blockB.ID()] diff --git a/engine/execution/ingestion/stop/stop_control.go b/engine/execution/ingestion/stop/stop_control.go new file mode 100644 index 00000000000..10e2fc35120 --- /dev/null +++ b/engine/execution/ingestion/stop/stop_control.go @@ -0,0 +1,696 @@ +package stop + +import ( + "errors" + "fmt" + "math" + "strings" + "sync" + "time" + + "github.com/coreos/go-semver/semver" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/execution/state" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/state/protocol" + psEvents "github.com/onflow/flow-go/state/protocol/events" + "github.com/onflow/flow-go/storage" +) + +const ( + // TODO: figure out an appropriate graceful stop time (is 10 min. enough?) + DefaultMaxGracefulStopDuration = 10 * time.Minute +) + +// StopControl is a specialized component used by ingestion.Engine to encapsulate +// control of stopping blocks execution. +// It is intended to work tightly with the Engine, not as a general mechanism or interface. +// +// StopControl can stop execution or crash the node at a specific block height. The stop +// height can be set manually or by the version beacon service event. This leads to some +// edge cases that are handled by the StopControl: +// +// 1. stop is already set manually and is set again manually. +// This is considered as an attempt to move the stop height. The resulting stop +// height is the new one. Note, the new height can be either lower or higher than +// previous value. +// 2. stop is already set manually and is set by the version beacon. +// The resulting stop height is the lower one. +// 3. stop is already set by the version beacon and is set manually. +// The resulting stop height is the lower one. +// 4. stop is already set by the version beacon and is set by the version beacon. +// This means version boundaries were edited. The resulting stop +// height is the new one. +type StopControl struct { + unit *engine.Unit + maxGracefulStopDuration time.Duration + + // Stop control needs to consume BlockFinalized events. + // adding psEvents.Noop makes it a protocol.Consumer + psEvents.Noop + sync.RWMutex + component.Component + + blockFinalizedChan chan *flow.Header + + headers StopControlHeaders + exeState state.ReadOnlyExecutionState + versionBeacons storage.VersionBeacons + + // stopped is true if node should no longer be executing blocks. + stopped bool + // stopBoundary is when the node should stop. + stopBoundary stopBoundary + // nodeVersion could be nil right now. See NewStopControl. + nodeVersion *semver.Version + // last seen version beacon, used to detect version beacon changes + versionBeacon *flow.SealedVersionBeacon + // if the node should crash on version boundary from a version beacon is reached + crashOnVersionBoundaryReached bool + + log zerolog.Logger +} + +var _ protocol.Consumer = (*StopControl)(nil) + +var NoStopHeight = uint64(math.MaxUint64) + +type StopParameters struct { + // desired StopBeforeHeight, the first value new version should be used, + // so this height WON'T be executed + StopBeforeHeight uint64 + + // if the node should crash or just pause after reaching StopBeforeHeight + ShouldCrash bool +} + +func (p StopParameters) Set() bool { + return p.StopBeforeHeight != NoStopHeight +} + +type stopBoundarySource string + +const ( + stopBoundarySourceManual stopBoundarySource = "manual" + stopBoundarySourceVersionBeacon stopBoundarySource = "versionBeacon" +) + +type stopBoundary struct { + StopParameters + + // The stop control will prevent execution of blocks higher than StopBeforeHeight + // once this happens the stop control is affecting execution and StopParameters can + // no longer be changed + immutable bool + + // This is the block ID of the block that should be executed last. + stopAfterExecuting flow.Identifier + + // if the stop parameters were set by the version beacon or manually + source stopBoundarySource +} + +// String returns string in the format "crash@20023[stopBoundarySourceVersionBeacon]" or +// "stop@20023@blockID[manual]" +// block ID is only present if stopAfterExecuting is set +// the ID is from the block that should be executed last and has height one +// less than StopBeforeHeight +func (s stopBoundary) String() string { + if !s.Set() { + return "none" + } + + sb := strings.Builder{} + if s.ShouldCrash { + sb.WriteString("crash") + } else { + sb.WriteString("stop") + } + sb.WriteString("@") + sb.WriteString(fmt.Sprintf("%d", s.StopBeforeHeight)) + + if s.stopAfterExecuting != flow.ZeroID { + sb.WriteString("@") + sb.WriteString(s.stopAfterExecuting.String()) + } + + sb.WriteString("[") + sb.WriteString(string(s.source)) + sb.WriteString("]") + + return sb.String() +} + +// StopControlHeaders is an interface for fetching headers +// Its jut a small subset of storage.Headers for comments see storage.Headers +type StopControlHeaders interface { + ByHeight(height uint64) (*flow.Header, error) +} + +// NewStopControl creates new StopControl. +// +// We currently have no strong guarantee that the node version is a valid semver. +// See build.SemverV2 for more details. That is why nil is a valid input for node version +// without a node version, the stop control can still be used for manual stopping. +func NewStopControl( + unit *engine.Unit, + maxGracefulStopDuration time.Duration, + log zerolog.Logger, + exeState state.ReadOnlyExecutionState, + headers StopControlHeaders, + versionBeacons storage.VersionBeacons, + nodeVersion *semver.Version, + latestFinalizedBlock *flow.Header, + withStoppedExecution bool, + crashOnVersionBoundaryReached bool, +) *StopControl { + // We should not miss block finalized events, and we should be able to handle them + // faster than they are produced anyway. + blockFinalizedChan := make(chan *flow.Header, 1000) + + sc := &StopControl{ + unit: unit, + maxGracefulStopDuration: maxGracefulStopDuration, + log: log.With(). + Str("component", "stop_control"). + Logger(), + + blockFinalizedChan: blockFinalizedChan, + + exeState: exeState, + headers: headers, + nodeVersion: nodeVersion, + versionBeacons: versionBeacons, + stopped: withStoppedExecution, + crashOnVersionBoundaryReached: crashOnVersionBoundaryReached, + // the default is to never stop + stopBoundary: stopBoundary{ + StopParameters: StopParameters{ + StopBeforeHeight: NoStopHeight, + }, + }, + } + + if sc.nodeVersion != nil { + log = log.With(). + Stringer("node_version", sc.nodeVersion). + Bool("crash_on_version_boundary_reached", + sc.crashOnVersionBoundaryReached). + Logger() + } + + log.Info().Msgf("Created") + + cm := component.NewComponentManagerBuilder() + cm.AddWorker(sc.processEvents) + cm.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + sc.checkInitialVersionBeacon(ctx, ready, latestFinalizedBlock) + }) + + sc.Component = cm.Build() + + // TODO: handle version beacon already indicating a stop + // right now the stop will happen on first BlockFinalized + // which is fine, but ideally we would stop right away. + + return sc +} + +// BlockFinalized is called when a block is finalized. +// +// This is a protocol event consumer. See protocol.Consumer. +func (s *StopControl) BlockFinalized(h *flow.Header) { + s.blockFinalizedChan <- h +} + +// processEvents is a worker that processes block finalized events. +func (s *StopControl) processEvents( + ctx irrecoverable.SignalerContext, + ready component.ReadyFunc, +) { + ready() + + for { + select { + case <-ctx.Done(): + return + case h := <-s.blockFinalizedChan: + s.blockFinalized(ctx, h) + } + } +} + +// BlockFinalizedForTesting is used for testing only. +func (s *StopControl) BlockFinalizedForTesting(h *flow.Header) { + s.blockFinalized(irrecoverable.MockSignalerContext{}, h) +} + +func (s *StopControl) checkInitialVersionBeacon( + ctx irrecoverable.SignalerContext, + ready component.ReadyFunc, + latestFinalizedBlock *flow.Header, +) { + // component is not ready until we checked the initial version beacon + defer ready() + + // the most straightforward way to check it is to simply pretend we just finalized the + // last finalized block + s.blockFinalized(ctx, latestFinalizedBlock) + +} + +// IsExecutionStopped returns true is block execution has been stopped +func (s *StopControl) IsExecutionStopped() bool { + s.RLock() + defer s.RUnlock() + + return s.stopped +} + +// SetStopParameters sets new stop parameters manually. +// +// Expected error returns during normal operations: +// - ErrCannotChangeStop: this indicates that new stop parameters cannot be set. +// See stop.validateStopChange. +func (s *StopControl) SetStopParameters( + stop StopParameters, +) error { + s.Lock() + defer s.Unlock() + + boundary := stopBoundary{ + StopParameters: stop, + source: stopBoundarySourceManual, + } + + return s.setStopParameters(boundary) +} + +// setStopParameters sets new stop parameters. +// stopBoundary is the new stop parameters. If nil, the stop is removed. +// +// Expected error returns during normal operations: +// - ErrCannotChangeStop: this indicates that new stop parameters cannot be set. +// See stop.validateStopChange. +// +// Caller must acquire the lock. +func (s *StopControl) setStopParameters( + stopBoundary stopBoundary, +) error { + log := s.log.With(). + Stringer("old_stop", s.stopBoundary). + Stringer("new_stop", stopBoundary). + Logger() + + err := s.validateStopChange(stopBoundary) + if err != nil { + log.Info().Err(err).Msg("cannot set stopHeight") + return err + } + + log.Info().Msg("new stop set") + s.stopBoundary = stopBoundary + + return nil +} + +var ErrCannotChangeStop = errors.New("cannot change stop control stopping parameters") + +// validateStopChange verifies if the stop parameters can be changed +// returns the error with the reason if the parameters cannot be changed. +// +// Stop parameters cannot be changed if: +// 1. node is already stopped +// 2. stop parameters are immutable (due to them already affecting execution see +// ShouldExecuteBlock) +// 3. stop parameters are already set by a different source and the new stop is later than +// the existing one +// +// Expected error returns during normal operations: +// - ErrCannotChangeStop: this indicates that new stop parameters cannot be set. +// +// Caller must acquire the lock. +func (s *StopControl) validateStopChange( + newStopBoundary stopBoundary, +) error { + + errf := func(reason string) error { + return fmt.Errorf("%s: %w", reason, ErrCannotChangeStop) + } + + // 1. + if s.stopped { + return errf("cannot update stop parameters, already stopped") + } + + // 2. + if s.stopBoundary.immutable { + return errf( + fmt.Sprintf( + "cannot update stopHeight, stopping commenced for %s", + s.stopBoundary), + ) + } + + if !s.stopBoundary.Set() { + // if the current stop is no stop, we can always update + return nil + } + + // 3. + if s.stopBoundary.source == newStopBoundary.source { + // if the stop was set by the same source, we can always update + return nil + } + + // 3. + // if one stop was set by the version beacon and the other one was manual + // we can only update if the new stop is strictly earlier + if newStopBoundary.StopBeforeHeight < s.stopBoundary.StopBeforeHeight { + return nil + + } + // this prevents users moving the stopHeight forward when a version newStopBoundary + // is earlier, and prevents version beacons from moving the stopHeight forward + // when a manual stop is earlier. + return errf("cannot update stopHeight, " + + "new stop height is later than the current one") +} + +// GetStopParameters returns the upcoming stop parameters or nil if no stop is set. +func (s *StopControl) GetStopParameters() StopParameters { + s.RLock() + defer s.RUnlock() + + return s.stopBoundary.StopParameters +} + +// ShouldExecuteBlock should be called when new block can be executed. +// The block should not be executed if its height is above or equal to +// s.stopBoundary.StopBeforeHeight. +// +// It returns a boolean indicating if the block should be executed. +func (s *StopControl) ShouldExecuteBlock(b *flow.Header) bool { + s.Lock() + defer s.Unlock() + + // don't process anymore blocks if stopped + if s.stopped { + return false + } + + // Skips blocks at or above requested stopHeight + // doing so means we have started the stopping process + if b.Height < s.stopBoundary.StopBeforeHeight { + return true + } + + s.log.Info(). + Msgf("Skipping execution of %s at height %d"+ + " because stop has been requested %s", + b.ID(), + b.Height, + s.stopBoundary) + + // stopBoundary is now immutable, because it started affecting execution + s.stopBoundary.immutable = true + return false +} + +// blockFinalized is called when a block is marked as finalized +// +// Once finalization reached stopHeight we can be sure no other fork will be valid at +// this height, if this block's parent has been executed, we are safe to stop. +// This will happen during normal execution, where blocks are executed +// before they are finalized. However, it is possible that EN block computation +// progress can fall behind. In this case, we want to crash only after the execution +// reached the stopHeight. +func (s *StopControl) blockFinalized( + ctx irrecoverable.SignalerContext, + h *flow.Header, +) { + s.Lock() + defer s.Unlock() + + // already stopped, nothing to do + if s.stopped { + return + } + + // We already know the ID of the block that should be executed last nothing to do. + // Node is stopping. + if s.stopBoundary.stopAfterExecuting != flow.ZeroID { + return + } + + handleErr := func(err error) { + s.log.Err(err). + Stringer("block_id", h.ID()). + Stringer("stop", s.stopBoundary). + Msg("Error in stop control BlockFinalized") + + ctx.Throw(err) + } + + s.processNewVersionBeacons(ctx, h.Height) + + // we are not at the stop yet, nothing to do + if h.Height < s.stopBoundary.StopBeforeHeight { + return + } + + parentID := h.ParentID + + if h.Height != s.stopBoundary.StopBeforeHeight { + // we are past the stop. This can happen if stop was set before + // last finalized block + s.log.Warn(). + Uint64("finalization_height", h.Height). + Stringer("block_id", h.ID()). + Stringer("stop", s.stopBoundary). + Msg("Block finalization already beyond stop.") + + // Let's find the ID of the block that should be executed last + // which is the parent of the block at the stopHeight + header, err := s.headers.ByHeight(s.stopBoundary.StopBeforeHeight - 1) + if err != nil { + handleErr(fmt.Errorf("failed to get header by height: %w", err)) + return + } + parentID = header.ID() + } + + s.stopBoundary.stopAfterExecuting = parentID + + s.log.Info(). + Stringer("block_id", h.ID()). + Stringer("stop", s.stopBoundary). + Stringer("stop_after_executing", s.stopBoundary.stopAfterExecuting). + Msgf("Found ID of the block that should be executed last") + + // check if the parent block has been executed then stop right away + executed, err := state.IsBlockExecuted(ctx, s.exeState, h.ParentID) + if err != nil { + handleErr(fmt.Errorf( + "failed to check if the block has been executed: %w", + err, + )) + return + } + + if executed { + // we already reached the point where we should stop + s.stopExecution() + return + } +} + +// OnBlockExecuted should be called after a block has finished execution +func (s *StopControl) OnBlockExecuted(h *flow.Header) { + s.Lock() + defer s.Unlock() + + if s.stopped { + return + } + + if s.stopBoundary.stopAfterExecuting != h.ID() { + return + } + + // double check. Even if requested stopHeight has been changed multiple times, + // as long as it matches this block we are safe to terminate + if h.Height != s.stopBoundary.StopBeforeHeight-1 { + s.log.Warn(). + Msgf( + "Inconsistent stopping state. "+ + "Scheduled to stop after executing block ID %s and height %d, "+ + "but this block has a height %d. ", + h.ID().String(), + s.stopBoundary.StopBeforeHeight-1, + h.Height, + ) + return + } + + s.stopExecution() +} + +// stopExecution stops the node execution and crashes the node if ShouldCrash is true. +// Caller must acquire the lock. +func (s *StopControl) stopExecution() { + log := s.log.With(). + Stringer("requested_stop", s.stopBoundary). + Uint64("last_executed_height", s.stopBoundary.StopBeforeHeight). + Stringer("last_executed_id", s.stopBoundary.stopAfterExecuting). + Logger() + + s.stopped = true + log.Warn().Msg("Stopping as finalization reached requested stop") + + if s.stopBoundary.ShouldCrash { + log.Info(). + Dur("max-graceful-stop-duration", s.maxGracefulStopDuration). + Msg("Attempting graceful stop as finalization reached requested stop") + doneChan := s.unit.Done() + select { + case <-doneChan: + log.Info().Msg("Engine gracefully stopped") + case <-time.After(s.maxGracefulStopDuration): + log.Info(). + Msg("Engine did not stop within max graceful stop duration") + } + log.Fatal().Msg("Crashing as finalization reached requested stop") + return + } +} + +// processNewVersionBeacons processes version beacons and updates the stop control stop +// height if needed. +// +// When a block is finalized it is possible that a new version beacon is indexed. +// This new version beacon might have added/removed/moved a version boundary. +// The old version beacon is considered invalid, and the stop height must be updated +// according to the new version beacon. +// +// Caller must acquire the lock. +func (s *StopControl) processNewVersionBeacons( + ctx irrecoverable.SignalerContext, + height uint64, +) { + // TODO: remove when we can guarantee that the node will always have a valid version + if s.nodeVersion == nil { + return + } + + if s.versionBeacon != nil && s.versionBeacon.SealHeight >= height { + // already processed this or a higher version beacon + return + } + + vb, err := s.versionBeacons.Highest(height) + if err != nil { + s.log.Err(err). + Uint64("height", height). + Msg("Failed to get highest version beacon for stop control") + + ctx.Throw( + fmt.Errorf( + "failed to get highest version beacon for stop control: %w", + err)) + return + } + + if vb == nil { + // no version beacon found + // this is unexpected as there should always be at least the + // starting version beacon, but not fatal. + // It can happen if the node starts before bootstrap is finished. + // TODO: remove when we can guarantee that there will always be a version beacon + s.log.Info(). + Uint64("height", height). + Msg("No version beacon found for stop control") + return + } + + if s.versionBeacon != nil && s.versionBeacon.SealHeight >= vb.SealHeight { + // we already processed this or a higher version beacon + return + } + + s.log.Info(). + Uint64("vb_seal_height", vb.SealHeight). + Uint64("vb_sequence", vb.Sequence). + Msg("New version beacon found") + + // this is now the last handled version beacon + s.versionBeacon = vb + + // this is a new version beacon check what boundary it sets + stopHeight, err := s.getVersionBeaconStopHeight(vb) + if err != nil { + s.log.Err(err). + Interface("version_beacon", vb). + Msg("Failed to get stop height from version beacon") + + ctx.Throw( + fmt.Errorf("failed to get stop height from version beacon: %w", err)) + return + } + + var newStop = stopBoundary{ + StopParameters: StopParameters{ + StopBeforeHeight: stopHeight, + ShouldCrash: s.crashOnVersionBoundaryReached, + }, + source: stopBoundarySourceVersionBeacon, + } + + err = s.setStopParameters(newStop) + if err != nil { + // This is just informational and is expected to sometimes happen during + // normal operation. The causes for this are described here: validateStopChange. + s.log.Info(). + Err(err). + Msg("Cannot change stop boundary when detecting new version beacon") + } +} + +// getVersionBeaconStopHeight returns the stop height that should be set +// based on the version beacon +// +// No error is expected during normal operation since the version beacon +// should have been validated when indexing. +// +// Caller must acquire the lock. +func (s *StopControl) getVersionBeaconStopHeight( + vb *flow.SealedVersionBeacon, +) ( + uint64, + error, +) { + // version boundaries are sorted by version + for _, boundary := range vb.VersionBoundaries { + ver, err := boundary.Semver() + if err != nil || ver == nil { + // this should never happen as we already validated the version beacon + // when indexing it + return 0, fmt.Errorf("failed to parse semver: %w", err) + } + + // This condition can be tweaked in the future. For example if we guarantee that + // all nodes with the same major version have compatible execution, + // we can stop only on major version change. + if s.nodeVersion.LessThan(*ver) { + // we need to stop here + return boundary.BlockHeight, nil + } + } + + // no stop boundary should be set + return NoStopHeight, nil +} diff --git a/engine/execution/ingestion/stop/stop_control_test.go b/engine/execution/ingestion/stop/stop_control_test.go new file mode 100644 index 00000000000..b4fb326c061 --- /dev/null +++ b/engine/execution/ingestion/stop/stop_control_test.go @@ -0,0 +1,881 @@ +package stop + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/coreos/go-semver/semver" + testifyMock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/execution/state/mock" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/storage" + storageMock "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +// If stopping mechanism has caused any changes to execution flow +// (skipping execution of blocks) we disallow setting new values +func TestCannotSetNewValuesAfterStoppingCommenced(t *testing.T) { + + t.Run("when processing block at stop height", func(t *testing.T) { + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + nil, + nil, + nil, + nil, + &flow.Header{Height: 1}, + false, + false, + ) + + require.False(t, sc.GetStopParameters().Set()) + + // first update is always successful + stop := StopParameters{StopBeforeHeight: 21} + err := sc.SetStopParameters(stop) + require.NoError(t, err) + + require.Equal(t, stop, sc.GetStopParameters()) + + // no stopping has started yet, block below stop height + header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + require.True(t, sc.ShouldExecuteBlock(header)) + + stop2 := StopParameters{StopBeforeHeight: 37} + err = sc.SetStopParameters(stop2) + require.NoError(t, err) + + // block at stop height, it should be skipped + header = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(37)) + require.False(t, sc.ShouldExecuteBlock(header)) + + // cannot set new stop height after stopping has started + err = sc.SetStopParameters(StopParameters{StopBeforeHeight: 2137}) + require.ErrorIs(t, err, ErrCannotChangeStop) + + // state did not change + require.Equal(t, stop2, sc.GetStopParameters()) + }) + + t.Run("when processing finalized blocks", func(t *testing.T) { + + execState := mock.NewExecutionState(t) + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + nil, + nil, + nil, + &flow.Header{Height: 1}, + false, + false, + ) + + require.False(t, sc.GetStopParameters().Set()) + + // first update is always successful + stop := StopParameters{StopBeforeHeight: 21} + err := sc.SetStopParameters(stop) + require.NoError(t, err) + require.Equal(t, stop, sc.GetStopParameters()) + + // make execution check pretends block has been executed + execState.On("StateCommitmentByBlockID", testifyMock.Anything, testifyMock.Anything).Return(nil, nil) + + // no stopping has started yet, block below stop height + header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + sc.BlockFinalizedForTesting(header) + + stop2 := StopParameters{StopBeforeHeight: 37} + err = sc.SetStopParameters(stop2) + require.NoError(t, err) + require.Equal(t, stop2, sc.GetStopParameters()) + + // block at stop height, it should be triggered stop + header = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(37)) + sc.BlockFinalizedForTesting(header) + + // since we set shouldCrash to false, execution should be stopped + require.True(t, sc.IsExecutionStopped()) + + err = sc.SetStopParameters(StopParameters{StopBeforeHeight: 2137}) + require.ErrorIs(t, err, ErrCannotChangeStop) + }) +} + +// TestExecutionFallingBehind check if StopControl behaves properly even if EN runs behind +// and blocks are finalized before they are executed +func TestExecutionFallingBehind(t *testing.T) { + + execState := mock.NewExecutionState(t) + + headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21 + headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22 + headerD := unittest.BlockHeaderWithParentFixture(headerC) // 23 + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + nil, + nil, + nil, + &flow.Header{Height: 1}, + false, + false, + ) + + // set stop at 22, so 21 is the last height which should be processed + stop := StopParameters{StopBeforeHeight: 22} + err := sc.SetStopParameters(stop) + require.NoError(t, err) + require.Equal(t, stop, sc.GetStopParameters()) + + execState. + On("StateCommitmentByBlockID", testifyMock.Anything, headerC.ParentID). + Return(nil, storage.ErrNotFound) + + // finalize blocks first + sc.BlockFinalizedForTesting(headerA) + sc.BlockFinalizedForTesting(headerB) + sc.BlockFinalizedForTesting(headerC) + sc.BlockFinalizedForTesting(headerD) + + // simulate execution + sc.OnBlockExecuted(headerA) + sc.OnBlockExecuted(headerB) + require.True(t, sc.IsExecutionStopped()) +} + +type stopControlMockHeaders struct { + headers map[uint64]*flow.Header +} + +func (m *stopControlMockHeaders) ByHeight(height uint64) (*flow.Header, error) { + h, ok := m.headers[height] + if !ok { + return nil, fmt.Errorf("header not found") + } + return h, nil +} + +func TestAddStopForPastBlocks(t *testing.T) { + execState := mock.NewExecutionState(t) + + headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21 + headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22 + headerD := unittest.BlockHeaderWithParentFixture(headerC) // 23 + + headers := &stopControlMockHeaders{ + headers: map[uint64]*flow.Header{ + headerA.Height: headerA, + headerB.Height: headerB, + headerC.Height: headerC, + headerD.Height: headerD, + }, + } + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + headers, + nil, + nil, + &flow.Header{Height: 1}, + false, + false, + ) + + // finalize blocks first + sc.BlockFinalizedForTesting(headerA) + sc.BlockFinalizedForTesting(headerB) + sc.BlockFinalizedForTesting(headerC) + + // simulate execution + sc.OnBlockExecuted(headerA) + sc.OnBlockExecuted(headerB) + sc.OnBlockExecuted(headerC) + + // block is executed + execState. + On("StateCommitmentByBlockID", testifyMock.Anything, headerD.ParentID). + Return(nil, nil) + + // set stop at 22, but finalization and execution is at 23 + // so stop right away + stop := StopParameters{StopBeforeHeight: 22} + err := sc.SetStopParameters(stop) + require.NoError(t, err) + require.Equal(t, stop, sc.GetStopParameters()) + + // finalize one more block after stop is set + sc.BlockFinalizedForTesting(headerD) + + require.True(t, sc.IsExecutionStopped()) +} + +func TestAddStopForPastBlocksExecutionFallingBehind(t *testing.T) { + execState := mock.NewExecutionState(t) + + headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21 + headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22 + headerD := unittest.BlockHeaderWithParentFixture(headerC) // 23 + + headers := &stopControlMockHeaders{ + headers: map[uint64]*flow.Header{ + headerA.Height: headerA, + headerB.Height: headerB, + headerC.Height: headerC, + headerD.Height: headerD, + }, + } + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + headers, + nil, + nil, + &flow.Header{Height: 1}, + false, + false, + ) + + execState. + On("StateCommitmentByBlockID", testifyMock.Anything, headerD.ParentID). + Return(nil, storage.ErrNotFound) + + // finalize blocks first + sc.BlockFinalizedForTesting(headerA) + sc.BlockFinalizedForTesting(headerB) + sc.BlockFinalizedForTesting(headerC) + + // set stop at 22, but finalization is at 23 so 21 + // is the last height which wil be executed + stop := StopParameters{StopBeforeHeight: 22} + err := sc.SetStopParameters(stop) + require.NoError(t, err) + require.Equal(t, stop, sc.GetStopParameters()) + + // finalize one more block after stop is set + sc.BlockFinalizedForTesting(headerD) + + // simulate execution + sc.OnBlockExecuted(headerA) + sc.OnBlockExecuted(headerB) + require.True(t, sc.IsExecutionStopped()) +} + +func TestStopControlWithVersionControl(t *testing.T) { + t.Run("normal case", func(t *testing.T) { + execState := mock.NewExecutionState(t) + versionBeacons := new(storageMock.VersionBeacons) + + headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21 + headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22 + + headers := &stopControlMockHeaders{ + headers: map[uint64]*flow.Header{ + headerA.Height: headerA, + headerB.Height: headerB, + headerC.Height: headerC, + }, + } + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + headers, + versionBeacons, + semver.New("1.0.0"), + &flow.Header{Height: 1}, + false, + false, + ) + + // setting this means all finalized blocks are considered already executed + execState. + On("StateCommitmentByBlockID", testifyMock.Anything, headerC.ParentID). + Return(nil, nil) + + versionBeacons. + On("Highest", testifyMock.Anything). + Return(&flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + // zero boundary is expected if there + // is no boundary set by the contract yet + flow.VersionBoundary{ + BlockHeight: 0, + Version: "0.0.0", + }), + ), + SealHeight: headerA.Height, + }, nil).Once() + + // finalize first block + sc.BlockFinalizedForTesting(headerA) + require.False(t, sc.IsExecutionStopped()) + require.False(t, sc.GetStopParameters().Set()) + + // new version beacon + versionBeacons. + On("Highest", testifyMock.Anything). + Return(&flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + // zero boundary is expected if there + // is no boundary set by the contract yet + flow.VersionBoundary{ + BlockHeight: 0, + Version: "0.0.0", + }, flow.VersionBoundary{ + BlockHeight: 21, + Version: "1.0.0", + }), + ), + SealHeight: headerB.Height, + }, nil).Once() + + // finalize second block. we are still ok as the node version + // is the same as the version beacon one + sc.BlockFinalizedForTesting(headerB) + require.False(t, sc.IsExecutionStopped()) + require.False(t, sc.GetStopParameters().Set()) + + // new version beacon + versionBeacons. + On("Highest", testifyMock.Anything). + Return(&flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + // The previous version is included in the new version beacon + flow.VersionBoundary{ + BlockHeight: 21, + Version: "1.0.0", + }, flow.VersionBoundary{ + BlockHeight: 22, + Version: "2.0.0", + }), + ), + SealHeight: headerC.Height, + }, nil).Once() + sc.BlockFinalizedForTesting(headerC) + // should be stopped as this is height 22 and height 21 is already considered executed + require.True(t, sc.IsExecutionStopped()) + }) + + t.Run("version boundary removed", func(t *testing.T) { + + // future version boundaries can be removed + // in which case they will be missing from the version beacon + execState := mock.NewExecutionState(t) + versionBeacons := storageMock.NewVersionBeacons(t) + + headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21 + headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22 + + headers := &stopControlMockHeaders{ + headers: map[uint64]*flow.Header{ + headerA.Height: headerA, + headerB.Height: headerB, + headerC.Height: headerC, + }, + } + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + headers, + versionBeacons, + semver.New("1.0.0"), + &flow.Header{Height: 1}, + false, + false, + ) + + versionBeacons. + On("Highest", testifyMock.Anything). + Return(&flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + // set to stop at height 21 + flow.VersionBoundary{ + BlockHeight: 0, + Version: "0.0.0", + }, flow.VersionBoundary{ + BlockHeight: 21, + Version: "2.0.0", + }), + ), + SealHeight: headerA.Height, + }, nil).Once() + + // finalize first block + sc.BlockFinalizedForTesting(headerA) + require.False(t, sc.IsExecutionStopped()) + require.Equal(t, StopParameters{ + StopBeforeHeight: 21, + ShouldCrash: false, + }, sc.GetStopParameters()) + + // new version beacon + versionBeacons. + On("Highest", testifyMock.Anything). + Return(&flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + // stop removed + flow.VersionBoundary{ + BlockHeight: 0, + Version: "0.0.0", + }), + ), + SealHeight: headerB.Height, + }, nil).Once() + + // finalize second block. we are still ok as the node version + // is the same as the version beacon one + sc.BlockFinalizedForTesting(headerB) + require.False(t, sc.IsExecutionStopped()) + require.False(t, sc.GetStopParameters().Set()) + }) + + t.Run("manual not cleared by version beacon", func(t *testing.T) { + // future version boundaries can be removed + // in which case they will be missing from the version beacon + execState := mock.NewExecutionState(t) + versionBeacons := storageMock.NewVersionBeacons(t) + + headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21 + headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22 + + headers := &stopControlMockHeaders{ + headers: map[uint64]*flow.Header{ + headerA.Height: headerA, + headerB.Height: headerB, + headerC.Height: headerC, + }, + } + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + headers, + versionBeacons, + semver.New("1.0.0"), + &flow.Header{Height: 1}, + false, + false, + ) + + versionBeacons. + On("Highest", testifyMock.Anything). + Return(&flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + // set to stop at height 21 + flow.VersionBoundary{ + BlockHeight: 0, + Version: "0.0.0", + }), + ), + SealHeight: headerA.Height, + }, nil).Once() + + // finalize first block + sc.BlockFinalizedForTesting(headerA) + require.False(t, sc.IsExecutionStopped()) + require.False(t, sc.GetStopParameters().Set()) + + // set manual stop + stop := StopParameters{ + StopBeforeHeight: 22, + ShouldCrash: false, + } + err := sc.SetStopParameters(stop) + require.NoError(t, err) + require.Equal(t, stop, sc.GetStopParameters()) + + // new version beacon + versionBeacons. + On("Highest", testifyMock.Anything). + Return(&flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + // stop removed + flow.VersionBoundary{ + BlockHeight: 0, + Version: "0.0.0", + }), + ), + SealHeight: headerB.Height, + }, nil).Once() + + sc.BlockFinalizedForTesting(headerB) + require.False(t, sc.IsExecutionStopped()) + // stop is not cleared due to being set manually + require.Equal(t, stop, sc.GetStopParameters()) + }) + + t.Run("version beacon not cleared by manual", func(t *testing.T) { + // future version boundaries can be removed + // in which case they will be missing from the version beacon + execState := mock.NewExecutionState(t) + versionBeacons := storageMock.NewVersionBeacons(t) + + headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21 + + headers := &stopControlMockHeaders{ + headers: map[uint64]*flow.Header{ + headerA.Height: headerA, + headerB.Height: headerB, + }, + } + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + headers, + versionBeacons, + semver.New("1.0.0"), + &flow.Header{Height: 1}, + false, + false, + ) + + vbStop := StopParameters{ + StopBeforeHeight: 22, + ShouldCrash: false, + } + versionBeacons. + On("Highest", testifyMock.Anything). + Return(&flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + // set to stop at height 21 + flow.VersionBoundary{ + BlockHeight: 0, + Version: "0.0.0", + }, flow.VersionBoundary{ + BlockHeight: vbStop.StopBeforeHeight, + Version: "2.0.0", + }), + ), + SealHeight: headerA.Height, + }, nil).Once() + + // finalize first block + sc.BlockFinalizedForTesting(headerA) + require.False(t, sc.IsExecutionStopped()) + require.Equal(t, vbStop, sc.GetStopParameters()) + + // set manual stop + stop := StopParameters{ + StopBeforeHeight: 23, + ShouldCrash: false, + } + err := sc.SetStopParameters(stop) + require.ErrorIs(t, err, ErrCannotChangeStop) + // stop is not cleared due to being set earlier by a version beacon + require.Equal(t, vbStop, sc.GetStopParameters()) + }) +} + +// StopControl created as stopped will keep the state +func TestStartingStopped(t *testing.T) { + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + nil, + nil, + nil, + nil, + &flow.Header{Height: 1}, + true, + false, + ) + require.True(t, sc.IsExecutionStopped()) +} + +func TestStoppedStateRejectsAllBlocksAndChanged(t *testing.T) { + + // make sure we don't even query executed status if stopped + // mock should fail test on any method call + execState := mock.NewExecutionState(t) + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + nil, + nil, + nil, + &flow.Header{Height: 1}, + true, + false, + ) + require.True(t, sc.IsExecutionStopped()) + + err := sc.SetStopParameters(StopParameters{ + StopBeforeHeight: 2137, + ShouldCrash: true, + }) + require.ErrorIs(t, err, ErrCannotChangeStop) + + header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + + sc.BlockFinalizedForTesting(header) + require.True(t, sc.IsExecutionStopped()) +} + +func Test_StopControlWorkers(t *testing.T) { + + t.Run("start and stop, stopped = true", func(t *testing.T) { + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + nil, + nil, + nil, + nil, + &flow.Header{Height: 1}, + true, + false, + ) + + ctx, cancel := context.WithCancel(context.Background()) + ictx := irrecoverable.NewMockSignalerContext(t, ctx) + + sc.Start(ictx) + + unittest.AssertClosesBefore(t, sc.Ready(), 10*time.Second) + + cancel() + + unittest.AssertClosesBefore(t, sc.Done(), 10*time.Second) + }) + + t.Run("start and stop, stopped = false", func(t *testing.T) { + + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + nil, + nil, + nil, + nil, + &flow.Header{Height: 1}, + false, + false, + ) + + ctx, cancel := context.WithCancel(context.Background()) + ictx := irrecoverable.NewMockSignalerContext(t, ctx) + + sc.Start(ictx) + + unittest.AssertClosesBefore(t, sc.Ready(), 10*time.Second) + + cancel() + + unittest.AssertClosesBefore(t, sc.Done(), 10*time.Second) + }) + + t.Run("start as stopped if execution is at version boundary", func(t *testing.T) { + + headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21 + + versionBeacons := storageMock.NewVersionBeacons(t) + versionBeacons.On("Highest", headerB.Height). + Return(&flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + BlockHeight: headerB.Height, + Version: "2.0.0", + }, + ), + ), + SealHeight: 1, // sealed in the past + }, nil). + Once() + + execState := mock.NewExecutionState(t) + execState.On( + "StateCommitmentByBlockID", + testifyMock.Anything, + headerA.ID(), + ).Return(flow.StateCommitment{}, nil). + Once() + + headers := &stopControlMockHeaders{ + headers: map[uint64]*flow.Header{ + headerA.Height: headerA, + headerB.Height: headerB, + }, + } + + // This is a likely scenario where the node stopped because of a version + // boundary but was restarted without being upgraded to the new version. + // In this case, the node should start as stopped. + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + headers, + versionBeacons, + semver.New("1.0.0"), + headerB, + false, + false, + ) + + ctx, cancel := context.WithCancel(context.Background()) + ictx := irrecoverable.NewMockSignalerContext(t, ctx) + + sc.Start(ictx) + + unittest.AssertClosesBefore(t, sc.Ready(), 10*time.Second) + + // should start as stopped + require.True(t, sc.IsExecutionStopped()) + require.Equal(t, StopParameters{ + StopBeforeHeight: headerB.Height, + ShouldCrash: false, + }, sc.GetStopParameters()) + + cancel() + + unittest.AssertClosesBefore(t, sc.Done(), 10*time.Second) + }) + + t.Run("test stopping with block finalized events", func(t *testing.T) { + + headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) + headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21 + headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22 + + vb := &flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + BlockHeight: headerC.Height, + Version: "2.0.0", + }, + ), + ), + SealHeight: 1, // sealed in the past + } + + versionBeacons := storageMock.NewVersionBeacons(t) + versionBeacons.On("Highest", headerB.Height). + Return(vb, nil). + Once() + versionBeacons.On("Highest", headerC.Height). + Return(vb, nil). + Once() + + execState := mock.NewExecutionState(t) + execState.On( + "StateCommitmentByBlockID", + testifyMock.Anything, + headerB.ID(), + ).Return(flow.StateCommitment{}, nil). + Once() + + headers := &stopControlMockHeaders{ + headers: map[uint64]*flow.Header{ + headerA.Height: headerA, + headerB.Height: headerB, + headerC.Height: headerC, + }, + } + + // The stop is set by a previous version beacon and is in one blocks time. + sc := NewStopControl( + engine.NewUnit(), + time.Second, + unittest.Logger(), + execState, + headers, + versionBeacons, + semver.New("1.0.0"), + headerB, + false, + false, + ) + + ctx, cancel := context.WithCancel(context.Background()) + ictx := irrecoverable.NewMockSignalerContext(t, ctx) + + sc.Start(ictx) + + unittest.AssertClosesBefore(t, sc.Ready(), 10*time.Second) + + require.False(t, sc.IsExecutionStopped()) + require.Equal(t, StopParameters{ + StopBeforeHeight: headerC.Height, + ShouldCrash: false, + }, sc.GetStopParameters()) + + sc.BlockFinalized(headerC) + + done := make(chan struct{}) + go func() { + for !sc.IsExecutionStopped() { + <-time.After(100 * time.Millisecond) + } + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for stop control to stop execution") + } + + cancel() + unittest.AssertClosesBefore(t, sc.Done(), 10*time.Second) + }) +} diff --git a/engine/execution/ingestion/stop_control.go b/engine/execution/ingestion/stop_control.go deleted file mode 100644 index 49d09f07194..00000000000 --- a/engine/execution/ingestion/stop_control.go +++ /dev/null @@ -1,320 +0,0 @@ -package ingestion - -import ( - "context" - "fmt" - "sync" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/engine/execution/state" - "github.com/onflow/flow-go/model/flow" -) - -// StopControl is a specialized component used by ingestion.Engine to encapsulate -// control of pausing/stopping blocks execution. -// It is intended to work tightly with the Engine, not as a general mechanism or interface. -// StopControl follows states described in StopState -type StopControl struct { - sync.RWMutex - // desired stopHeight, the first value new version should be used, - // so this height WON'T be executed - stopHeight uint64 - - // if the node should crash or just pause after reaching stopHeight - crash bool - - // This is the block ID of the block that should be executed last. - stopAfterExecuting flow.Identifier - - log zerolog.Logger - state StopControlState - - // used to prevent setting stopHeight to block which has already been executed - highestExecutingHeight uint64 -} - -type StopControlState byte - -const ( - // StopControlOff default state, envisioned to be used most of the time. - // Stopping module is simply off, blocks will be processed "as usual". - StopControlOff StopControlState = iota - - // StopControlSet means stopHeight is set but not reached yet, - // and nothing related to stopping happened yet. - // We could still go back to StopControlOff or progress to StopControlCommenced. - StopControlSet - - // StopControlCommenced indicates that stopping process has commenced - // and no parameters can be changed anymore. - // For example, blocks at or above stopHeight has been received, - // but finalization didn't reach stopHeight yet. - // It can only progress to StopControlPaused - StopControlCommenced - - // StopControlPaused means EN has stopped processing blocks. - // It can happen by reaching the set stopping `stopHeight`, or - // if the node was started in pause mode. - // It is a final state and cannot be changed - StopControlPaused -) - -// NewStopControl creates new empty NewStopControl -func NewStopControl( - log zerolog.Logger, - paused bool, - lastExecutedHeight uint64, -) *StopControl { - state := StopControlOff - if paused { - state = StopControlPaused - } - log.Debug().Msgf("created StopControl module with paused = %t", paused) - return &StopControl{ - log: log, - state: state, - highestExecutingHeight: lastExecutedHeight, - } -} - -// GetState returns current state of StopControl module -func (s *StopControl) GetState() StopControlState { - s.RLock() - defer s.RUnlock() - return s.state -} - -// IsPaused returns true is block execution has been paused -func (s *StopControl) IsPaused() bool { - s.RLock() - defer s.RUnlock() - return s.state == StopControlPaused -} - -// SetStopHeight sets new stopHeight and crash mode, and return old values: -// - stopHeight -// - crash -// -// Returns error if the stopping process has already commenced, new values will be rejected. -func (s *StopControl) SetStopHeight( - height uint64, - crash bool, -) (uint64, bool, error) { - s.Lock() - defer s.Unlock() - - oldHeight := s.stopHeight - oldCrash := s.crash - - if s.state == StopControlCommenced { - return oldHeight, - oldCrash, - fmt.Errorf( - "cannot update stopHeight, "+ - "stopping commenced for stopHeight %d with crash=%t", - oldHeight, - oldCrash, - ) - } - - if s.state == StopControlPaused { - return oldHeight, - oldCrash, - fmt.Errorf("cannot update stopHeight, already paused") - } - - // cannot set stopHeight to block which is already executing - // so the lowest possible stopHeight is highestExecutingHeight+1 - if height <= s.highestExecutingHeight { - return oldHeight, - oldCrash, - fmt.Errorf( - "cannot update stopHeight, "+ - "given stopHeight %d below or equal to highest executing height %d", - height, - s.highestExecutingHeight, - ) - } - - s.log.Info(). - Int8("previous_state", int8(s.state)). - Int8("new_state", int8(StopControlSet)). - Uint64("stopHeight", height). - Bool("crash", crash). - Uint64("old_height", oldHeight). - Bool("old_crash", oldCrash). - Msg("new stopHeight set") - - s.state = StopControlSet - - s.stopHeight = height - s.crash = crash - s.stopAfterExecuting = flow.ZeroID - - return oldHeight, oldCrash, nil -} - -// GetStopHeight returns: -// - stopHeight -// - crash -// -// Values are undefined if they were not previously set -func (s *StopControl) GetStopHeight() (uint64, bool) { - s.RLock() - defer s.RUnlock() - - return s.stopHeight, s.crash -} - -// blockProcessable should be called when new block is processable. -// It returns boolean indicating if the block should be processed. -func (s *StopControl) blockProcessable(b *flow.Header) bool { - s.Lock() - defer s.Unlock() - - if s.state == StopControlOff { - return true - } - - if s.state == StopControlPaused { - return false - } - - // skips blocks at or above requested stopHeight - if b.Height >= s.stopHeight { - s.log.Warn(). - Int8("previous_state", int8(s.state)). - Int8("new_state", int8(StopControlCommenced)). - Msgf( - "Skipping execution of %s at height %d"+ - " because stop has been requested at height %d", - b.ID(), - b.Height, - s.stopHeight, - ) - - s.state = StopControlCommenced // if block was skipped, move into commenced state - return false - } - - return true -} - -// blockFinalized should be called when a block is marked as finalized -func (s *StopControl) blockFinalized( - ctx context.Context, - execState state.ReadOnlyExecutionState, - h *flow.Header, -) { - - s.Lock() - defer s.Unlock() - - if s.state == StopControlOff || s.state == StopControlPaused { - return - } - - // Once finalization reached stopHeight we can be sure no other fork will be valid at this height, - // if this block's parent has been executed, we are safe to stop or crash. - // This will happen during normal execution, where blocks are executed before they are finalized. - // However, it is possible that EN block computation progress can fall behind. In this case, - // we want to crash only after the execution reached the stopHeight. - if h.Height == s.stopHeight { - - executed, err := state.IsBlockExecuted(ctx, execState, h.ParentID) - if err != nil { - // any error here would indicate unexpected storage error, so we crash the node - // TODO: what if the error is due to the node being stopped? - // i.e. context cancelled? - s.log.Fatal(). - Err(err). - Str("block_id", h.ID().String()). - Msg("failed to check if the block has been executed") - return - } - - if executed { - s.stopExecution() - } else { - s.stopAfterExecuting = h.ParentID - s.log.Info(). - Msgf( - "Node scheduled to stop executing"+ - " after executing block %s at height %d", - s.stopAfterExecuting.String(), - h.Height-1, - ) - } - } -} - -// blockExecuted should be called after a block has finished execution -func (s *StopControl) blockExecuted(h *flow.Header) { - s.Lock() - defer s.Unlock() - - if s.state == StopControlPaused || s.state == StopControlOff { - return - } - - if s.stopAfterExecuting == h.ID() { - // double check. Even if requested stopHeight has been changed multiple times, - // as long as it matches this block we are safe to terminate - if h.Height == s.stopHeight-1 { - s.stopExecution() - } else { - s.log.Warn(). - Msgf( - "Inconsistent stopping state. "+ - "Scheduled to stop after executing block ID %s and height %d, "+ - "but this block has a height %d. ", - h.ID().String(), - s.stopHeight-1, - h.Height, - ) - } - } -} - -func (s *StopControl) stopExecution() { - if s.crash { - s.log.Fatal().Msgf( - "Crashing as finalization reached requested "+ - "stop height %d and the highest executed block is (%d - 1)", - s.stopHeight, - s.stopHeight, - ) - return - } - - s.log.Debug(). - Int8("previous_state", int8(s.state)). - Int8("new_state", int8(StopControlPaused)). - Msg("StopControl state transition") - - s.state = StopControlPaused - - s.log.Warn().Msgf( - "Pausing execution as finalization reached "+ - "the requested stop height %d", - s.stopHeight, - ) - -} - -// executingBlockHeight should be called while execution of height starts, -// used for internal tracking of the minimum possible value of stopHeight -func (s *StopControl) executingBlockHeight(height uint64) { - // TODO: should we lock here? - - if s.state == StopControlPaused { - return - } - - // updating the highest executing height, which will be used to reject setting - // stopHeight that is too low. - if height > s.highestExecutingHeight { - s.highestExecutingHeight = height - } -} diff --git a/engine/execution/ingestion/stop_control_test.go b/engine/execution/ingestion/stop_control_test.go deleted file mode 100644 index 500278f56f5..00000000000 --- a/engine/execution/ingestion/stop_control_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package ingestion - -import ( - "context" - "testing" - - "github.com/onflow/flow-go/storage" - - testifyMock "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/engine/execution/state/mock" - - "github.com/onflow/flow-go/utils/unittest" -) - -// If stopping mechanism has caused any changes to execution flow (skipping execution of blocks) -// we disallow setting new values -func TestCannotSetNewValuesAfterStoppingCommenced(t *testing.T) { - - t.Run("when processing block at stop height", func(t *testing.T) { - sc := NewStopControl(unittest.Logger(), false, 0) - - require.Equal(t, sc.GetState(), StopControlOff) - - // first update is always successful - _, _, err := sc.SetStopHeight(21, false) - require.NoError(t, err) - - require.Equal(t, sc.GetState(), StopControlSet) - - // no stopping has started yet, block below stop height - header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) - sc.blockProcessable(header) - - require.Equal(t, sc.GetState(), StopControlSet) - - _, _, err = sc.SetStopHeight(37, false) - require.NoError(t, err) - - // block at stop height, it should be skipped - header = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(37)) - sc.blockProcessable(header) - - require.Equal(t, sc.GetState(), StopControlCommenced) - - _, _, err = sc.SetStopHeight(2137, false) - require.Error(t, err) - - // state did not change - require.Equal(t, sc.GetState(), StopControlCommenced) - }) - - t.Run("when processing finalized blocks", func(t *testing.T) { - - execState := new(mock.ReadOnlyExecutionState) - - sc := NewStopControl(unittest.Logger(), false, 0) - - require.Equal(t, sc.GetState(), StopControlOff) - - // first update is always successful - _, _, err := sc.SetStopHeight(21, false) - require.NoError(t, err) - require.Equal(t, sc.GetState(), StopControlSet) - - // make execution check pretends block has been executed - execState.On("StateCommitmentByBlockID", testifyMock.Anything, testifyMock.Anything).Return(nil, nil) - - // no stopping has started yet, block below stop height - header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) - sc.blockFinalized(context.TODO(), execState, header) - - _, _, err = sc.SetStopHeight(37, false) - require.NoError(t, err) - require.Equal(t, sc.GetState(), StopControlSet) - - // block at stop height, it should be trigger stop - header = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(37)) - sc.blockFinalized(context.TODO(), execState, header) - - // since we set crash to false, execution should be paused - require.Equal(t, sc.GetState(), StopControlPaused) - - _, _, err = sc.SetStopHeight(2137, false) - require.Error(t, err) - - execState.AssertExpectations(t) - }) -} - -// TestExecutionFallingBehind check if StopControl behaves properly even if EN runs behind -// and blocks are finalized before they are executed -func TestExecutionFallingBehind(t *testing.T) { - - execState := new(mock.ReadOnlyExecutionState) - - headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) - headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21 - headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22 - headerD := unittest.BlockHeaderWithParentFixture(headerC) // 23 - - sc := NewStopControl(unittest.Logger(), false, 0) - - require.Equal(t, sc.GetState(), StopControlOff) - - // set stop at 22, so 21 is the last height which should be processed - _, _, err := sc.SetStopHeight(22, false) - require.NoError(t, err) - require.Equal(t, sc.GetState(), StopControlSet) - - execState.On("StateCommitmentByBlockID", testifyMock.Anything, headerC.ParentID).Return(nil, storage.ErrNotFound) - - // finalize blocks first - sc.blockFinalized(context.TODO(), execState, headerA) - require.Equal(t, StopControlSet, sc.GetState()) - - sc.blockFinalized(context.TODO(), execState, headerB) - require.Equal(t, StopControlSet, sc.GetState()) - - sc.blockFinalized(context.TODO(), execState, headerC) - require.Equal(t, StopControlSet, sc.GetState()) - - sc.blockFinalized(context.TODO(), execState, headerD) - require.Equal(t, StopControlSet, sc.GetState()) - - // simulate execution - sc.blockExecuted(headerA) - require.Equal(t, StopControlSet, sc.GetState()) - - sc.blockExecuted(headerB) - require.Equal(t, StopControlPaused, sc.GetState()) - - execState.AssertExpectations(t) -} - -// TestCannotSetHeightBelowLastExecuted check if StopControl -// tracks last executed height and prevents from setting stop height -// below or too close to it -func TestCannotSetHeightBelowLastExecuted(t *testing.T) { - - sc := NewStopControl(unittest.Logger(), false, 0) - - require.Equal(t, sc.GetState(), StopControlOff) - - sc.executingBlockHeight(20) - require.Equal(t, StopControlOff, sc.GetState()) - - _, _, err := sc.SetStopHeight(20, false) - require.Error(t, err) - require.Equal(t, StopControlOff, sc.GetState()) - - _, _, err = sc.SetStopHeight(25, false) - require.NoError(t, err) - require.Equal(t, StopControlSet, sc.GetState()) -} - -// StopControl started as paused will keep the state -func TestStartingPaused(t *testing.T) { - - sc := NewStopControl(unittest.Logger(), true, 0) - require.Equal(t, StopControlPaused, sc.GetState()) -} - -func TestPausedStateRejectsAllBlocksAndChanged(t *testing.T) { - - sc := NewStopControl(unittest.Logger(), true, 0) - require.Equal(t, StopControlPaused, sc.GetState()) - - _, _, err := sc.SetStopHeight(2137, true) - require.Error(t, err) - - // make sure we don't even query executed status if paused - // mock should fail test on any method call - execState := new(mock.ReadOnlyExecutionState) - - header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20)) - - sc.blockFinalized(context.TODO(), execState, header) - require.Equal(t, StopControlPaused, sc.GetState()) - - execState.AssertExpectations(t) -} diff --git a/engine/execution/ingestion/uploader/model.go b/engine/execution/ingestion/uploader/model.go index 555f6121c08..fc39dd08393 100644 --- a/engine/execution/ingestion/uploader/model.go +++ b/engine/execution/ingestion/uploader/model.go @@ -23,16 +23,16 @@ type BlockData struct { func ComputationResultToBlockData(computationResult *execution.ComputationResult) *BlockData { - txResults := make([]*flow.TransactionResult, len(computationResult.TransactionResults)) - for i := 0; i < len(computationResult.TransactionResults); i++ { - txResults[i] = &computationResult.TransactionResults[i] + AllResults := computationResult.AllTransactionResults() + txResults := make([]*flow.TransactionResult, len(AllResults)) + for i := 0; i < len(AllResults); i++ { + txResults[i] = &AllResults[i] } - events := make([]*flow.Event, 0) - for _, eventsList := range computationResult.Events { - for i := 0; i < len(eventsList); i++ { - events = append(events, &eventsList[i]) - } + eventsList := computationResult.AllEvents() + events := make([]*flow.Event, len(eventsList)) + for i := 0; i < len(eventsList); i++ { + events[i] = &eventsList[i] } trieUpdates := make( @@ -49,7 +49,7 @@ func ComputationResultToBlockData(computationResult *execution.ComputationResult TxResults: txResults, Events: events, TrieUpdates: trieUpdates, - FinalStateCommitment: computationResult.EndState, + FinalStateCommitment: computationResult.CurrentEndState(), } } diff --git a/engine/execution/ingestion/uploader/model_test.go b/engine/execution/ingestion/uploader/model_test.go index df09eeede50..5f78824ebe4 100644 --- a/engine/execution/ingestion/uploader/model_test.go +++ b/engine/execution/ingestion/uploader/model_test.go @@ -7,11 +7,11 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/engine/execution" + "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/pathfinder" "github.com/onflow/flow-go/ledger/complete" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/utils/unittest" ) @@ -23,24 +23,33 @@ func Test_ComputationResultToBlockDataConversion(t *testing.T) { assert.Equal(t, cr.ExecutableBlock.Block, blockData.Block) assert.Equal(t, cr.ExecutableBlock.Collections(), blockData.Collections) - require.Equal(t, len(cr.TransactionResults), len(blockData.TxResults)) - for i, result := range cr.TransactionResults { + + allTxResults := cr.AllTransactionResults() + require.Equal(t, len(allTxResults), len(blockData.TxResults)) + for i, result := range allTxResults { assert.Equal(t, result, *blockData.TxResults[i]) } - eventsCombined := make([]flow.Event, 0) - for _, eventsList := range cr.Events { - eventsCombined = append(eventsCombined, eventsList...) + // Since returned events are not preserving orders, + // use map with event.ID() as key to confirm all events + // are included. + allEvents := cr.AllEvents() + require.Equal(t, len(allEvents), len(blockData.Events)) + + eventsInBlockData := make(map[flow.Identifier]flow.Event) + for _, e := range blockData.Events { + eventsInBlockData[e.ID()] = *e } - require.Equal(t, len(eventsCombined), len(blockData.Events)) - for i, event := range eventsCombined { - assert.Equal(t, event, *blockData.Events[i]) + for _, expectedEvent := range allEvents { + event, ok := eventsInBlockData[expectedEvent.ID()] + require.True(t, ok) + require.Equal(t, expectedEvent, event) } - assert.Equal(t, expectedTrieUpdates, blockData.TrieUpdates) + assert.Equal(t, len(expectedTrieUpdates), len(blockData.TrieUpdates)) - assert.Equal(t, cr.EndState, blockData.FinalStateCommitment) + assert.Equal(t, cr.CurrentEndState(), blockData.FinalStateCommitment) } func generateComputationResult( @@ -105,81 +114,10 @@ func generateComputationResult( trieUpdate4, err := pathfinder.UpdateToTrieUpdate(update4, complete.DefaultPathFinderVersion) require.NoError(t, err) - - return &execution.ComputationResult{ - ExecutableBlock: unittest.ExecutableBlockFixture([][]flow.Identifier{ - {unittest.IdentifierFixture()}, - {unittest.IdentifierFixture()}, - {unittest.IdentifierFixture()}, - }), - StateSnapshots: nil, - Events: []flow.EventsList{ - { - unittest.EventFixture("what", 0, 0, unittest.IdentifierFixture(), 2), - unittest.EventFixture("ever", 0, 1, unittest.IdentifierFixture(), 22), - }, - {}, - { - unittest.EventFixture("what", 2, 0, unittest.IdentifierFixture(), 2), - unittest.EventFixture("ever", 2, 1, unittest.IdentifierFixture(), 22), - unittest.EventFixture("ever", 2, 2, unittest.IdentifierFixture(), 2), - unittest.EventFixture("ever", 2, 3, unittest.IdentifierFixture(), 22), - }, - {}, // system chunk events - }, - EventsHashes: nil, - ServiceEvents: nil, - TransactionResults: []flow.TransactionResult{ - { - TransactionID: unittest.IdentifierFixture(), - ErrorMessage: "", - ComputationUsed: 23, - }, - { - TransactionID: unittest.IdentifierFixture(), - ErrorMessage: "fail", - ComputationUsed: 1, - }, - }, - TransactionResultIndex: []int{1, 1, 2, 2}, - BlockExecutionData: &execution_data.BlockExecutionData{ - ChunkExecutionDatas: []*execution_data.ChunkExecutionData{ - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate1, - }, - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate2, - }, - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate3, - }, - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate4, - }, - }, - }, - ExecutionReceipt: &flow.ExecutionReceipt{ - ExecutionResult: flow.ExecutionResult{ - Chunks: flow.ChunkList{ - { - EndState: unittest.StateCommitmentFixture(), - }, - { - EndState: unittest.StateCommitmentFixture(), - }, - { - EndState: unittest.StateCommitmentFixture(), - }, - { - EndState: unittest.StateCommitmentFixture(), - }, - }, - }, - }, - }, []*ledger.TrieUpdate{ - trieUpdate1, - trieUpdate2, - trieUpdate3, - trieUpdate4, - } + return testutil.ComputationResultFixture(t), []*ledger.TrieUpdate{ + trieUpdate1, + trieUpdate2, + trieUpdate3, + trieUpdate4, + } } diff --git a/engine/execution/ingestion/uploader/retryable_uploader_wrapper.go b/engine/execution/ingestion/uploader/retryable_uploader_wrapper.go index b010a14c2f0..ecad4801741 100644 --- a/engine/execution/ingestion/uploader/retryable_uploader_wrapper.go +++ b/engine/execution/ingestion/uploader/retryable_uploader_wrapper.go @@ -181,7 +181,7 @@ func (b *BadgerRetryableUploaderWrapper) reconstructComputationResult( executionDataID := executionResult.ExecutionDataID // retrieving BlockExecutionData from EDS - executionData, err := b.execDataDownloader.Download(b.unit.Ctx(), executionDataID) + executionData, err := b.execDataDownloader.Get(b.unit.Ctx(), executionDataID) if executionData == nil || err != nil { log.Error().Err(err).Msgf( "failed to retrieve BlockExecutionData from EDS with ID %s", executionDataID.String()) @@ -237,15 +237,41 @@ func (b *BadgerRetryableUploaderWrapper) reconstructComputationResult( log.Warn().Msgf("failed to retrieve StateCommitment with BlockID %s. Error: %s", blockID.String(), err.Error()) } + executableBlock := &entity.ExecutableBlock{ + Block: block, + CompleteCollections: completeCollections, + } + + compRes := execution.NewEmptyComputationResult(executableBlock) + + eventsByTxIndex := make(map[int]flow.EventsList, 0) + for _, event := range events { + idx := int(event.TransactionIndex) + eventsByTxIndex[idx] = append(eventsByTxIndex[idx], event) + } + + lastChunk := len(completeCollections) + lastCollection := compRes.CollectionExecutionResultAt(lastChunk) + for i, txRes := range transactionResults { + lastCollection.AppendTransactionResults( + eventsByTxIndex[i], + nil, + nil, + txRes, + ) + } + + compRes.AppendCollectionAttestationResult( + endState, + endState, + nil, + flow.ZeroID, + nil, + ) + + compRes.BlockExecutionData = executionData + // for now we only care about fields in BlockData - return &execution.ComputationResult{ - ExecutableBlock: &entity.ExecutableBlock{ - Block: block, - CompleteCollections: completeCollections, - }, - Events: []flow.EventsList{events}, - TransactionResults: transactionResults, - BlockExecutionData: executionData, - EndState: endState, - }, nil + // Warning: this seems so broken just do the job, i only maintained previous behviour + return compRes, nil } diff --git a/engine/execution/ingestion/uploader/retryable_uploader_wrapper_test.go b/engine/execution/ingestion/uploader/retryable_uploader_wrapper_test.go index 9e7cf641c60..491307705eb 100644 --- a/engine/execution/ingestion/uploader/retryable_uploader_wrapper_test.go +++ b/engine/execution/ingestion/uploader/retryable_uploader_wrapper_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/rs/zerolog" "github.com/onflow/flow-go/ledger" @@ -32,8 +31,8 @@ func Test_Upload_invoke(t *testing.T) { dummyUploader := &DummyUploader{ f: func() error { - wg.Done() uploaderCalled = true + wg.Done() return nil }, } @@ -64,8 +63,8 @@ func Test_RetryUpload(t *testing.T) { uploaderCalled := false dummyUploader := &DummyUploader{ f: func() error { - wg.Done() uploaderCalled = true + wg.Done() return nil }, } @@ -110,18 +109,20 @@ func Test_ReconstructComputationResultFromStorage(t *testing.T) { testBlockID := flow.HashToID([]byte{1, 2, 3}) testEDID := flow.HashToID([]byte{4, 5, 6}) testTrieUpdateRootHash, _ := ledger.ToRootHash([]byte{7, 8, 9}) + testTrieUpdate := &ledger.TrieUpdate{ + RootHash: testTrieUpdateRootHash, + } testChunkExecutionDatas := []*execution_data.ChunkExecutionData{ { - TrieUpdate: &ledger.TrieUpdate{ - RootHash: testTrieUpdateRootHash, - }, + TrieUpdate: testTrieUpdate, }, } testEvents := []flow.Event{ - unittest.EventFixture(flow.EventAccountCreated, 1, 0, flow.HashToID([]byte{11, 22, 33}), 200), + unittest.EventFixture(flow.EventAccountCreated, 0, 0, flow.HashToID([]byte{11, 22, 33}), 200), } testCollectionID := flow.HashToID([]byte{0xA, 0xB, 0xC}) testBlock := &flow.Block{ + Header: &flow.Header{}, Payload: &flow.Payload{ Guarantees: []*flow.CollectionGuarantee{ { @@ -168,7 +169,7 @@ func Test_ReconstructComputationResultFromStorage(t *testing.T) { mockComputationResultStorage.On("Upsert", testBlockID, mock.Anything).Return(nil) mockExecutionDataDowloader := new(executionDataMock.Downloader) - mockExecutionDataDowloader.On("Download", mock.Anything, testEDID).Return( + mockExecutionDataDowloader.On("Get", mock.Anything, testEDID).Return( &execution_data.BlockExecutionData{ BlockID: testBlockID, ChunkExecutionDatas: testChunkExecutionDatas, @@ -196,40 +197,33 @@ func Test_ReconstructComputationResultFromStorage(t *testing.T) { reconstructedComputationResult, err := testRetryableUploaderWrapper.reconstructComputationResult(testBlockID) assert.NilError(t, err) - expectedCompleteCollections := make(map[flow.Identifier]*entity.CompleteCollection) - expectedCompleteCollections[testCollectionID] = &entity.CompleteCollection{ + expectedCompleteCollections := make([]*entity.CompleteCollection, 1) + expectedCompleteCollections[0] = &entity.CompleteCollection{ Guarantee: &flow.CollectionGuarantee{ CollectionID: testCollectionID, }, Transactions: []*flow.TransactionBody{testTransactionBody}, } - expectedComputationResult := &execution.ComputationResult{ - ExecutableBlock: &entity.ExecutableBlock{ - Block: testBlock, - CompleteCollections: expectedCompleteCollections, - }, - Events: []flow.EventsList{testEvents}, - TransactionResults: []flow.TransactionResult{ - testTransactionResult, - }, - BlockExecutionData: &execution_data.BlockExecutionData{ - BlockID: testBlockID, - ChunkExecutionDatas: []*execution_data.ChunkExecutionData{ - &execution_data.ChunkExecutionData{ - TrieUpdate: &ledger.TrieUpdate{ - RootHash: testTrieUpdateRootHash, - }, - }, - }, - }, - EndState: testStateCommit, + + expectedTestEvents := make([]*flow.Event, len(testEvents)) + for i, event := range testEvents { + expectedTestEvents[i] = &event + } + + expectedBlockData := &BlockData{ + Block: testBlock, + Collections: expectedCompleteCollections, + TxResults: []*flow.TransactionResult{&testTransactionResult}, + Events: expectedTestEvents, + TrieUpdates: []*ledger.TrieUpdate{testTrieUpdate}, + FinalStateCommitment: testStateCommit, } assert.DeepEqual( t, - expectedComputationResult, - reconstructedComputationResult, - cmpopts.IgnoreUnexported(entity.ExecutableBlock{})) + expectedBlockData, + ComputationResultToBlockData(reconstructedComputationResult), + ) } // createTestBadgerRetryableUploaderWrapper() create BadgerRetryableUploaderWrapper instance with given @@ -265,8 +259,7 @@ func createTestBadgerRetryableUploaderWrapper(asyncUploader *AsyncUploader) *Bad mockComputationResultStorage.On("Upsert", mock.Anything, mock.Anything).Return(nil) mockExecutionDataDowloader := new(executionDataMock.Downloader) - mockExecutionDataDowloader.On("Add", mock.Anything, mock.Anything).Return(flow.ZeroID, nil, nil) - mockExecutionDataDowloader.On("Download", mock.Anything, mock.Anything).Return( + mockExecutionDataDowloader.On("Get", mock.Anything, mock.Anything).Return( &execution_data.BlockExecutionData{ BlockID: flow.ZeroID, ChunkExecutionDatas: make([]*execution_data.ChunkExecutionData, 0), @@ -288,9 +281,9 @@ func createTestBadgerRetryableUploaderWrapper(asyncUploader *AsyncUploader) *Bad // createTestComputationResult() creates ComputationResult with valid ExecutableBlock ID func createTestComputationResult() *execution.ComputationResult { - testComputationResult := &execution.ComputationResult{} blockA := unittest.BlockHeaderFixture() - blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA) - testComputationResult.ExecutableBlock = blockB + start := unittest.StateCommitmentFixture() + blockB := unittest.ExecutableBlockFixtureWithParent(nil, blockA, &start) + testComputationResult := execution.NewEmptyComputationResult(blockB) return testComputationResult } diff --git a/engine/execution/messages.go b/engine/execution/messages.go index 4ee1b1a061f..64763ff0a46 100644 --- a/engine/execution/messages.go +++ b/engine/execution/messages.go @@ -1,112 +1,34 @@ package execution import ( - "github.com/onflow/flow-go/fvm/meter" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/module/mempool/entity" ) -// TODO(patrick): rm unaccessed fields type ComputationResult struct { - *entity.ExecutableBlock - StateSnapshots []*state.ExecutionSnapshot - Events []flow.EventsList - EventsHashes []flow.Identifier - ServiceEvents flow.EventsList - TransactionResults []flow.TransactionResult - TransactionResultIndex []int + *BlockExecutionResult + *BlockAttestationResult - // TODO(patrick): switch this to execution snapshot - ComputationIntensities meter.MeteredComputationIntensities - - ChunkDataPacks []*flow.ChunkDataPack - EndState flow.StateCommitment - - *execution_data.BlockExecutionData *flow.ExecutionReceipt } func NewEmptyComputationResult( block *entity.ExecutableBlock, ) *ComputationResult { - numCollections := len(block.CompleteCollections) + 1 + ber := NewPopulatedBlockExecutionResult(block) + aer := NewEmptyBlockAttestationResult(ber) return &ComputationResult{ - ExecutableBlock: block, - StateSnapshots: make([]*state.ExecutionSnapshot, 0, numCollections), - Events: make([]flow.EventsList, numCollections), - EventsHashes: make([]flow.Identifier, 0, numCollections), - ServiceEvents: make(flow.EventsList, 0), - TransactionResults: make([]flow.TransactionResult, 0), - TransactionResultIndex: make([]int, 0), - ComputationIntensities: make(meter.MeteredComputationIntensities), - ChunkDataPacks: make([]*flow.ChunkDataPack, 0, numCollections), - EndState: *block.StartState, - BlockExecutionData: &execution_data.BlockExecutionData{ - BlockID: block.ID(), - ChunkExecutionDatas: make( - []*execution_data.ChunkExecutionData, - 0, - numCollections), - }, - } -} - -func (cr ComputationResult) transactionResultsByCollectionIndex(colIndex int) []flow.TransactionResult { - var startTxnIndex int - if colIndex > 0 { - startTxnIndex = cr.TransactionResultIndex[colIndex-1] + BlockExecutionResult: ber, + BlockAttestationResult: aer, } - endTxnIndex := cr.TransactionResultIndex[colIndex] - return cr.TransactionResults[startTxnIndex:endTxnIndex] } -func (cr *ComputationResult) CollectionResult(colIndex int) *ColResSnapshot { - if colIndex < 0 && colIndex > len(cr.CompleteCollections) { - return nil +// CurrentEndState returns the most recent end state +// if no attestation appended yet, it returns start state of block +// TODO(ramtin): we probably don't need this long term as part of this method +func (cr *ComputationResult) CurrentEndState() flow.StateCommitment { + if len(cr.collectionAttestationResults) == 0 { + return *cr.StartState } - return &ColResSnapshot{ - blockHeader: cr.Block.Header, - collection: &flow.Collection{ - Transactions: cr.CollectionAt(colIndex).Transactions, - }, - updatedRegisters: cr.StateSnapshots[colIndex].UpdatedRegisters(), - readRegisterIDs: cr.StateSnapshots[colIndex].ReadRegisterIDs(), - emittedEvents: cr.Events[colIndex], - transactionResults: cr.transactionResultsByCollectionIndex(colIndex), - } -} - -type ColResSnapshot struct { - blockHeader *flow.Header - collection *flow.Collection - updatedRegisters flow.RegisterEntries - readRegisterIDs flow.RegisterIDs - emittedEvents flow.EventsList - transactionResults flow.TransactionResults -} - -func (c *ColResSnapshot) BlockHeader() *flow.Header { - return c.blockHeader -} - -func (c *ColResSnapshot) Collection() *flow.Collection { - return c.collection -} - -func (c *ColResSnapshot) UpdatedRegisters() flow.RegisterEntries { - return c.updatedRegisters -} - -func (c *ColResSnapshot) ReadRegisterIDs() flow.RegisterIDs { - return c.readRegisterIDs -} - -func (c *ColResSnapshot) EmittedEvents() flow.EventsList { - return c.emittedEvents -} - -func (c *ColResSnapshot) TransactionResults() flow.TransactionResults { - return c.transactionResults + return cr.collectionAttestationResults[len(cr.collectionAttestationResults)-1].endStateCommit } diff --git a/engine/execution/ingestion/mock/ingest_rpc.go b/engine/execution/mock/engines.go similarity index 74% rename from engine/execution/ingestion/mock/ingest_rpc.go rename to engine/execution/mock/engines.go index 0359b5e4a0c..658b10db2cb 100644 --- a/engine/execution/ingestion/mock/ingest_rpc.go +++ b/engine/execution/mock/engines.go @@ -10,13 +10,13 @@ import ( mock "github.com/stretchr/testify/mock" ) -// IngestRPC is an autogenerated mock type for the IngestRPC type -type IngestRPC struct { +// ScriptExecutor is an autogenerated mock type for the ScriptExecutor type +type ScriptExecutor struct { mock.Mock } // ExecuteScriptAtBlockID provides a mock function with given fields: ctx, script, arguments, blockID -func (_m *IngestRPC) ExecuteScriptAtBlockID(ctx context.Context, script []byte, arguments [][]byte, blockID flow.Identifier) ([]byte, error) { +func (_m *ScriptExecutor) ExecuteScriptAtBlockID(ctx context.Context, script []byte, arguments [][]byte, blockID flow.Identifier) ([]byte, error) { ret := _m.Called(ctx, script, arguments, blockID) var r0 []byte @@ -42,7 +42,7 @@ func (_m *IngestRPC) ExecuteScriptAtBlockID(ctx context.Context, script []byte, } // GetAccount provides a mock function with given fields: ctx, address, blockID -func (_m *IngestRPC) GetAccount(ctx context.Context, address flow.Address, blockID flow.Identifier) (*flow.Account, error) { +func (_m *ScriptExecutor) GetAccount(ctx context.Context, address flow.Address, blockID flow.Identifier) (*flow.Account, error) { ret := _m.Called(ctx, address, blockID) var r0 *flow.Account @@ -68,7 +68,7 @@ func (_m *IngestRPC) GetAccount(ctx context.Context, address flow.Address, block } // GetRegisterAtBlockID provides a mock function with given fields: ctx, owner, key, blockID -func (_m *IngestRPC) GetRegisterAtBlockID(ctx context.Context, owner []byte, key []byte, blockID flow.Identifier) ([]byte, error) { +func (_m *ScriptExecutor) GetRegisterAtBlockID(ctx context.Context, owner []byte, key []byte, blockID flow.Identifier) ([]byte, error) { ret := _m.Called(ctx, owner, key, blockID) var r0 []byte @@ -93,14 +93,14 @@ func (_m *IngestRPC) GetRegisterAtBlockID(ctx context.Context, owner []byte, key return r0, r1 } -type mockConstructorTestingTNewIngestRPC interface { +type mockConstructorTestingTNewScriptExecutor interface { mock.TestingT Cleanup(func()) } -// NewIngestRPC creates a new instance of IngestRPC. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewIngestRPC(t mockConstructorTestingTNewIngestRPC) *IngestRPC { - mock := &IngestRPC{} +// NewScriptExecutor creates a new instance of ScriptExecutor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewScriptExecutor(t mockConstructorTestingTNewScriptExecutor) *ScriptExecutor { + mock := &ScriptExecutor{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) diff --git a/engine/execution/provider/engine.go b/engine/execution/provider/engine.go index bea81dc26b5..3343dab1648 100644 --- a/engine/execution/provider/engine.go +++ b/engine/execution/provider/engine.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "math/rand" "time" "github.com/rs/zerolog" @@ -25,6 +24,7 @@ import ( "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/logging" + "github.com/onflow/flow-go/utils/rand" ) type ProviderEngine interface { @@ -266,6 +266,10 @@ func (e *Engine) onChunkDataRequest(request *mempool.ChunkDataPackRequest) { Logger() lg.Info().Msg("started processing chunk data pack request") + // TODO(ramtin): we might add a future logic to do extra checks on the origin of the request + // currently the networking layer checks that the requested is a valid node operator + // that has not been ejected. + // increases collector metric e.metrics.ChunkDataPackRequestProcessed() chunkDataPack, err := e.execState.ChunkDataPackByChunkID(request.ChunkId) @@ -293,14 +297,6 @@ func (e *Engine) onChunkDataRequest(request *mempool.ChunkDataPackRequest) { Msg("chunk data pack query takes longer than expected timeout") } - _, err = e.ensureAuthorized(chunkDataPack.ChunkID, request.RequesterId) - if err != nil { - lg.Error(). - Err(err). - Msg("could not verify authorization of identity of chunk data pack request") - return - } - e.deliverChunkDataResponse(chunkDataPack, request.RequesterId) } @@ -315,12 +311,24 @@ func (e *Engine) deliverChunkDataResponse(chunkDataPack *flow.ChunkDataPack, req // sends requested chunk data pack to the requester deliveryStartTime := time.Now() + nonce, err := rand.Uint64() + if err != nil { + // TODO: this error should be returned by deliverChunkDataResponse + // it is logged for now since the only error possible is related to a failure + // of the system entropy generation. Such error is going to cause failures in other + // components where it's handled properly and will lead to crashing the module. + lg.Error(). + Err(err). + Msg("could not generate nonce for chunk data response") + return + } + response := &messages.ChunkDataResponse{ ChunkDataPack: *chunkDataPack, - Nonce: rand.Uint64(), + Nonce: nonce, } - err := e.chunksConduit.Unicast(response, requesterId) + err = e.chunksConduit.Unicast(response, requesterId) if err != nil { lg.Warn(). Err(err). @@ -346,36 +354,6 @@ func (e *Engine) deliverChunkDataResponse(chunkDataPack *flow.ChunkDataPack, req lg.Info().Msg("chunk data pack request successfully replied") } -func (e *Engine) ensureAuthorized(chunkID flow.Identifier, originID flow.Identifier) (*flow.Identity, error) { - blockID, err := e.execState.GetBlockIDByChunkID(chunkID) - if err != nil { - return nil, engine.NewInvalidInputErrorf("cannot find blockID corresponding to chunk data pack: %w", err) - } - - authorizedAt, err := e.checkAuthorizedAtBlock(blockID) - if err != nil { - return nil, engine.NewInvalidInputErrorf("cannot check block staking status: %w", err) - } - if !authorizedAt { - return nil, engine.NewInvalidInputErrorf("this node is not authorized at the block (%s) corresponding to chunk data pack (%s)", blockID.String(), chunkID.String()) - } - - origin, err := e.state.AtBlockID(blockID).Identity(originID) - if err != nil { - return nil, engine.NewInvalidInputErrorf("invalid origin id (%s): %w", origin, err) - } - - // only verifier nodes are allowed to request chunk data packs - if origin.Role != flow.RoleVerification { - return nil, engine.NewInvalidInputErrorf("invalid role for receiving collection: %s", origin.Role) - } - - if origin.Weight == 0 { - return nil, engine.NewInvalidInputErrorf("node %s has zero weight at the block (%s) corresponding to chunk data pack (%s)", originID, blockID.String(), chunkID.String()) - } - return origin, nil -} - func (e *Engine) BroadcastExecutionReceipt(ctx context.Context, receipt *flow.ExecutionReceipt) error { finalState, err := receipt.ExecutionResult.FinalStateCommitment() if err != nil { diff --git a/engine/execution/provider/engine_test.go b/engine/execution/provider/engine_test.go index 1411061b123..d47f4b0ccae 100644 --- a/engine/execution/provider/engine_test.go +++ b/engine/execution/provider/engine_test.go @@ -11,7 +11,6 @@ import ( _ "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "go.uber.org/atomic" state "github.com/onflow/flow-go/engine/execution/state/mock" "github.com/onflow/flow-go/model/flow" @@ -22,189 +21,11 @@ import ( "github.com/onflow/flow-go/module/trace" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/mocknetwork" - "github.com/onflow/flow-go/state/protocol" mockprotocol "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) func TestProviderEngine_onChunkDataRequest(t *testing.T) { - t.Run("non-verification engine", func(t *testing.T) { - ps := mockprotocol.NewState(t) - ss := mockprotocol.NewSnapshot(t) - net := mocknetwork.NewNetwork(t) - chunkConduit := mocknetwork.NewConduit(t) - execState := state.NewExecutionState(t) - - net.On("Register", channels.PushReceipts, mock.Anything).Return(&mocknetwork.Conduit{}, nil) - net.On("Register", channels.ProvideChunks, mock.Anything).Return(chunkConduit, nil) - requestQueue := queue.NewHeroStore(10, unittest.Logger(), metrics.NewNoopCollector()) - - e, err := New( - unittest.Logger(), - trace.NewNoopTracer(), - net, - ps, - execState, - metrics.NewNoopCollector(), - func(_ flow.Identifier) (bool, error) { return true, nil }, - requestQueue, - DefaultChunkDataPackRequestWorker, - DefaultChunkDataPackQueryTimeout, - DefaultChunkDataPackDeliveryTimeout) - require.NoError(t, err) - - originID := unittest.IdentifierFixture() - chunkID := unittest.IdentifierFixture() - blockID := unittest.IdentifierFixture() - chunkDataPack := unittest.ChunkDataPackFixture(chunkID) - - ps.On("AtBlockID", blockID).Return(ss).Once() - ss.On("Identity", originID).Return(unittest.IdentityFixture(unittest.WithRole(flow.RoleExecution)), nil) - execState.On("ChunkDataPackByChunkID", mock.Anything).Return(chunkDataPack, nil) - execState.On("GetBlockIDByChunkID", chunkID).Return(blockID, nil) - - req := &messages.ChunkDataRequest{ - ChunkID: chunkID, - Nonce: rand.Uint64(), - } - - cancelCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - ctx, _ := irrecoverable.WithSignaler(cancelCtx) - e.Start(ctx) - // submit using origin ID with invalid role - unittest.RequireCloseBefore(t, e.Ready(), 100*time.Millisecond, "could not start engine") - require.NoError(t, e.Process(channels.RequestChunks, originID, req)) - - require.Eventually(t, func() bool { - _, ok := requestQueue.Get() // ensuring all requests have been picked up from the queue. - return !ok - }, 1*time.Second, 10*time.Millisecond) - - cancel() - unittest.RequireCloseBefore(t, e.Done(), 100*time.Millisecond, "could not stop engine") - - // no chunk data pack response should be sent to an invalid role's request - chunkConduit.AssertNotCalled(t, "Unicast") - }) - - t.Run("unauthorized (0 weight) origin", func(t *testing.T) { - ps := mockprotocol.NewState(t) - ss := mockprotocol.NewSnapshot(t) - net := mocknetwork.NewNetwork(t) - chunkConduit := mocknetwork.NewConduit(t) - execState := state.NewExecutionState(t) - - net.On("Register", channels.PushReceipts, mock.Anything).Return(&mocknetwork.Conduit{}, nil) - net.On("Register", channels.ProvideChunks, mock.Anything).Return(chunkConduit, nil) - requestQueue := queue.NewHeroStore(10, unittest.Logger(), metrics.NewNoopCollector()) - - e, err := New( - unittest.Logger(), - trace.NewNoopTracer(), - net, - ps, - execState, - metrics.NewNoopCollector(), - func(_ flow.Identifier) (bool, error) { return true, nil }, - requestQueue, - DefaultChunkDataPackRequestWorker, - DefaultChunkDataPackQueryTimeout, - DefaultChunkDataPackDeliveryTimeout) - require.NoError(t, err) - - originID := unittest.IdentifierFixture() - chunkID := unittest.IdentifierFixture() - blockID := unittest.IdentifierFixture() - chunkDataPack := unittest.ChunkDataPackFixture(chunkID) - - ps.On("AtBlockID", blockID).Return(ss).Once() - ss.On("Identity", originID).Return(unittest.IdentityFixture(unittest.WithRole(flow.RoleExecution), unittest.WithWeight(0)), nil) - execState.On("ChunkDataPackByChunkID", mock.Anything).Return(chunkDataPack, nil) - execState.On("GetBlockIDByChunkID", chunkID).Return(blockID, nil) - - req := &messages.ChunkDataRequest{ - ChunkID: chunkID, - Nonce: rand.Uint64(), - } - cancelCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - ctx, _ := irrecoverable.WithSignaler(cancelCtx) - e.Start(ctx) - // submit using origin ID with zero weight - unittest.RequireCloseBefore(t, e.Ready(), 100*time.Millisecond, "could not start engine") - require.NoError(t, e.Process(channels.RequestChunks, originID, req)) - - require.Eventually(t, func() bool { - _, ok := requestQueue.Get() // ensuring all requests have been picked up from the queue. - return !ok - }, 1*time.Second, 10*time.Millisecond) - - cancel() - unittest.RequireCloseBefore(t, e.Done(), 100*time.Millisecond, "could not stop engine") - - // no chunk data pack response should be sent to a request coming from 0-weight node - chunkConduit.AssertNotCalled(t, "Unicast") - }) - - t.Run("un-authorized (not found origin) origin", func(t *testing.T) { - ps := mockprotocol.NewState(t) - ss := mockprotocol.NewSnapshot(t) - net := mocknetwork.NewNetwork(t) - chunkConduit := mocknetwork.NewConduit(t) - execState := state.NewExecutionState(t) - - net.On("Register", channels.PushReceipts, mock.Anything).Return(&mocknetwork.Conduit{}, nil) - net.On("Register", channels.ProvideChunks, mock.Anything).Return(chunkConduit, nil) - requestQueue := queue.NewHeroStore(10, unittest.Logger(), metrics.NewNoopCollector()) - - e, err := New( - unittest.Logger(), - trace.NewNoopTracer(), - net, - ps, - execState, - metrics.NewNoopCollector(), - func(_ flow.Identifier) (bool, error) { return true, nil }, - requestQueue, - DefaultChunkDataPackRequestWorker, - DefaultChunkDataPackQueryTimeout, - DefaultChunkDataPackDeliveryTimeout) - require.NoError(t, err) - - originID := unittest.IdentifierFixture() - chunkID := unittest.IdentifierFixture() - blockID := unittest.IdentifierFixture() - chunkDataPack := unittest.ChunkDataPackFixture(chunkID) - - ps.On("AtBlockID", blockID).Return(ss).Once() - ss.On("Identity", originID).Return(nil, protocol.IdentityNotFoundError{}) - execState.On("ChunkDataPackByChunkID", mock.Anything).Return(chunkDataPack, nil) - execState.On("GetBlockIDByChunkID", chunkID).Return(blockID, nil) - - req := &messages.ChunkDataRequest{ - ChunkID: chunkID, - Nonce: rand.Uint64(), - } - cancelCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - ctx, _ := irrecoverable.WithSignaler(cancelCtx) - e.Start(ctx) - // submit using non-existing origin ID - unittest.RequireCloseBefore(t, e.Ready(), 100*time.Millisecond, "could not start engine") - require.NoError(t, e.Process(channels.RequestChunks, originID, req)) - - require.Eventually(t, func() bool { - _, ok := requestQueue.Get() // ensuring all requests have been picked up from the queue. - return !ok - }, 1*time.Second, 10*time.Millisecond) - - cancel() - unittest.RequireCloseBefore(t, e.Done(), 100*time.Millisecond, "could not stop engine") - - // no chunk data pack response should be sent to a request coming from a non-existing origin ID - chunkConduit.AssertNotCalled(t, "Unicast") - }) t.Run("non-existent chunk", func(t *testing.T) { ps := mockprotocol.NewState(t) @@ -304,7 +125,6 @@ func TestProviderEngine_onChunkDataRequest(t *testing.T) { }). Return(nil) - execState.On("GetBlockIDByChunkID", chunkID).Return(blockID, nil) execState.On("ChunkDataPackByChunkID", chunkID).Return(chunkDataPack, nil) req := &messages.ChunkDataRequest{ @@ -329,82 +149,4 @@ func TestProviderEngine_onChunkDataRequest(t *testing.T) { unittest.RequireCloseBefore(t, e.Done(), 100*time.Millisecond, "could not stop engine") }) - t.Run("reply to chunk data pack request only when authorized", func(t *testing.T) { - currentAuthorizedState := atomic.Bool{} - currentAuthorizedState.Store(true) - ps := mockprotocol.NewState(t) - ss := mockprotocol.NewSnapshot(t) - net := mocknetwork.NewNetwork(t) - chunkConduit := mocknetwork.NewConduit(t) - execState := state.NewExecutionState(t) - - net.On("Register", channels.PushReceipts, mock.Anything).Return(&mocknetwork.Conduit{}, nil) - net.On("Register", channels.ProvideChunks, mock.Anything).Return(chunkConduit, nil) - requestQueue := queue.NewHeroStore(10, unittest.Logger(), metrics.NewNoopCollector()) - - e, err := New( - unittest.Logger(), - trace.NewNoopTracer(), - net, - ps, - execState, - metrics.NewNoopCollector(), - func(_ flow.Identifier) (bool, error) { return currentAuthorizedState.Load(), nil }, - requestQueue, - DefaultChunkDataPackRequestWorker, - DefaultChunkDataPackQueryTimeout, - DefaultChunkDataPackDeliveryTimeout) - require.NoError(t, err) - - originIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) - - chunkID := unittest.IdentifierFixture() - chunkDataPack := unittest.ChunkDataPackFixture(chunkID) - blockID := unittest.IdentifierFixture() - - execState.On("GetBlockIDByChunkID", chunkID).Return(blockID, nil) - ps.On("AtBlockID", blockID).Return(ss).Once() - ss.On("Identity", originIdentity.NodeID).Return(originIdentity, nil).Once() - - // channel tracking for the first chunk data pack request responded. - chunkConduit.On("Unicast", mock.Anything, originIdentity.NodeID). - Run(func(args mock.Arguments) { - res, ok := args[0].(*messages.ChunkDataResponse) - require.True(t, ok) - - actualChunkID := res.ChunkDataPack.ChunkID - assert.Equal(t, chunkID, actualChunkID) - }). - Return(nil).Once() - - execState.On("ChunkDataPackByChunkID", chunkID).Return(chunkDataPack, nil).Twice() - - req := &messages.ChunkDataRequest{ - ChunkID: chunkID, - Nonce: rand.Uint64(), - } - - cancelCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - ctx, _ := irrecoverable.WithSignaler(cancelCtx) - e.Start(ctx) - // submit using non-existing origin ID - unittest.RequireCloseBefore(t, e.Ready(), 100*time.Millisecond, "could not start engine") - require.NoError(t, e.Process(channels.RequestChunks, originIdentity.NodeID, req)) - - require.Eventually(t, func() bool { - _, ok := requestQueue.Get() // ensuring first request has been picked up from the queue. - return !ok - }, 1*time.Second, 100*time.Millisecond) - currentAuthorizedState.Store(false) - - require.NoError(t, e.Process(channels.RequestChunks, originIdentity.NodeID, req)) - require.Eventually(t, func() bool { - _, ok := requestQueue.Get() // ensuring second request has been picked up from the queue as well. - return !ok - }, 1*time.Second, 10*time.Millisecond) - - cancel() - unittest.RequireCloseBefore(t, e.Done(), 100*time.Millisecond, "could not stop engine") - }) } diff --git a/engine/execution/rpc/engine.go b/engine/execution/rpc/engine.go index 407fc3bedef..08de8c5919c 100644 --- a/engine/execution/rpc/engine.go +++ b/engine/execution/rpc/engine.go @@ -20,7 +20,7 @@ import ( "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" - "github.com/onflow/flow-go/engine/execution/ingestion" + exeEng "github.com/onflow/flow-go/engine/execution" fvmerrors "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" @@ -47,7 +47,7 @@ type Engine struct { func New( log zerolog.Logger, config Config, - e *ingestion.Engine, + scriptsExecutor exeEng.ScriptExecutor, headers storage.Headers, state protocol.State, events storage.Events, @@ -88,7 +88,7 @@ func New( log: log, unit: engine.NewUnit(), handler: &handler{ - engine: e, + engine: scriptsExecutor, chain: chainID, headers: headers, state: state, @@ -147,7 +147,7 @@ func (e *Engine) serve() { // handler implements a subset of the Observation API. type handler struct { - engine ingestion.IngestRPC + engine exeEng.ScriptExecutor chain flow.ChainID headers storage.Headers state protocol.State @@ -256,7 +256,8 @@ func (h *handler) GetEventsForBlockIDs(_ context.Context, } return &execution.GetEventsForBlockIDsResponse{ - Results: results, + Results: results, + EventEncodingVersion: execution.EventEncodingVersion_CCF_V0, }, nil } @@ -316,9 +317,10 @@ func (h *handler) GetTransactionResult( // compose a response with the events and the transaction error return &execution.GetTransactionResultResponse{ - StatusCode: statusCode, - ErrorMessage: errMsg, - Events: events, + StatusCode: statusCode, + ErrorMessage: errMsg, + Events: events, + EventEncodingVersion: execution.EventEncodingVersion_CCF_V0, }, nil } @@ -374,9 +376,10 @@ func (h *handler) GetTransactionResultByIndex( // compose a response with the events and the transaction error return &execution.GetTransactionResultResponse{ - StatusCode: statusCode, - ErrorMessage: errMsg, - Events: events, + StatusCode: statusCode, + ErrorMessage: errMsg, + Events: events, + EventEncodingVersion: execution.EventEncodingVersion_CCF_V0, }, nil } @@ -452,7 +455,8 @@ func (h *handler) GetTransactionResultsByBlockID( // compose a response return &execution.GetTransactionResultsResponse{ - TransactionResults: responseTxResults, + TransactionResults: responseTxResults, + EventEncodingVersion: execution.EventEncodingVersion_CCF_V0, }, nil } diff --git a/engine/execution/rpc/engine_test.go b/engine/execution/rpc/engine_test.go index ffbbd4b0a37..ed85f6043e3 100644 --- a/engine/execution/rpc/engine_test.go +++ b/engine/execution/rpc/engine_test.go @@ -18,7 +18,7 @@ import ( "github.com/onflow/flow/protobuf/go/flow/execution" "github.com/onflow/flow-go/engine/common/rpc/convert" - ingestion "github.com/onflow/flow-go/engine/execution/ingestion/mock" + mockEng "github.com/onflow/flow-go/engine/execution/mock" "github.com/onflow/flow-go/model/flow" realstorage "github.com/onflow/flow-go/storage" storage "github.com/onflow/flow-go/storage/mock" @@ -51,7 +51,7 @@ func (suite *Suite) SetupTest() { // TestExecuteScriptAtBlockID tests the ExecuteScriptAtBlockID API call func (suite *Suite) TestExecuteScriptAtBlockID() { // setup handler - mockEngine := new(ingestion.IngestRPC) + mockEngine := new(mockEng.ScriptExecutor) handler := &handler{ engine: mockEngine, chain: flow.Mainnet, @@ -242,7 +242,7 @@ func (suite *Suite) TestGetAccountAtBlockID() { Address: serviceAddress, } - mockEngine := new(ingestion.IngestRPC) + mockEngine := new(mockEng.ScriptExecutor) // create the handler handler := &handler{ @@ -301,7 +301,7 @@ func (suite *Suite) TestGetRegisterAtBlockID() { serviceAddress := flow.Mainnet.Chain().ServiceAddress() validKey := []byte("exists") - mockEngine := new(ingestion.IngestRPC) + mockEngine := new(mockEng.ScriptExecutor) // create the handler handler := &handler{ diff --git a/engine/execution/scripts/engine.go b/engine/execution/scripts/engine.go new file mode 100644 index 00000000000..35ab0dbec55 --- /dev/null +++ b/engine/execution/scripts/engine.go @@ -0,0 +1,135 @@ +package scripts + +import ( + "context" + "encoding/hex" + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/execution" + "github.com/onflow/flow-go/engine/execution/computation" + "github.com/onflow/flow-go/engine/execution/state" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" +) + +type Engine struct { + unit *engine.Unit + log zerolog.Logger + state protocol.State + computationManager computation.ComputationManager + execState state.ExecutionState +} + +var _ execution.ScriptExecutor = (*Engine)(nil) + +func New( + logger zerolog.Logger, + state protocol.State, + computationManager computation.ComputationManager, + execState state.ExecutionState, +) *Engine { + return &Engine{ + unit: engine.NewUnit(), + log: logger.With().Str("engine", "scripts").Logger(), + state: state, + execState: execState, + computationManager: computationManager, + } +} + +func (e *Engine) Ready() <-chan struct{} { + return e.unit.Ready() +} + +func (e *Engine) Done() <-chan struct{} { + return e.unit.Done() +} + +func (e *Engine) ExecuteScriptAtBlockID( + ctx context.Context, + script []byte, + arguments [][]byte, + blockID flow.Identifier, +) ([]byte, error) { + + stateCommit, err := e.execState.StateCommitmentByBlockID(ctx, blockID) + if err != nil { + return nil, fmt.Errorf("failed to get state commitment for block (%s): %w", blockID, err) + } + + // return early if state with the given state commitment is not in memory + // and already purged. This reduces allocations for scripts targeting old blocks. + if !e.execState.HasState(stateCommit) { + return nil, fmt.Errorf("failed to execute script at block (%s): state commitment not found (%s). this error usually happens if the reference block for this script is not set to a recent block", blockID.String(), hex.EncodeToString(stateCommit[:])) + } + + header, err := e.state.AtBlockID(blockID).Head() + if err != nil { + return nil, fmt.Errorf("failed to get header (%s): %w", blockID, err) + } + + blockSnapshot := e.execState.NewStorageSnapshot(stateCommit) + + return e.computationManager.ExecuteScript( + ctx, + script, + arguments, + header, + blockSnapshot) +} + +func (e *Engine) GetRegisterAtBlockID( + ctx context.Context, + owner, key []byte, + blockID flow.Identifier, +) ([]byte, error) { + + stateCommit, err := e.execState.StateCommitmentByBlockID(ctx, blockID) + if err != nil { + return nil, fmt.Errorf("failed to get state commitment for block (%s): %w", blockID, err) + } + + blockSnapshot := e.execState.NewStorageSnapshot(stateCommit) + + id := flow.NewRegisterID(string(owner), string(key)) + data, err := blockSnapshot.Get(id) + if err != nil { + return nil, fmt.Errorf("failed to get the register (%s): %w", id, err) + } + + return data, nil +} + +func (e *Engine) GetAccount( + ctx context.Context, + addr flow.Address, + blockID flow.Identifier, +) (*flow.Account, error) { + stateCommit, err := e.execState.StateCommitmentByBlockID(ctx, blockID) + if err != nil { + return nil, fmt.Errorf("failed to get state commitment for block (%s): %w", blockID, err) + } + + // return early if state with the given state commitment is not in memory + // and already purged. This reduces allocations for get accounts targeting old blocks. + if !e.execState.HasState(stateCommit) { + return nil, fmt.Errorf( + "failed to get account at block (%s): state commitment not "+ + "found (%s). this error usually happens if the reference "+ + "block for this script is not set to a recent block.", + blockID.String(), + hex.EncodeToString(stateCommit[:])) + } + + block, err := e.state.AtBlockID(blockID).Head() + if err != nil { + return nil, fmt.Errorf("failed to get block (%s): %w", blockID, err) + } + + blockSnapshot := e.execState.NewStorageSnapshot(stateCommit) + + return e.computationManager.GetAccount(ctx, addr, block, blockSnapshot) +} diff --git a/engine/execution/scripts/engine_test.go b/engine/execution/scripts/engine_test.go new file mode 100644 index 00000000000..9f954530f0e --- /dev/null +++ b/engine/execution/scripts/engine_test.go @@ -0,0 +1,114 @@ +package scripts + +import ( + "context" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + computation "github.com/onflow/flow-go/engine/execution/computation/mock" + stateMock "github.com/onflow/flow-go/engine/execution/state/mock" + "github.com/onflow/flow-go/model/flow" + protocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +type testingContext struct { + t *testing.T + engine *Engine + state *protocol.State + executionState *stateMock.ExecutionState + computationManager *computation.ComputationManager + mu *sync.Mutex +} + +func (ctx *testingContext) stateCommitmentExist(blockID flow.Identifier, commit flow.StateCommitment) { + ctx.executionState.On("StateCommitmentByBlockID", mock.Anything, blockID).Return(commit, nil) +} + +func runWithEngine(t *testing.T, fn func(ctx testingContext)) { + log := unittest.Logger() + + computationManager := new(computation.ComputationManager) + protocolState := new(protocol.State) + execState := new(stateMock.ExecutionState) + + engine := New(log, protocolState, computationManager, execState) + fn(testingContext{ + t: t, + engine: engine, + computationManager: computationManager, + executionState: execState, + state: protocolState, + }) +} + +func TestExecuteScriptAtBlockID(t *testing.T) { + t.Run("happy path", func(t *testing.T) { + runWithEngine(t, func(ctx testingContext) { + // Meaningless script + script := []byte{1, 1, 2, 3, 5, 8, 11} + scriptResult := []byte{1} + + // Ensure block we're about to query against is executable + blockA := unittest.ExecutableBlockFixture(nil, unittest.StateCommitmentPointerFixture()) + + snapshot := new(protocol.Snapshot) + snapshot.On("Head").Return(blockA.Block.Header, nil) + + commits := make(map[flow.Identifier]flow.StateCommitment) + commits[blockA.ID()] = *blockA.StartState + + ctx.stateCommitmentExist(blockA.ID(), *blockA.StartState) + + ctx.state.On("AtBlockID", blockA.Block.ID()).Return(snapshot) + ctx.executionState.On("NewStorageSnapshot", *blockA.StartState).Return(nil) + + ctx.executionState.On("HasState", *blockA.StartState).Return(true) + + // Successful call to computation manager + ctx.computationManager. + On("ExecuteScript", mock.Anything, script, [][]byte(nil), blockA.Block.Header, nil). + Return(scriptResult, nil) + + // Execute our script and expect no error + res, err := ctx.engine.ExecuteScriptAtBlockID(context.Background(), script, nil, blockA.Block.ID()) + assert.NoError(t, err) + assert.Equal(t, scriptResult, res) + + // Assert other components were called as expected + ctx.computationManager.AssertExpectations(t) + ctx.executionState.AssertExpectations(t) + ctx.state.AssertExpectations(t) + }) + }) + + t.Run("return early when state commitment not exist", func(t *testing.T) { + runWithEngine(t, func(ctx testingContext) { + // Meaningless script + script := []byte{1, 1, 2, 3, 5, 8, 11} + + // Ensure block we're about to query against is executable + blockA := unittest.ExecutableBlockFixture(nil, unittest.StateCommitmentPointerFixture()) + + // make sure blockID to state commitment mapping exist + ctx.executionState.On("StateCommitmentByBlockID", mock.Anything, blockA.ID()).Return(*blockA.StartState, nil) + + // but the state commitment does not exist (e.g. purged) + ctx.executionState.On("HasState", *blockA.StartState).Return(false) + + // Execute our script and expect no error + _, err := ctx.engine.ExecuteScriptAtBlockID(context.Background(), script, nil, blockA.Block.ID()) + assert.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "state commitment not found")) + + // Assert other components were called as expected + ctx.executionState.AssertExpectations(t) + ctx.state.AssertExpectations(t) + }) + }) + +} diff --git a/engine/execution/state/bootstrap/bootstrap.go b/engine/execution/state/bootstrap/bootstrap.go index 1808b77cfb6..0addc1665d0 100644 --- a/engine/execution/state/bootstrap/bootstrap.go +++ b/engine/execution/state/bootstrap/bootstrap.go @@ -9,7 +9,7 @@ import ( "github.com/onflow/flow-go/engine/execution/state" "github.com/onflow/flow-go/fvm" - fvmstate "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/storage" @@ -94,27 +94,36 @@ func (b *Bootstrapper) IsBootstrapped(db *badger.DB) (flow.StateCommitment, bool return commit, true, nil } -func (b *Bootstrapper) BootstrapExecutionDatabase(db *badger.DB, commit flow.StateCommitment, genesis *flow.Header) error { +func (b *Bootstrapper) BootstrapExecutionDatabase( + db *badger.DB, + rootSeal *flow.Seal, +) error { + commit := rootSeal.FinalState err := operation.RetryOnConflict(db.Update, func(txn *badger.Txn) error { - err := operation.InsertExecutedBlock(genesis.ID())(txn) + err := operation.InsertExecutedBlock(rootSeal.BlockID)(txn) if err != nil { return fmt.Errorf("could not index initial genesis execution block: %w", err) } + err = operation.SkipDuplicates(operation.IndexExecutionResult(rootSeal.BlockID, rootSeal.ResultID))(txn) + if err != nil { + return fmt.Errorf("could not index result for root result: %w", err) + } + err = operation.IndexStateCommitment(flow.ZeroID, commit)(txn) if err != nil { return fmt.Errorf("could not index void state commitment: %w", err) } - err = operation.IndexStateCommitment(genesis.ID(), commit)(txn) + err = operation.IndexStateCommitment(rootSeal.BlockID, commit)(txn) if err != nil { return fmt.Errorf("could not index genesis state commitment: %w", err) } - snapshots := make([]*fvmstate.ExecutionSnapshot, 0) - err = operation.InsertExecutionStateInteractions(genesis.ID(), snapshots)(txn) + snapshots := make([]*snapshot.ExecutionSnapshot, 0) + err = operation.InsertExecutionStateInteractions(rootSeal.BlockID, snapshots)(txn) if err != nil { return fmt.Errorf("could not bootstrap execution state interactions: %w", err) } diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index 43a136bd93a..1d84b2938db 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("af1e147676cda8cf292a1725cd9414ac81d8b6dc07e72ad346ab1f30c3453803") + expectedStateCommitmentBytes, _ := hex.DecodeString("986c540657fdb3b4154311069d901223a3268492f678ae706010cd537cc328ad") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/engine/execution/state/delta/view.go b/engine/execution/state/delta/view.go deleted file mode 100644 index f56dd21eec9..00000000000 --- a/engine/execution/state/delta/view.go +++ /dev/null @@ -1,11 +0,0 @@ -package delta - -// TODO(patrick): rm after updating emulator - -import ( - "github.com/onflow/flow-go/fvm/state" -) - -func NewDeltaView(storage state.StorageSnapshot) state.View { - return state.NewSpockState(storage) -} diff --git a/engine/execution/state/mock/execution_state.go b/engine/execution/state/mock/execution_state.go index 864660e79d8..f847632cd94 100644 --- a/engine/execution/state/mock/execution_state.go +++ b/engine/execution/state/mock/execution_state.go @@ -8,9 +8,9 @@ import ( execution "github.com/onflow/flow-go/engine/execution" flow "github.com/onflow/flow-go/model/flow" - fvmstate "github.com/onflow/flow-go/fvm/state" - mock "github.com/stretchr/testify/mock" + + snapshot "github.com/onflow/flow-go/fvm/storage/snapshot" ) // ExecutionState is an autogenerated mock type for the ExecutionState type @@ -44,32 +44,6 @@ func (_m *ExecutionState) ChunkDataPackByChunkID(_a0 flow.Identifier) (*flow.Chu return r0, r1 } -// GetBlockIDByChunkID provides a mock function with given fields: chunkID -func (_m *ExecutionState) GetBlockIDByChunkID(chunkID flow.Identifier) (flow.Identifier, error) { - ret := _m.Called(chunkID) - - var r0 flow.Identifier - var r1 error - if rf, ok := ret.Get(0).(func(flow.Identifier) (flow.Identifier, error)); ok { - return rf(chunkID) - } - if rf, ok := ret.Get(0).(func(flow.Identifier) flow.Identifier); ok { - r0 = rf(chunkID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.Identifier) - } - } - - if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { - r1 = rf(chunkID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetExecutionResultID provides a mock function with given fields: _a0, _a1 func (_m *ExecutionState) GetExecutionResultID(_a0 context.Context, _a1 flow.Identifier) (flow.Identifier, error) { ret := _m.Called(_a0, _a1) @@ -144,15 +118,15 @@ func (_m *ExecutionState) HasState(_a0 flow.StateCommitment) bool { } // NewStorageSnapshot provides a mock function with given fields: _a0 -func (_m *ExecutionState) NewStorageSnapshot(_a0 flow.StateCommitment) fvmstate.StorageSnapshot { +func (_m *ExecutionState) NewStorageSnapshot(_a0 flow.StateCommitment) snapshot.StorageSnapshot { ret := _m.Called(_a0) - var r0 fvmstate.StorageSnapshot - if rf, ok := ret.Get(0).(func(flow.StateCommitment) fvmstate.StorageSnapshot); ok { + var r0 snapshot.StorageSnapshot + if rf, ok := ret.Get(0).(func(flow.StateCommitment) snapshot.StorageSnapshot); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(fvmstate.StorageSnapshot) + r0 = ret.Get(0).(snapshot.StorageSnapshot) } } diff --git a/engine/execution/state/mock/read_only_execution_state.go b/engine/execution/state/mock/read_only_execution_state.go index 246a54fc4f9..24f230ed316 100644 --- a/engine/execution/state/mock/read_only_execution_state.go +++ b/engine/execution/state/mock/read_only_execution_state.go @@ -5,10 +5,10 @@ package mock import ( context "context" - fvmstate "github.com/onflow/flow-go/fvm/state" flow "github.com/onflow/flow-go/model/flow" - mock "github.com/stretchr/testify/mock" + + snapshot "github.com/onflow/flow-go/fvm/storage/snapshot" ) // ReadOnlyExecutionState is an autogenerated mock type for the ReadOnlyExecutionState type @@ -42,32 +42,6 @@ func (_m *ReadOnlyExecutionState) ChunkDataPackByChunkID(_a0 flow.Identifier) (* return r0, r1 } -// GetBlockIDByChunkID provides a mock function with given fields: chunkID -func (_m *ReadOnlyExecutionState) GetBlockIDByChunkID(chunkID flow.Identifier) (flow.Identifier, error) { - ret := _m.Called(chunkID) - - var r0 flow.Identifier - var r1 error - if rf, ok := ret.Get(0).(func(flow.Identifier) (flow.Identifier, error)); ok { - return rf(chunkID) - } - if rf, ok := ret.Get(0).(func(flow.Identifier) flow.Identifier); ok { - r0 = rf(chunkID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.Identifier) - } - } - - if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { - r1 = rf(chunkID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetExecutionResultID provides a mock function with given fields: _a0, _a1 func (_m *ReadOnlyExecutionState) GetExecutionResultID(_a0 context.Context, _a1 flow.Identifier) (flow.Identifier, error) { ret := _m.Called(_a0, _a1) @@ -142,15 +116,15 @@ func (_m *ReadOnlyExecutionState) HasState(_a0 flow.StateCommitment) bool { } // NewStorageSnapshot provides a mock function with given fields: _a0 -func (_m *ReadOnlyExecutionState) NewStorageSnapshot(_a0 flow.StateCommitment) fvmstate.StorageSnapshot { +func (_m *ReadOnlyExecutionState) NewStorageSnapshot(_a0 flow.StateCommitment) snapshot.StorageSnapshot { ret := _m.Called(_a0) - var r0 fvmstate.StorageSnapshot - if rf, ok := ret.Get(0).(func(flow.StateCommitment) fvmstate.StorageSnapshot); ok { + var r0 snapshot.StorageSnapshot + if rf, ok := ret.Get(0).(func(flow.StateCommitment) snapshot.StorageSnapshot); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(fvmstate.StorageSnapshot) + r0 = ret.Get(0).(snapshot.StorageSnapshot) } } diff --git a/engine/execution/state/state.go b/engine/execution/state/state.go index 497cc87a8fc..0f754520493 100644 --- a/engine/execution/state/state.go +++ b/engine/execution/state/state.go @@ -9,7 +9,7 @@ import ( "github.com/dgraph-io/badger/v2" "github.com/onflow/flow-go/engine/execution" - fvmState "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -23,7 +23,7 @@ import ( // ReadOnlyExecutionState allows to read the execution state type ReadOnlyExecutionState interface { // NewStorageSnapshot creates a new ready-only view at the given state commitment. - NewStorageSnapshot(flow.StateCommitment) fvmState.StorageSnapshot + NewStorageSnapshot(flow.StateCommitment) snapshot.StorageSnapshot // StateCommitmentByBlockID returns the final state commitment for the provided block ID. StateCommitmentByBlockID(context.Context, flow.Identifier) (flow.StateCommitment, error) @@ -37,8 +37,6 @@ type ReadOnlyExecutionState interface { GetExecutionResultID(context.Context, flow.Identifier) (flow.Identifier, error) GetHighestExecutedBlockID(context.Context) (uint64, flow.Identifier, error) - - GetBlockIDByChunkID(chunkID flow.Identifier) (flow.Identifier, error) } // TODO Many operations here are should be transactional, so we need to refactor this @@ -143,8 +141,6 @@ func RegisterEntriesToKeysValues( return keys, values } -// TODO(patrick): revisit caching. readCache needs to be mutex guarded for -// parallel execution. type LedgerStorageSnapshot struct { ledger ledger.Ledger commitment flow.StateCommitment @@ -156,7 +152,7 @@ type LedgerStorageSnapshot struct { func NewLedgerStorageSnapshot( ldg ledger.Ledger, commitment flow.StateCommitment, -) fvmState.StorageSnapshot { +) snapshot.StorageSnapshot { return &LedgerStorageSnapshot{ ledger: ldg, commitment: commitment, @@ -225,7 +221,7 @@ func (storage *LedgerStorageSnapshot) Get( func (s *state) NewStorageSnapshot( commitment flow.StateCommitment, -) fvmState.StorageSnapshot { +) snapshot.StorageSnapshot { return NewLedgerStorageSnapshot(s.ls, commitment) } @@ -297,36 +293,26 @@ func (s *state) SaveExecutionResults( // but it's the closest thing to atomicity we could have batch := badgerstorage.NewBatch(s.db) - for _, chunkDataPack := range result.ChunkDataPacks { + for _, chunkDataPack := range result.AllChunkDataPacks() { err := s.chunkDataPacks.BatchStore(chunkDataPack, batch) if err != nil { return fmt.Errorf("cannot store chunk data pack: %w", err) } - - err = s.headers.BatchIndexByChunkID(blockID, chunkDataPack.ChunkID, batch) - if err != nil { - return fmt.Errorf("cannot index chunk data pack by blockID: %w", err) - } - } - - err := s.commits.BatchStore(blockID, result.EndState, batch) - if err != nil { - return fmt.Errorf("cannot store state commitment: %w", err) } - err = s.events.BatchStore(blockID, result.Events, batch) + err := s.events.BatchStore(blockID, []flow.EventsList{result.AllEvents()}, batch) if err != nil { return fmt.Errorf("cannot store events: %w", err) } - err = s.serviceEvents.BatchStore(blockID, result.ServiceEvents, batch) + err = s.serviceEvents.BatchStore(blockID, result.AllServiceEvents(), batch) if err != nil { return fmt.Errorf("cannot store service events: %w", err) } err = s.transactionResults.BatchStore( blockID, - result.TransactionResults, + result.AllTransactionResults(), batch) if err != nil { return fmt.Errorf("cannot store transaction result: %w", err) @@ -348,6 +334,14 @@ func (s *state) SaveExecutionResults( return fmt.Errorf("could not persist execution result: %w", err) } + // the state commitment is the last data item to be stored, so that + // IsBlockExecuted can be implemented by checking whether state commitment exists + // in the database + err = s.commits.BatchStore(blockID, result.CurrentEndState(), batch) + if err != nil { + return fmt.Errorf("cannot store state commitment: %w", err) + } + err = batch.Flush() if err != nil { return fmt.Errorf("batch flush error: %w", err) @@ -361,10 +355,6 @@ func (s *state) SaveExecutionResults( return nil } -func (s *state) GetBlockIDByChunkID(chunkID flow.Identifier) (flow.Identifier, error) { - return s.headers.IDByChunkID(chunkID) -} - func (s *state) UpdateHighestExecutedBlockIfHigher(ctx context.Context, header *flow.Header) error { if s.tracer != nil { span, _ := s.tracer.StartSpanFromContext(ctx, trace.EXEUpdateHighestExecutedBlockIfHigher) diff --git a/engine/execution/state/state_test.go b/engine/execution/state/state_test.go index 3a0946dd375..6d6833837f0 100644 --- a/engine/execution/state/state_test.go +++ b/engine/execution/state/state_test.go @@ -14,7 +14,7 @@ import ( "github.com/onflow/flow-go/ledger/common/pathfinder" "github.com/onflow/flow-go/engine/execution/state" - fvmstate "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" ledger "github.com/onflow/flow-go/ledger/complete" "github.com/onflow/flow-go/ledger/complete/wal/fixtures" "github.com/onflow/flow-go/model/flow" @@ -77,7 +77,7 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { sc1, err := es.StateCommitmentByBlockID(context.Background(), flow.Identifier{}) assert.NoError(t, err) - executionSnapshot := &fvmstate.ExecutionSnapshot{ + executionSnapshot := &snapshot.ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ registerID1: flow.RegisterValue("apple"), registerID2: flow.RegisterValue("carrot"), @@ -138,7 +138,7 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { sc1, err := es.StateCommitmentByBlockID(context.Background(), flow.Identifier{}) assert.NoError(t, err) - executionSnapshot1 := &fvmstate.ExecutionSnapshot{ + executionSnapshot1 := &snapshot.ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ registerID1: []byte("apple"), }, @@ -148,7 +148,7 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { assert.NoError(t, err) // update value and get resulting state commitment - executionSnapshot2 := &fvmstate.ExecutionSnapshot{ + executionSnapshot2 := &snapshot.ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ registerID1: []byte("orange"), }, @@ -180,7 +180,7 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { assert.NoError(t, err) // set initial value - executionSnapshot1 := &fvmstate.ExecutionSnapshot{ + executionSnapshot1 := &snapshot.ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ registerID1: []byte("apple"), registerID2: []byte("apple"), @@ -191,7 +191,7 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { assert.NoError(t, err) // update value and get resulting state commitment - executionSnapshot2 := &fvmstate.ExecutionSnapshot{ + executionSnapshot2 := &snapshot.ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ registerID1: nil, }, @@ -223,7 +223,7 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { assert.NoError(t, err) // set initial value - executionSnapshot1 := &fvmstate.ExecutionSnapshot{ + executionSnapshot1 := &snapshot.ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ registerID1: flow.RegisterValue("apple"), registerID2: flow.RegisterValue("apple"), diff --git a/engine/execution/state/unittest/fixtures.go b/engine/execution/state/unittest/fixtures.go index 607fbb07433..71733cd8054 100644 --- a/engine/execution/state/unittest/fixtures.go +++ b/engine/execution/state/unittest/fixtures.go @@ -3,24 +3,23 @@ package unittest import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/execution" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/module/mempool/entity" "github.com/onflow/flow-go/utils/unittest" ) -func StateInteractionsFixture() *state.ExecutionSnapshot { - return &state.ExecutionSnapshot{} +func StateInteractionsFixture() *snapshot.ExecutionSnapshot { + return &snapshot.ExecutionSnapshot{} } func ComputationResultFixture( parentBlockExecutionResultID flow.Identifier, collectionsSignerIDs [][]flow.Identifier, ) *execution.ComputationResult { - block := unittest.ExecutableBlockFixture(collectionsSignerIDs) + startState := unittest.StateCommitmentFixture() - block.StartState = &startState + block := unittest.ExecutableBlockFixture(collectionsSignerIDs, &startState) return ComputationResultForBlockFixture( parentBlockExecutionResultID, @@ -32,77 +31,43 @@ func ComputationResultForBlockFixture( completeBlock *entity.ExecutableBlock, ) *execution.ComputationResult { collections := completeBlock.Collections() + computationResult := execution.NewEmptyComputationResult(completeBlock) - numChunks := len(collections) + 1 - stateSnapshots := make([]*state.ExecutionSnapshot, numChunks) - events := make([]flow.EventsList, numChunks) - eventHashes := make([]flow.Identifier, numChunks) - spockHashes := make([]crypto.Signature, numChunks) - chunks := make([]*flow.Chunk, 0, numChunks) - chunkDataPacks := make([]*flow.ChunkDataPack, 0, numChunks) - chunkExecutionDatas := make( - []*execution_data.ChunkExecutionData, - 0, - numChunks) - for i := 0; i < numChunks; i++ { - stateSnapshots[i] = StateInteractionsFixture() - events[i] = make(flow.EventsList, 0) - eventHashes[i] = unittest.IdentifierFixture() - - chunk := flow.NewChunk( - completeBlock.ID(), - i, + numberOfChunks := len(collections) + 1 + for i := 0; i < numberOfChunks; i++ { + computationResult.CollectionExecutionResultAt(i).UpdateExecutionSnapshot(StateInteractionsFixture()) + computationResult.AppendCollectionAttestationResult( + *completeBlock.StartState, *completeBlock.StartState, - 0, + nil, unittest.IdentifierFixture(), - *completeBlock.StartState) - chunks = append(chunks, chunk) + nil, + ) - var collection *flow.Collection - if i < len(collections) { - colStruct := collections[i].Collection() - collection = &colStruct - } + } - chunkDataPacks = append( - chunkDataPacks, - flow.NewChunkDataPack( - chunk.ID(), - *completeBlock.StartState, - unittest.RandomBytes(6), - collection)) + _, serviceEventEpochCommitProtocol := unittest.EpochCommitFixtureByChainID(flow.Localnet) + _, serviceEventEpochSetupProtocol := unittest.EpochSetupFixtureByChainID(flow.Localnet) + _, serviceEventVersionBeaconProtocol := unittest.VersionBeaconFixtureByChainID(flow.Localnet) - chunkExecutionDatas = append( - chunkExecutionDatas, - &execution_data.ChunkExecutionData{ - Collection: collection, - Events: nil, - TrieUpdate: nil, - }) + convertedServiceEvents := flow.ServiceEventList{ + serviceEventEpochCommitProtocol.ServiceEvent(), + serviceEventEpochSetupProtocol.ServiceEvent(), + serviceEventVersionBeaconProtocol.ServiceEvent(), } + executionResult := flow.NewExecutionResult( parentBlockExecutionResultID, completeBlock.ID(), - chunks, - nil, + computationResult.AllChunks(), + convertedServiceEvents, flow.ZeroID) - return &execution.ComputationResult{ - TransactionResultIndex: make([]int, numChunks), - ExecutableBlock: completeBlock, - StateSnapshots: stateSnapshots, - Events: events, - EventsHashes: eventHashes, - ChunkDataPacks: chunkDataPacks, - EndState: *completeBlock.StartState, - BlockExecutionData: &execution_data.BlockExecutionData{ - BlockID: completeBlock.ID(), - ChunkExecutionDatas: chunkExecutionDatas, - }, - ExecutionReceipt: &flow.ExecutionReceipt{ - ExecutionResult: *executionResult, - Spocks: spockHashes, - ExecutorSignature: crypto.Signature{}, - }, + computationResult.ExecutionReceipt = &flow.ExecutionReceipt{ + ExecutionResult: *executionResult, + Spocks: make([]crypto.Signature, numberOfChunks), + ExecutorSignature: crypto.Signature{}, } + + return computationResult } diff --git a/engine/execution/testutil/fixtures.go b/engine/execution/testutil/fixtures.go index a68e801ab82..3113f2df9af 100644 --- a/engine/execution/testutil/fixtures.go +++ b/engine/execution/testutil/fixtures.go @@ -8,16 +8,27 @@ import ( "testing" "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/crypto/hash" + "github.com/onflow/flow-go/engine/execution" "github.com/onflow/flow-go/engine/execution/utils" "github.com/onflow/flow-go/fvm" - "github.com/onflow/flow-go/fvm/storage" + "github.com/onflow/flow-go/fvm/environment" + envMock "github.com/onflow/flow-go/fvm/environment/mock" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/pathfinder" + "github.com/onflow/flow-go/ledger/complete" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/epochs" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/state/protocol" + protocolMock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -187,11 +198,11 @@ func GenerateAccountPrivateKey() (flow.AccountPrivateKey, error) { // CreateAccounts inserts accounts into the ledger using the provided private keys. func CreateAccounts( vm fvm.VM, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, privateKeys []flow.AccountPrivateKey, chain flow.Chain, ) ( - storage.SnapshotTree, + snapshot.SnapshotTree, []flow.Address, error, ) { @@ -204,11 +215,11 @@ func CreateAccounts( func CreateAccountsWithSimpleAddresses( vm fvm.VM, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, privateKeys []flow.AccountPrivateKey, chain flow.Chain, ) ( - storage.SnapshotTree, + snapshot.SnapshotTree, []flow.Address, error, ) { @@ -276,7 +287,7 @@ func CreateAccountsWithSimpleAddresses( for _, event := range output.Events { if event.Type == flow.EventAccountCreated { - data, err := jsoncdc.Decode(nil, event.Payload) + data, err := ccf.Decode(nil, event.Payload) if err != nil { return snapshotTree, nil, errors.New( "error decoding events") @@ -300,7 +311,7 @@ func RootBootstrappedLedger( vm fvm.VM, ctx fvm.Context, additionalOptions ...fvm.BootstrapProcedureOption, -) storage.SnapshotTree { +) snapshot.SnapshotTree { // set 0 clusters to pass n_collectors >= n_clusters check epochConfig := epochs.DefaultEpochConfig() epochConfig.NumCollectorClusters = 0 @@ -317,11 +328,11 @@ func RootBootstrappedLedger( options..., ) - snapshot, _, err := vm.Run(ctx, bootstrap, nil) + executionSnapshot, _, err := vm.Run(ctx, bootstrap, nil) if err != nil { panic(err) } - return storage.NewSnapshotTree(nil).Append(snapshot) + return snapshot.NewSnapshotTree(nil).Append(executionSnapshot) } func BytesToCadenceArray(l []byte) cadence.Array { @@ -496,3 +507,154 @@ func bytesToCadenceArray(l []byte) cadence.Array { return cadence.NewArray(values) } + +// TODO(ramtin): when we get rid of BlockExecutionData, this could move to the global unittest fixtures +// TrieUpdates are internal data to the ledger package and should not have leaked into +// packages like uploader in the first place +func ComputationResultFixture(t *testing.T) *execution.ComputationResult { + startState := unittest.StateCommitmentFixture() + update1, err := ledger.NewUpdate( + ledger.State(startState), + []ledger.Key{ + ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(3, []byte{33})}), + ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(1, []byte{11})}), + ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(2, []byte{1, 1}), ledger.NewKeyPart(3, []byte{2, 5})}), + }, + []ledger.Value{ + []byte{21, 37}, + nil, + []byte{3, 3, 3, 3, 3}, + }, + ) + require.NoError(t, err) + + trieUpdate1, err := pathfinder.UpdateToTrieUpdate(update1, complete.DefaultPathFinderVersion) + require.NoError(t, err) + + update2, err := ledger.NewUpdate( + ledger.State(unittest.StateCommitmentFixture()), + []ledger.Key{}, + []ledger.Value{}, + ) + require.NoError(t, err) + + trieUpdate2, err := pathfinder.UpdateToTrieUpdate(update2, complete.DefaultPathFinderVersion) + require.NoError(t, err) + + update3, err := ledger.NewUpdate( + ledger.State(unittest.StateCommitmentFixture()), + []ledger.Key{ + ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(9, []byte{6})}), + }, + []ledger.Value{ + []byte{21, 37}, + }, + ) + require.NoError(t, err) + + trieUpdate3, err := pathfinder.UpdateToTrieUpdate(update3, complete.DefaultPathFinderVersion) + require.NoError(t, err) + + update4, err := ledger.NewUpdate( + ledger.State(unittest.StateCommitmentFixture()), + []ledger.Key{ + ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(9, []byte{6})}), + }, + []ledger.Value{ + []byte{21, 37}, + }, + ) + require.NoError(t, err) + + trieUpdate4, err := pathfinder.UpdateToTrieUpdate(update4, complete.DefaultPathFinderVersion) + require.NoError(t, err) + + executableBlock := unittest.ExecutableBlockFixture([][]flow.Identifier{ + {unittest.IdentifierFixture()}, + {unittest.IdentifierFixture()}, + {unittest.IdentifierFixture()}, + }, &startState) + + blockExecResult := execution.NewPopulatedBlockExecutionResult(executableBlock) + blockExecResult.CollectionExecutionResultAt(0).AppendTransactionResults( + flow.EventsList{ + unittest.EventFixture("what", 0, 0, unittest.IdentifierFixture(), 2), + unittest.EventFixture("ever", 0, 1, unittest.IdentifierFixture(), 22), + }, + nil, + nil, + flow.TransactionResult{ + TransactionID: unittest.IdentifierFixture(), + ErrorMessage: "", + ComputationUsed: 23, + MemoryUsed: 101, + }, + ) + blockExecResult.CollectionExecutionResultAt(1).AppendTransactionResults( + flow.EventsList{ + unittest.EventFixture("what", 2, 0, unittest.IdentifierFixture(), 2), + unittest.EventFixture("ever", 2, 1, unittest.IdentifierFixture(), 22), + unittest.EventFixture("ever", 2, 2, unittest.IdentifierFixture(), 2), + unittest.EventFixture("ever", 2, 3, unittest.IdentifierFixture(), 22), + }, + nil, + nil, + flow.TransactionResult{ + TransactionID: unittest.IdentifierFixture(), + ErrorMessage: "fail", + ComputationUsed: 1, + MemoryUsed: 22, + }, + ) + + return &execution.ComputationResult{ + BlockExecutionResult: blockExecResult, + BlockAttestationResult: &execution.BlockAttestationResult{ + BlockExecutionData: &execution_data.BlockExecutionData{ + ChunkExecutionDatas: []*execution_data.ChunkExecutionData{ + {TrieUpdate: trieUpdate1}, + {TrieUpdate: trieUpdate2}, + {TrieUpdate: trieUpdate3}, + {TrieUpdate: trieUpdate4}, + }, + }, + }, + ExecutionReceipt: &flow.ExecutionReceipt{ + ExecutionResult: flow.ExecutionResult{ + Chunks: flow.ChunkList{ + {EndState: unittest.StateCommitmentFixture()}, + {EndState: unittest.StateCommitmentFixture()}, + {EndState: unittest.StateCommitmentFixture()}, + {EndState: unittest.StateCommitmentFixture()}, + }, + }, + }, + } +} + +// EntropyProviderFixture returns an entropy provider mock that +// supports RandomSource(). +// If input is nil, a random source fixture is generated. +func EntropyProviderFixture(source []byte) environment.EntropyProvider { + if source == nil { + source = unittest.SignatureFixture() + } + provider := envMock.EntropyProvider{} + provider.On("RandomSource").Return(source, nil) + return &provider +} + +// ProtocolStateWithSourceFixture returns a protocol state mock that only +// supports AtBlockID to return a snapshot mock. +// The snapshot mock only supports RandomSource(). +// If input is nil, a random source fixture is generated. +func ProtocolStateWithSourceFixture(source []byte) protocol.State { + if source == nil { + source = unittest.SignatureFixture() + } + snapshot := &protocolMock.Snapshot{} + snapshot.On("RandomSource").Return(source, nil) + state := protocolMock.State{} + state.On("AtBlockID", mock.Anything).Return(snapshot) + return &state +} diff --git a/engine/protocol/api.go b/engine/protocol/api.go index 319be377605..5f0451896d2 100644 --- a/engine/protocol/api.go +++ b/engine/protocol/api.go @@ -13,6 +13,7 @@ import ( type NetworkAPI interface { GetNetworkParameters(ctx context.Context) access.NetworkParameters GetLatestProtocolStateSnapshot(ctx context.Context) ([]byte, error) + GetNodeVersionInfo(ctx context.Context) (*access.NodeVersionInfo, error) } type API interface { diff --git a/engine/protocol/api_test.go b/engine/protocol/api_test.go index e2b7234eb42..f5e029181ed 100644 --- a/engine/protocol/api_test.go +++ b/engine/protocol/api_test.go @@ -2,9 +2,7 @@ package protocol import ( "context" - "math/rand" "testing" - "time" "github.com/stretchr/testify/suite" @@ -37,7 +35,6 @@ func TestHandler(t *testing.T) { } func (suite *Suite) SetupTest() { - rand.Seed(time.Now().UnixNano()) suite.snapshot = new(protocol.Snapshot) suite.state = new(protocol.State) diff --git a/engine/protocol/handler.go b/engine/protocol/handler.go index a7b96e0c841..ef77ad70e43 100644 --- a/engine/protocol/handler.go +++ b/engine/protocol/handler.go @@ -48,6 +48,25 @@ func (h *Handler) GetNetworkParameters( }, nil } +func (h *Handler) GetNodeVersionInfo( + ctx context.Context, + request *access.GetNodeVersionInfoRequest, +) (*access.GetNodeVersionInfoResponse, error) { + nodeVersionInfo, err := h.api.GetNodeVersionInfo(ctx) + if err != nil { + return nil, err + } + + return &access.GetNodeVersionInfoResponse{ + Info: &entities.NodeVersionInfo{ + Semver: nodeVersionInfo.Semver, + Commit: nodeVersionInfo.Commit, + SporkId: nodeVersionInfo.SporkId[:], + ProtocolVersion: nodeVersionInfo.ProtocolVersion, + }, + }, nil +} + // GetLatestProtocolStateSnapshot returns the latest serializable Snapshot func (h *Handler) GetLatestProtocolStateSnapshot(ctx context.Context, req *access.GetLatestProtocolStateSnapshotRequest) (*access.ProtocolStateSnapshotResponse, error) { snapshot, err := h.api.GetLatestProtocolStateSnapshot(ctx) diff --git a/engine/protocol/mock/api.go b/engine/protocol/mock/api.go index bb45baf8062..6ece771befd 100644 --- a/engine/protocol/mock/api.go +++ b/engine/protocol/mock/api.go @@ -213,6 +213,32 @@ func (_m *API) GetNetworkParameters(ctx context.Context) access.NetworkParameter return r0 } +// GetNodeVersionInfo provides a mock function with given fields: ctx +func (_m *API) GetNodeVersionInfo(ctx context.Context) (*access.NodeVersionInfo, error) { + ret := _m.Called(ctx) + + var r0 *access.NodeVersionInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*access.NodeVersionInfo, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *access.NodeVersionInfo); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*access.NodeVersionInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + type mockConstructorTestingTNewAPI interface { mock.TestingT Cleanup(func()) diff --git a/engine/testutil/mock/nodes.go b/engine/testutil/mock/nodes.go index 7022dbb98b6..191dde0e28b 100644 --- a/engine/testutil/mock/nodes.go +++ b/engine/testutil/mock/nodes.go @@ -134,6 +134,7 @@ func (n CollectionNode) Start(t *testing.T) { go unittest.FailOnIrrecoverableError(t, n.Ctx.Done(), n.Errs) n.IngestionEngine.Start(n.Ctx) n.EpochManagerEngine.Start(n.Ctx) + n.ProviderEngine.Start(n.Ctx) } func (n CollectionNode) Ready() <-chan struct{} { @@ -212,6 +213,7 @@ func (en ExecutionNode) Ready(ctx context.Context) { en.ReceiptsEngine.Start(irctx) en.FollowerCore.Start(irctx) en.FollowerEngine.Start(irctx) + en.SyncEngine.Start(irctx) <-util.AllReady( en.Ledger, diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 7532995fae0..8639e1ee1f7 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/coreos/go-semver/semver" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" blockstore "github.com/ipfs/go-ipfs-blockstore" @@ -17,6 +18,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/atomic" + "github.com/onflow/flow-go/cmd/build" "github.com/onflow/flow-go/consensus" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees" @@ -25,9 +27,11 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/notifications" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/collection/epochmgr" "github.com/onflow/flow-go/engine/collection/epochmgr/factories" collectioningest "github.com/onflow/flow-go/engine/collection/ingest" + mockcollection "github.com/onflow/flow-go/engine/collection/mock" "github.com/onflow/flow-go/engine/collection/pusher" "github.com/onflow/flow-go/engine/common/follower" "github.com/onflow/flow-go/engine/common/provider" @@ -41,6 +45,7 @@ import ( "github.com/onflow/flow-go/engine/execution/computation/committer" "github.com/onflow/flow-go/engine/execution/computation/query" "github.com/onflow/flow-go/engine/execution/ingestion" + "github.com/onflow/flow-go/engine/execution/ingestion/stop" "github.com/onflow/flow-go/engine/execution/ingestion/uploader" executionprovider "github.com/onflow/flow-go/engine/execution/provider" executionState "github.com/onflow/flow-go/engine/execution/state" @@ -64,6 +69,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/chainsync" "github.com/onflow/flow-go/module/chunks" + "github.com/onflow/flow-go/module/compliance" "github.com/onflow/flow-go/module/executiondatasync/execution_data" exedataprovider "github.com/onflow/flow-go/module/executiondatasync/provider" mocktracker "github.com/onflow/flow-go/module/executiondatasync/tracker/mock" @@ -119,7 +125,7 @@ func GenericNodeFromParticipants(t testing.TB, hub *stub.Hub, identity *flow.Ide metrics := metrics.NewNoopCollector() // creates state fixture and bootstrap it. - rootSnapshot := unittest.RootSnapshotFixture(participants) + rootSnapshot := unittest.RootSnapshotFixtureWithChainID(participants, chainID) stateFixture := CompleteStateFixture(t, log, metrics, tracer, rootSnapshot) require.NoError(t, err) @@ -245,6 +251,7 @@ func CompleteStateFixture( s.Setups, s.EpochCommits, s.Statuses, + s.VersionBeacons, rootSnapshot, ) require.NoError(t, err) @@ -273,7 +280,7 @@ func CompleteStateFixture( } // CollectionNode returns a mock collection node. -func CollectionNode(t *testing.T, ctx irrecoverable.SignalerContext, hub *stub.Hub, identity bootstrap.NodeInfo, rootSnapshot protocol.Snapshot) testmock.CollectionNode { +func CollectionNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, rootSnapshot protocol.Snapshot) testmock.CollectionNode { node := GenericNode(t, hub, identity.Identity(), rootSnapshot) privKeys, err := identity.PrivateKeys() @@ -309,8 +316,6 @@ func CollectionNode(t *testing.T, ctx irrecoverable.SignalerContext, hub *stub.H selector, retrieve) require.NoError(t, err) - // TODO: move this start logic to a more generalized test utility (we need all engines to be startable). - providerEngine.Start(ctx) pusherEngine, err := pusher.New(node.Log, node.Net, node.State, node.Metrics, node.Metrics, node.Me, collections, transactions) require.NoError(t, err) @@ -324,6 +329,7 @@ func CollectionNode(t *testing.T, ctx irrecoverable.SignalerContext, hub *stub.H builderFactory, err := factories.NewBuilderFactory( node.PublicDB, + node.State, node.Headers, node.Tracer, node.Metrics, @@ -339,6 +345,7 @@ func CollectionNode(t *testing.T, ctx irrecoverable.SignalerContext, hub *stub.H node.Metrics, node.Metrics, node.Metrics, node.State, transactions, + compliance.DefaultConfig(), ) require.NoError(t, err) @@ -390,6 +397,8 @@ func CollectionNode(t *testing.T, ctx irrecoverable.SignalerContext, hub *stub.H rootQCVoter := new(mockmodule.ClusterRootQCVoter) rootQCVoter.On("Vote", mock.Anything, mock.Anything).Return(nil) + engineEventsDistributor := mockcollection.NewEngineEvents(t) + engineEventsDistributor.On("ActiveClustersChanged", mock.AnythingOfType("flow.ChainIDList")).Maybe() heights := gadgets.NewHeights() node.ProtocolEvents.AddConsumer(heights) @@ -401,9 +410,9 @@ func CollectionNode(t *testing.T, ctx irrecoverable.SignalerContext, hub *stub.H rootQCVoter, factory, heights, + engineEventsDistributor, ) require.NoError(t, err) - node.ProtocolEvents.AddConsumer(epochManager) return testmock.CollectionNode{ @@ -546,6 +555,8 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit results := storage.NewExecutionResults(node.Metrics, node.PublicDB) receipts := storage.NewExecutionReceipts(node.Metrics, node.PublicDB, results, storage.DefaultCacheSize) myReceipts := storage.NewMyExecutionReceipts(node.Metrics, node.PublicDB, receipts) + versionBeacons := storage.NewVersionBeacons(node.PublicDB) + checkAuthorizedAtBlock := func(blockID flow.Identifier) (bool, error) { return protocol.IsNodeAuthorizedAt(node.State.AtBlockID(blockID), node.Me.NodeID()) } @@ -595,7 +606,13 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit fvm.WithInitialTokenSupply(unittest.GenesisTokenSupply)) require.NoError(t, err) - err = bootstrapper.BootstrapExecutionDatabase(node.PublicDB, commit, genesisHead) + rootResult, rootSeal, err := protoState.Sealed().SealedResult() + require.NoError(t, err) + + require.Equal(t, fmt.Sprintf("%x", rootSeal.FinalState), fmt.Sprintf("%x", commit)) + require.Equal(t, rootSeal.ResultID, rootResult.ID()) + + err = bootstrapper.BootstrapExecutionDatabase(node.PublicDB, rootSeal) require.NoError(t, err) execState := executionState.NewExecutionState( @@ -657,6 +674,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit computation.ComputationConfig{ QueryConfig: query.NewDefaultConfig(), DerivedDataCacheSize: derived.DefaultDerivedDataCacheSize, + MaxConcurrency: 1, }, ) require.NoError(t, err) @@ -664,21 +682,42 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit syncCore, err := chainsync.New(node.Log, chainsync.DefaultConfig(), metrics.NewChainSyncCollector(genesisHead.ChainID), genesisHead.ChainID) require.NoError(t, err) - finalizationDistributor := pubsub.NewFinalizationDistributor() - - latestExecutedHeight, _, err := execState.GetHighestExecutedBlockID(context.TODO()) + followerDistributor := pubsub.NewFollowerDistributor() require.NoError(t, err) // disabled by default uploader := uploader.NewManager(node.Tracer) + _, err = build.Semver() + require.ErrorIs(t, err, build.UndefinedVersionError) + ver := semver.New("0.0.0") + + latestFinalizedBlock, err := node.State.Final().Head() + require.NoError(t, err) + + unit := engine.NewUnit() + stopControl := stop.NewStopControl( + unit, + time.Second, + node.Log, + execState, + node.Headers, + versionBeacons, + ver, + latestFinalizedBlock, + false, + true, + ) + rootHead, rootQC := getRoot(t, &node) ingestionEngine, err := ingestion.New( + unit, node.Log, node.Net, node.Me, requestEngine, node.State, + node.Headers, node.Blocks, collectionsStorage, eventsStorage, @@ -693,26 +732,24 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit checkAuthorizedAtBlock, nil, uploader, - ingestion.NewStopControl(node.Log.With().Str("compontent", "stop_control").Logger(), false, latestExecutedHeight), + stopControl, + false, ) require.NoError(t, err) requestEngine.WithHandle(ingestionEngine.OnCollection) node.ProtocolEvents.AddConsumer(ingestionEngine) - followerCore, finalizer := createFollowerCore(t, &node, followerState, finalizationDistributor, rootHead, rootQC) + followerCore, finalizer := createFollowerCore(t, &node, followerState, followerDistributor, rootHead, rootQC) // mock out hotstuff validator validator := new(mockhotstuff.Validator) validator.On("ValidateProposal", mock.Anything).Return(nil) - finalizedHeader, err := synchronization.NewFinalizedHeaderCache(node.Log, node.State, finalizationDistributor) - require.NoError(t, err) - core, err := follower.NewComplianceCore( node.Log, node.Metrics, node.Metrics, - finalizationDistributor, + followerDistributor, followerState, followerCore, validator, @@ -720,14 +757,18 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit node.Tracer, ) require.NoError(t, err) + + finalizedHeader, err := protoState.Final().Head() + require.NoError(t, err) followerEng, err := follower.NewComplianceLayer( node.Log, node.Net, node.Me, node.Metrics, node.Headers, - finalizedHeader.Get(), + finalizedHeader, core, + compliance.DefaultConfig(), ) require.NoError(t, err) @@ -738,10 +779,10 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit node.Metrics, node.Net, node.Me, + node.State, node.Blocks, followerEng, syncCore, - finalizedHeader, id.NewIdentityFilterIdentifierProvider( filter.And( filter.HasRole(flow.RoleConsensus), @@ -752,6 +793,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit synchronization.WithPollInterval(time.Duration(0)), ) require.NoError(t, err) + followerDistributor.AddFinalizationConsumer(syncEngine) return testmock.ExecutionNode{ GenericNode: node, @@ -776,7 +818,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit } func getRoot(t *testing.T, node *testmock.GenericNode) (*flow.Header, *flow.QuorumCertificate) { - rootHead, err := node.State.Params().Root() + rootHead, err := node.State.Params().FinalizedRoot() require.NoError(t, err) signers, err := node.State.AtHeight(0).Identities(filter.HasRole(flow.RoleConsensus)) @@ -849,23 +891,14 @@ func (s *RoundRobinLeaderSelection) DKG(_ uint64) (hotstuff.DKG, error) { return nil, fmt.Errorf("error") } -func createFollowerCore(t *testing.T, node *testmock.GenericNode, followerState *badgerstate.FollowerState, notifier hotstuff.FinalizationConsumer, - rootHead *flow.Header, rootQC *flow.QuorumCertificate) (module.HotStuffFollower, *confinalizer.Finalizer) { - - identities, err := node.State.AtHeight(0).Identities(filter.HasRole(flow.RoleConsensus)) - require.NoError(t, err) - - committee := &RoundRobinLeaderSelection{ - identities: identities, - me: node.Me.NodeID(), - } - - // mock finalization updater - verifier := &mockhotstuff.Verifier{} - verifier.On("VerifyVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - verifier.On("VerifyTC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - +func createFollowerCore( + t *testing.T, + node *testmock.GenericNode, + followerState *badgerstate.FollowerState, + notifier hotstuff.FollowerConsumer, + rootHead *flow.Header, + rootQC *flow.QuorumCertificate, +) (module.HotStuffFollower, *confinalizer.Finalizer) { finalizer := confinalizer.NewFinalizer(node.PublicDB, node.Headers, followerState, trace.NewNoopTracer()) pending := make([]*flow.Header, 0) @@ -873,10 +906,9 @@ func createFollowerCore(t *testing.T, node *testmock.GenericNode, followerState // creates a consensus follower with noop consumer as the notifier followerCore, err := consensus.NewFollower( node.Log, - committee, + node.Metrics, node.Headers, finalizer, - verifier, notifier, rootHead, rootQC, diff --git a/engine/verification/Readme.md b/engine/verification/Readme.md new file mode 100644 index 00000000000..ff527a432b0 --- /dev/null +++ b/engine/verification/Readme.md @@ -0,0 +1,170 @@ +# Verification Node +The Verification Node in the Flow blockchain network is a critical component responsible for +verifying `ExecutionResult`s and generating `ResultApproval`s. +Its primary role is to ensure the integrity and validity of block execution by performing verification processes. +In a nutshell, the Verification Node is responsible for the following: +1. Following the chain for new finalized blocks (`Follower` engine). +2. Processing the execution results in the finalized blocks and determining assigned chunks to the node (`Assigner` engine). +3. Requesting chunk data pack from Execution Nodes for the assigned chunks (`Fetcher` and `Requester` engines). +4. Verifying the assigned chunks and emitting `ResultApproval`s for the verified chunks to Consensus Nodes (`Verifier` engine). +![architecture.png](architecture.png) + + +## Block Consumer ([consumer.go](verification%2Fassigner%2Fblockconsumer%2Fconsumer.go)) +The `blockconsumer` package efficiently manages the processing of finalized blocks in Verification Node of Flow blockchain. +Specifically, it listens for notifications from the `Follower` engine regarding finalized blocks, and systematically +queues these blocks for processing. The package employs parallel workers, each an instance of the `Assigner` engine, +to fetch and process blocks from the queue. The `BlockConsumer` diligently coordinates this process by only assigning +a new block to a worker once it has completed processing its current block and signaled its availability. +This ensures that the processing is not only methodical but also resilient to any node crashes. +In case of a crash, the `BlockConsumer` resumes from where it left off by reading the processed block index from storage, reassigning blocks from the queue to workers, +thereby guaranteeing no loss of data. + +## Assigner Engine +The `Assigner` [engine](verification%2Fassigner%2Fengine.go) is an integral part of the verification process in Flow, +focusing on processing the execution results in the finalized blocks, performing chunk assignments on the results, and +queuing the assigned chunks for further processing. The Assigner engine is a worker of the `BlockConsumer` engine, +which assigns finalized blocks to the Assigner engine for processing. +This engine reads execution receipts included in each finalized block, +determines which chunks are assigned to the node for verification, +and stores the assigned chunks into the chunks queue for further processing (by the `Fetcher` engine). + +The core behavior of the Assigner engine is implemented in the `ProcessFinalizedBlock` function. +This function initiates the process of execution receipt indexing, chunk assignment, and processing the assigned chunks. +For every receipt in the block, the engine determines chunk assignments using the verifiable chunk assignment algorithm of Flow. +Each assigned chunk is then processed by the `processChunk` method. This method is responsible for storing a chunk locator in the chunks queue, +which is a crucial step for further processing of the chunks by the fetcher engine. +Deduplication of chunk locators is handled by the chunks queue. +The Assigner engine provides robustness by handling the situation where a node is not authorized at a specific block ID. +It verifies the role of the result executor, checks if the node has been ejected, and assesses the node's staked weight before granting authorization. +Lastly, once the Assigner engine has completed processing the receipts in a block, it sends a notification to the block consumer. This is inline with +Assigner engine as a worker of the block consumer informing the consumer that it is ready to process the next block. +This ensures a smooth and efficient flow of data in the system, promoting consistency across different parts of the Flow architecture. + +### Chunk Locator +A chunk locator in the Flow blockchain is an internal structure of the Verification Nodes that points to a specific chunk +within a specific execution result of a block. It's an important part of the verification process in the Flow network, +allowing verification nodes to efficiently identify, retrieve, and verify individual chunks of computation. + +```go +type ChunkLocator struct { + ResultID flow.Identifier // The identifier of the ExecutionResult + Index uint64 // Index of the chunk +} +``` +- `ResultID`: This is the identifier of the execution result that the chunk is a part of. The execution result contains a list of chunks which each represent a portion of the computation carried out by execution nodes. Each execution result is linked to a specific block in the blockchain. +- `Index`: This is the index of the chunk within the execution result's list of chunks. It's an easy way to refer to a specific chunk within a specific execution result. + +**Note-1**: The `ChunkLocator` doesn't contain the chunk itself but points to where the chunk can be found. In the context of the `Assigner` engine, the `ChunkLocator` is stored in a queue after chunk assignment is done, so the `Fetcher` engine can later retrieve the chunk for verification. +**Note-2**: The `ChunkLocator` is never meant to be sent over the networking layer to another Flow node. It's an internal structure of the verification nodes, and it's only used for internal communication between the `Assigner` and `Fetcher` engines. + + +## ChunkConsumer +The `ChunkConsumer` ([consumer](verification%2Ffetcher%2Fchunkconsumer%2Fconsumer.go)) package orchestrates the processing of chunks in the Verification Node of the Flow blockchain. +Specifically, it keeps tabs on chunks that are assigned for processing by the `Assigner` engine and systematically enqueues these chunks for further handling. +To expedite the processing, the package deploys parallel workers, with each worker being an instance of the `Fetcher` engine, which retrieves and processes the chunks from the queue. +The `ChunkConsumer` administers this process by ensuring that a new chunk is assigned to a worker only after it has finalized processing its current chunk and signaled that it is ready for more. +This systematic approach guarantees not only efficiency but also robustness against any node failures. In an event where a node crashes, +the `ChunkConsumer` picks up right where it left, redistributing chunks from the queue to the workers, ensuring that there is no loss of data or progress. + +## Fetcher Engine - The Journey of a `ChunkLocator` to a `VerifiableChunkData` +The Fetcher [engine.go](fetcher%2Fengine.go) of the Verification Nodes focuses on the lifecycle of a `ChunkLocator` as it transitions into a `VerifiableChunkData`. + +### `VerifiableChunkData` +`VerifiableChunkData` refers to a data structure that encapsulates all the necessary components and resources required to +verify a chunk within the Flow blockchain network. It represents a chunk that has undergone processing and is ready for verification. + +The `VerifiableChunkData` object contains the following key elements: +```go +type VerifiableChunkData struct { + IsSystemChunk bool // indicates whether this is a system chunk + Chunk *flow.Chunk // the chunk to be verified + Header *flow.Header // BlockHeader that contains this chunk + Result *flow.ExecutionResult // execution result of this block + ChunkDataPack *flow.ChunkDataPack // chunk data package needed to verify this chunk + EndState flow.StateCommitment // state commitment at the end of this chunk + TransactionOffset uint32 // index of the first transaction in a chunk within a block +} +``` +1. `IsSystemChunk`: A boolean value that indicates whether the chunk is a system chunk. System chunk is a specific chunk typically representing the last chunk within an execution result. +2. `Chunk`: The actual chunk that needs to be verified. It contains the relevant data and instructions related to the execution of transactions within the blockchain. +3. `Header`: The `BlockHeader` associated with the chunk. It provides important contextual information about the block that the chunk belongs to. +4. `Result`: The `ExecutionResult` object that corresponds to the execution of the block containing the chunk. It contains information about the execution status, including any errors or exceptions encountered during the execution process. +5. `ChunkDataPack`: The `ChunkDataPack`, which is a package containing additional data and resources specific to the chunk being verified. It provides supplementary information required for the verification process. +6. `EndState`: The state commitment at the end of the chunk. It represents the final state of the blockchain after executing all the transactions within the chunk. +7. `TransactionOffset`: An index indicating the position of the first transaction within the chunk in relation to the entire block. This offset helps in locating and tracking individual transactions within the blockchain. +By combining these elements, the VerifiableChunkData object forms a comprehensive representation of a chunk ready for verification. It serves as an input to the `Verifier` engine, which utilizes this data to perform the necessary checks and validations to ensure the integrity and correctness of the chunk within the Flow blockchain network. + +### The Journey of a `ChunkLocator` to a `VerifiableChunkData` +Upon receiving the `ChunkLocator`, the `Fetcher` engine’s `validateAuthorizedExecutionNodeAtBlockID` function is responsible +for validating the authenticity of the sender. It evaluates whether the sender is an authorized execution node for the respective block. +The function cross-references the sender’s credentials against the state snapshot of the specific block. +In the case of unauthorized or invalid credentials, an error is logged, and the `ChunkLocator` is rejected. +For authorized credentials, the processing continues. + +Once authenticated, the `ChunkLocator` is utilized to retrieve the associated Chunk Data Pack. +The `requestChunkDataPack` function takes the Chunk Locator and generates a `ChunkDataPackRequest`. +During this stage, the function segregates execution nodes into two categories - those which agree with the execution result (`agrees`) and those which do not (`disagrees`). +This information is encapsulated within the `ChunkDataPackRequest` and is forwarded to the `Requester` Engine. +The `Requester` Engine handles the retrieval of the `ChunkDataPack` from the network of execution nodes. + +After the Chunk Data Pack is successfully retrieved by the `Requester` Engine, +the next phase involves structuring this data for verification and constructing a `VerifiableChunkData`. +It’s imperative that this construction is performed with utmost accuracy to ensure that the data is in a state that can be properly verified. + +The final step in the lifecycle is forwarding the `VerifiableChunkData` to the `Verifier` Engine. The `Verifier` Engine is tasked with the critical function +of thoroughly analyzing and verifying the data. Depending on the outcome of this verification process, +the chunk may either pass verification successfully or be rejected due to discrepancies. + +### Handling Sealed Chunks +In parallel, the `Fetcher` engine remains vigilant regarding the sealed status of chunks. +The `NotifyChunkDataPackSealed` function monitors the sealing status. +If the Consensus Nodes seal a chunk, this function ensures that the `Fetcher` Engine acknowledges this update and discards the respective +`ChunkDataPack` from its processing pipeline as it is now sealed (i.e., has been verified by an acceptable quota of Verification Nodes). + +## Requester Engine - Retrieving the `ChunkDataPack` +The `Requester` [engine](requester%2Frequester.go) is responsible for handling the request and retrieval of chunk data packs in the Flow blockchain network. +It acts as an intermediary between the `Fetcher` engine and the Execution Nodes, facilitating the communication and coordination required +to obtain the necessary `ChunkDataPack` for verification. + +The `Requester` engine receives `ChunkDataPackRequest`s from the `Fetcher`. +These requests contain information such as the chunk ID, block height, agree and disagree executors, and other relevant details. +Upon receiving a `ChunkDataPackRequest`, the `Requester` engine adds it to the pending requests cache for tracking and further processing. +The Requester engine periodically checks the pending chunk data pack requests and dispatches them to the Execution Nodes for retrieval. +It ensures that only qualified requests are dispatched based on certain criteria, such as the chunk ID and request history. +The dispatching process involves creating a `ChunkDataRequest` message and publishing it to the network. +The request is sent to a selected number of Execution Nodes, determined by the `requestTargets` parameter. + +When an Execution Node receives a `ChunkDataPackRequest`, it processes the request and generates a `ChunkDataResponse` +message containing the requested chunk data pack. The execution node sends this response back to the`Requester` engine. +The `Requester` engine receives the chunk data pack response, verifies its integrity, and passes it to the registered `ChunkDataPackHandler`, +i.e., the `Fetcher` engine. + +### Retry and Backoff Mechanism +In case a `ChunkDataPackRequest` does not receive a response within a certain period, the `Requester` engine retries the request to ensure data retrieval. +It implements an exponential backoff mechanism for retrying failed requests. +The retry interval, backoff multiplier, and backoff intervals can be customized using the respective configuration parameters. + +### Handling Sealed Blocks +If a `ChunkDataPackRequest` pertains to a block that has already been sealed, the `Requester` engine recognizes this and +removes the corresponding request from the pending requests cache. +It notifies the `ChunkDataPackHandler` (i.e., the `Fetcher` engine) about the sealing of the block to ensure proper handling. + +### Parallel Chunk Data Pack Retrieval +The `Requester` processes a number of chunk data pack requests in parallel, +dispatching them to execution nodes and handling the received responses. +However, it is important to note that if a chunk data pack request does not receive a response from the execution nodes, +the `Requester` engine can become stuck in processing, waiting for the missing chunk data pack. +To mitigate this, the engine implements a retry and backoff mechanism, ensuring that requests are retried and backed off if necessary. +This mechanism helps to prevent prolonged waiting and allows the engine to continue processing other requests while waiting for the missing chunk data pack response. + +## Verifier Engine - Verifying Chunks +The `Verifier` [engine](verifier%2Fengine.go) is responsible for verifying chunks, generating `ResultApproval`s, and maintaining a cache of `ResultApproval`s. +It receives verifiable chunks along with the necessary data for verification, verifies the chunks by constructing a partial trie, +executing transactions, and checking the final state commitment and other chunk metadata. +If the verification is successful, it generates a `ResultApproval` and broadcasts it to the consensus nodes. + +The `Verifier` Engine offers the following key features: +1. **Verification of Chunks**: The engine receives verifiable chunks, which include the chunk to be verified, the associated header, execution result, and chunk data pack. It performs the verification process, which involves constructing a partial trie, executing transactions, and checking the final state commitment. The verification process ensures the integrity and validity of the chunk. +2. **Generation of Result Approvals**: If the verification process is successful, the engine generates a result approval for the verified chunk. The result approval includes the block ID, execution result ID, chunk index, attestation, approver ID, attestation signature, and SPoCK (Secure Proof of Confidential Knowledge) signature. The result approval provides a cryptographic proof of the chunk's validity and is used to seal the block. +3. **Cache of Result Approvals**: The engine maintains a cache of result approvals for efficient retrieval and lookup. The result approvals are stored in a storage module, allowing quick access to the approvals associated with specific chunks and execution results. diff --git a/engine/verification/architecture.png b/engine/verification/architecture.png new file mode 100644 index 00000000000..a1a16dec61b Binary files /dev/null and b/engine/verification/architecture.png differ diff --git a/engine/verification/assigner/blockconsumer/consumer.go b/engine/verification/assigner/blockconsumer/consumer.go index e0913a45fa6..982fe418688 100644 --- a/engine/verification/assigner/blockconsumer/consumer.go +++ b/engine/verification/assigner/blockconsumer/consumer.go @@ -98,12 +98,6 @@ func (c *BlockConsumer) OnFinalizedBlock(*model.Block) { c.unit.Launch(c.consumer.Check) } -// OnBlockIncorporated is to implement FinalizationConsumer -func (c *BlockConsumer) OnBlockIncorporated(*model.Block) {} - -// OnDoubleProposeDetected is to implement FinalizationConsumer -func (c *BlockConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} - func (c *BlockConsumer) Ready() <-chan struct{} { err := c.consumer.Start(c.defaultIndex) if err != nil { diff --git a/engine/verification/assigner/blockconsumer/consumer_test.go b/engine/verification/assigner/blockconsumer/consumer_test.go index 2a2bff2a343..67ea6773194 100644 --- a/engine/verification/assigner/blockconsumer/consumer_test.go +++ b/engine/verification/assigner/blockconsumer/consumer_test.go @@ -146,10 +146,11 @@ func withConsumer( // blocks (i.e., containing guarantees), and Cs are container blocks for their preceding reference block, // Container blocks only contain receipts of their preceding reference blocks. But they do not // hold any guarantees. - root, err := s.State.Params().Root() + root, err := s.State.Params().FinalizedRoot() require.NoError(t, err) clusterCommittee := participants.Filter(filter.HasRole(flow.RoleCollection)) - results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, vertestutils.WithClusterCommittee(clusterCommittee)) + sources := unittest.RandomSourcesFixture(110) + results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, sources, vertestutils.WithClusterCommittee(clusterCommittee)) blocks := vertestutils.ExtendStateWithFinalizedBlocks(t, results, s.State) // makes sure that we generated a block chain of requested length. require.Len(t, blocks, blockCount) diff --git a/engine/verification/fetcher/engine.go b/engine/verification/fetcher/engine.go index 23d02c02474..fd53417b720 100644 --- a/engine/verification/fetcher/engine.go +++ b/engine/verification/fetcher/engine.go @@ -526,8 +526,8 @@ func (e *Engine) pushToVerifier(chunk *flow.Chunk, if err != nil { return fmt.Errorf("could not get block: %w", err) } - - vchunk, err := e.makeVerifiableChunkData(chunk, header, result, chunkDataPack) + snapshot := e.state.AtBlockID(header.ID()) + vchunk, err := e.makeVerifiableChunkData(chunk, header, snapshot, result, chunkDataPack) if err != nil { return fmt.Errorf("could not verify chunk: %w", err) } @@ -545,6 +545,7 @@ func (e *Engine) pushToVerifier(chunk *flow.Chunk, // chunk data to verify it. func (e *Engine) makeVerifiableChunkData(chunk *flow.Chunk, header *flow.Header, + snapshot protocol.Snapshot, result *flow.ExecutionResult, chunkDataPack *flow.ChunkDataPack, ) (*verification.VerifiableChunkData, error) { @@ -566,6 +567,7 @@ func (e *Engine) makeVerifiableChunkData(chunk *flow.Chunk, IsSystemChunk: isSystemChunk, Chunk: chunk, Header: header, + Snapshot: snapshot, Result: result, ChunkDataPack: chunkDataPack, EndState: endState, diff --git a/engine/verification/requester/requester.go b/engine/verification/requester/requester.go index 10f91780c72..2285da61025 100644 --- a/engine/verification/requester/requester.go +++ b/engine/verification/requester/requester.go @@ -331,8 +331,11 @@ func (e *Engine) requestChunkDataPack(request *verification.ChunkDataPackRequest } // publishes the chunk data request to the network - targetIDs := request.SampleTargets(int(e.requestTargets)) - err := e.con.Publish(req, targetIDs...) + targetIDs, err := request.SampleTargets(int(e.requestTargets)) + if err != nil { + return fmt.Errorf("target sampling failed: %w", err) + } + err = e.con.Publish(req, targetIDs...) if err != nil { return fmt.Errorf("could not publish chunk data pack request for chunk (id=%s): %w", request.ChunkID, err) } diff --git a/engine/verification/utils/unittest/fixture.go b/engine/verification/utils/unittest/fixture.go index 1931d06347d..57c9916e62d 100644 --- a/engine/verification/utils/unittest/fixture.go +++ b/engine/verification/utils/unittest/fixture.go @@ -14,7 +14,7 @@ import ( "github.com/onflow/flow-go/engine/execution/computation/committer" "github.com/onflow/flow-go/engine/execution/computation/computer" - "github.com/onflow/flow-go/engine/execution/state" + exstate "github.com/onflow/flow-go/engine/execution/state" "github.com/onflow/flow-go/engine/execution/state/bootstrap" "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/fvm" @@ -39,6 +39,12 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) +const ( + // TODO: enable parallel execution once cadence type equivalence check issue + // is resolved. + testMaxConcurrency = 1 +) + // ExecutionReceiptData is a test helper struct that represents all data required // to verify the result of an execution receipt. type ExecutionReceiptData struct { @@ -183,8 +189,13 @@ func WithClusterCommittee(clusterCommittee flow.IdentityList) CompleteExecutionR // ExecutionResultFixture is a test helper that returns an execution result for the reference block header as well as the execution receipt data // for that result. -func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refBlkHeader *flow.Header, clusterCommittee flow.IdentityList) (*flow.ExecutionResult, - *ExecutionReceiptData) { +func ExecutionResultFixture(t *testing.T, + chunkCount int, + chain flow.Chain, + refBlkHeader *flow.Header, + clusterCommittee flow.IdentityList, + source []byte, +) (*flow.ExecutionResult, *ExecutionReceiptData) { // setups up the first collection of block consists of three transactions tx1 := testutil.DeployCounterContractTransaction(chain.ServiceAddress(), chain) @@ -256,7 +267,7 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB ) // create state.View - snapshot := state.NewLedgerStorageSnapshot( + snapshot := exstate.NewLedgerStorageSnapshot( led, startStateCommitment) committer := committer.NewLedgerViewCommitter(led, trace.NewNoopTracer()) @@ -288,7 +299,9 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB committer, me, prov, - nil) + nil, + testutil.ProtocolStateWithSourceFixture(source), + testMaxConcurrency) require.NoError(t, err) completeColls := make(map[flow.Identifier]*entity.CompleteCollection) @@ -334,14 +347,14 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB unittest.IdentifierFixture(), executableBlock, snapshot, - derived.NewEmptyDerivedBlockData()) + derived.NewEmptyDerivedBlockData(0)) require.NoError(t, err) - for _, snapshot := range computationResult.StateSnapshots { + for _, snapshot := range computationResult.AllExecutionSnapshots() { spockSecrets = append(spockSecrets, snapshot.SpockSecret) } - chunkDataPacks = computationResult.ChunkDataPacks + chunkDataPacks = computationResult.AllChunkDataPacks() result = &computationResult.ExecutionResult }) @@ -360,7 +373,12 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB // For sake of simplicity and test, container blocks (i.e., C) do not contain any guarantee. // // It returns a slice of complete execution receipt fixtures that contains a container block as well as all data to verify its contained receipts. -func CompleteExecutionReceiptChainFixture(t *testing.T, root *flow.Header, count int, opts ...CompleteExecutionReceiptBuilderOpt) []*CompleteExecutionReceipt { +func CompleteExecutionReceiptChainFixture(t *testing.T, + root *flow.Header, + count int, + sources [][]byte, + opts ...CompleteExecutionReceiptBuilderOpt, +) []*CompleteExecutionReceipt { completeERs := make([]*CompleteExecutionReceipt, 0, count) parent := root @@ -382,11 +400,14 @@ func CompleteExecutionReceiptChainFixture(t *testing.T, root *flow.Header, count require.GreaterOrEqual(t, len(builder.executorIDs), builder.executorCount, "number of executors in the tests should be greater than or equal to the number of receipts per block") + var sourcesIndex = 0 for i := 0; i < count; i++ { // Generates two blocks as parent <- R <- C where R is a reference block containing guarantees, // and C is a container block containing execution receipt for R. - receipts, allData, head := ExecutionReceiptsFromParentBlockFixture(t, parent, builder) - containerBlock := ContainerBlockFixture(head, receipts) + receipts, allData, head := ExecutionReceiptsFromParentBlockFixture(t, parent, builder, sources[sourcesIndex:]) + sourcesIndex += builder.resultsCount + containerBlock := ContainerBlockFixture(head, receipts, sources[sourcesIndex]) + sourcesIndex++ completeERs = append(completeERs, &CompleteExecutionReceipt{ ContainerBlock: containerBlock, Receipts: receipts, @@ -404,7 +425,10 @@ func CompleteExecutionReceiptChainFixture(t *testing.T, root *flow.Header, count // result (i.e., for the next result). // // Each result may appear in more than one receipt depending on the builder parameters. -func ExecutionReceiptsFromParentBlockFixture(t *testing.T, parent *flow.Header, builder *CompleteExecutionReceiptBuilder) ( +func ExecutionReceiptsFromParentBlockFixture(t *testing.T, + parent *flow.Header, + builder *CompleteExecutionReceiptBuilder, + sources [][]byte) ( []*flow.ExecutionReceipt, []*ExecutionReceiptData, *flow.Header) { @@ -412,7 +436,7 @@ func ExecutionReceiptsFromParentBlockFixture(t *testing.T, parent *flow.Header, allReceipts := make([]*flow.ExecutionReceipt, 0, builder.resultsCount*builder.executorCount) for i := 0; i < builder.resultsCount; i++ { - result, data := ExecutionResultFromParentBlockFixture(t, parent, builder) + result, data := ExecutionResultFromParentBlockFixture(t, parent, builder, sources[i:]) // makes several copies of the same result for cp := 0; cp < builder.executorCount; cp++ { @@ -430,16 +454,22 @@ func ExecutionReceiptsFromParentBlockFixture(t *testing.T, parent *flow.Header, } // ExecutionResultFromParentBlockFixture is a test helper that creates a child (reference) block from the parent, as well as an execution for it. -func ExecutionResultFromParentBlockFixture(t *testing.T, parent *flow.Header, builder *CompleteExecutionReceiptBuilder) (*flow.ExecutionResult, - *ExecutionReceiptData) { - refBlkHeader := unittest.BlockHeaderWithParentFixture(parent) - return ExecutionResultFixture(t, builder.chunksCount, builder.chain, refBlkHeader, builder.clusterCommittee) +func ExecutionResultFromParentBlockFixture(t *testing.T, + parent *flow.Header, + builder *CompleteExecutionReceiptBuilder, + sources [][]byte, +) (*flow.ExecutionResult, *ExecutionReceiptData) { + // create the block header including a QC with source a index `i` + refBlkHeader := unittest.BlockHeaderWithParentWithSoRFixture(parent, sources[0]) + // execute the block with the source a index `i+1` (which will be included later in the child block) + return ExecutionResultFixture(t, builder.chunksCount, builder.chain, refBlkHeader, builder.clusterCommittee, sources[1]) } // ContainerBlockFixture builds and returns a block that contains input execution receipts. -func ContainerBlockFixture(parent *flow.Header, receipts []*flow.ExecutionReceipt) *flow.Block { +func ContainerBlockFixture(parent *flow.Header, receipts []*flow.ExecutionReceipt, source []byte) *flow.Block { // container block is the block that contains the execution receipt of reference block containerBlock := unittest.BlockWithParentFixture(parent) + containerBlock.Header.ParentVoterSigData = unittest.QCSigDataWithSoRFixture(source) containerBlock.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipts...))) return containerBlock diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index 7c6e6eec323..62f26cd7f70 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -494,7 +494,11 @@ func withConsumers(t *testing.T, builder.clusterCommittee = participants.Filter(filter.HasRole(flow.RoleCollection)) }) - completeERs := CompleteExecutionReceiptChainFixture(t, root, blockCount, ops...) + // random sources for all blocks: + // - root block (block[0]) is executed with sources[0] (included in QC of child block[1]) + // - block[i] is executed with sources[i] (included in QC of child block[i+1]) + sources := unittest.RandomSourcesFixture(30) + completeERs := CompleteExecutionReceiptChainFixture(t, root, blockCount, sources, ops...) blocks := ExtendStateWithFinalizedBlocks(t, completeERs, s.State) // chunk assignment @@ -591,10 +595,10 @@ func withConsumers(t *testing.T, } // verifies memory resources are cleaned up all over pipeline - assert.True(t, verNode.BlockConsumer.Size() == 0) - assert.True(t, verNode.ChunkConsumer.Size() == 0) - assert.True(t, verNode.ChunkStatuses.Size() == 0) - assert.True(t, verNode.ChunkRequests.Size() == 0) + assert.Zero(t, verNode.BlockConsumer.Size()) + assert.Zero(t, verNode.ChunkConsumer.Size()) + assert.Zero(t, verNode.ChunkStatuses.Size()) + assert.Zero(t, verNode.ChunkRequests.Size()) } // bootstrapSystem is a test helper that bootstraps a flow system with node of each main roles (except execution nodes that are two). diff --git a/engine/verification/verifier/engine.go b/engine/verification/verifier/engine.go index 8b412dc2f66..f870e888340 100644 --- a/engine/verification/verifier/engine.go +++ b/engine/verification/verifier/engine.go @@ -200,19 +200,31 @@ func (e *Engine) verify(ctx context.Context, originID flow.Identifier, if chFault != nil { switch chFault.(type) { case *chmodels.CFMissingRegisterTouch: - e.log.Warn().Msg(chFault.String()) + e.log.Warn(). + Str("chunk_fault_type", "missing_register_touch"). + Str("chunk_fault", chFault.String()). + Msg("chunk fault found, could not verify chunk") // still create approvals for this case case *chmodels.CFNonMatchingFinalState: // TODO raise challenge - e.log.Warn().Msg(chFault.String()) + e.log.Warn(). + Str("chunk_fault_type", "final_state_mismatch"). + Str("chunk_fault", chFault.String()). + Msg("chunk fault found, could not verify chunk") return nil case *chmodels.CFInvalidVerifiableChunk: // TODO raise challenge - e.log.Error().Msg(chFault.String()) + e.log.Error(). + Str("chunk_fault_type", "invalid_verifiable_chunk"). + Str("chunk_fault", chFault.String()). + Msg("chunk fault found, could not verify chunk") return nil case *chmodels.CFInvalidEventsCollection: // TODO raise challenge - e.log.Error().Msg(chFault.String()) + e.log.Error(). + Str("chunk_fault_type", "invalid_event_collection"). + Str("chunk_fault", chFault.String()). + Msg("chunk fault found, could not verify chunk") return nil default: return engine.NewInvalidInputErrorf("unknown type of chunk fault is received (type: %T) : %v", diff --git a/follower/consensus_follower.go b/follower/consensus_follower.go index 97dd227480e..56863bcf530 100644 --- a/follower/consensus_follower.go +++ b/follower/consensus_follower.go @@ -203,7 +203,7 @@ func NewConsensusFollower( cf := &ConsensusFollowerImpl{logger: anb.Logger} anb.BaseConfig.NodeRole = "consensus_follower" - anb.FinalizationDistributor.AddOnBlockFinalizedConsumer(cf.onBlockFinalized) + anb.FollowerDistributor.AddOnBlockFinalizedConsumer(cf.onBlockFinalized) cf.NodeConfig = anb.NodeConfig cf.Component, err = anb.Build() diff --git a/follower/follower_builder.go b/follower/follower_builder.go index dad5247c820..27e69ce039c 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -14,6 +14,7 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/cmd" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/consensus" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees" @@ -31,30 +32,30 @@ import ( "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/module" synchronization "github.com/onflow/flow-go/module/chainsync" - "github.com/onflow/flow-go/module/compliance" finalizer "github.com/onflow/flow-go/module/finalizer/consensus" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/local" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/upstream" "github.com/onflow/flow-go/network" + alspmgr "github.com/onflow/flow-go/network/alsp/manager" netcache "github.com/onflow/flow-go/network/cache" "github.com/onflow/flow-go/network/channels" cborcodec "github.com/onflow/flow-go/network/codec/cbor" "github.com/onflow/flow-go/network/converter" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/cache" + "github.com/onflow/flow-go/network/p2p/conduit" p2pdht "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/keyutils" "github.com/onflow/flow-go/network/p2p/middleware" "github.com/onflow/flow-go/network/p2p/p2pbuilder" - "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/validator" "github.com/onflow/flow-go/state/protocol" badgerState "github.com/onflow/flow-go/state/protocol/badger" @@ -105,16 +106,14 @@ type FollowerServiceBuilder struct { *FollowerServiceConfig // components - LibP2PNode p2p.LibP2PNode - FollowerState protocol.FollowerState - SyncCore *synchronization.Core - FinalizationDistributor *pubsub.FinalizationDistributor - FinalizedHeader *synceng.FinalizedHeaderCache - Committee hotstuff.DynamicCommittee - Finalized *flow.Header - Pending []*flow.Header - FollowerCore module.HotStuffFollower - Validator hotstuff.Validator + LibP2PNode p2p.LibP2PNode + FollowerState protocol.FollowerState + SyncCore *synchronization.Core + FollowerDistributor *pubsub.FollowerDistributor + Committee hotstuff.DynamicCommittee + Finalized *flow.Header + Pending []*flow.Header + FollowerCore module.HotStuffFollower // for the observer, the sync engine participants provider is the libp2p peer store which is not // available until after the network has started. Hence, a factory function that needs to be called just before // creating the sync engine @@ -215,13 +214,17 @@ func (builder *FollowerServiceBuilder) buildFollowerCore() *FollowerServiceBuild // state when the follower detects newly finalized blocks final := finalizer.NewFinalizer(node.DB, node.Storage.Headers, builder.FollowerState, node.Tracer) - packer := hotsignature.NewConsensusSigDataPacker(builder.Committee) - // initialize the verifier for the protocol consensus - verifier := verification.NewCombinedVerifier(builder.Committee, packer) - builder.Validator = hotstuffvalidator.New(builder.Committee, verifier) - - followerCore, err := consensus.NewFollower(node.Logger, builder.Committee, node.Storage.Headers, final, verifier, - builder.FinalizationDistributor, node.RootBlock.Header, node.RootQC, builder.Finalized, builder.Pending) + followerCore, err := consensus.NewFollower( + node.Logger, + node.Metrics.Mempool, + node.Storage.Headers, + final, + builder.FollowerDistributor, + node.FinalizedRootBlock.Header, + node.RootQC, + builder.Finalized, + builder.Pending, + ) if err != nil { return nil, fmt.Errorf("could not initialize follower core: %w", err) } @@ -240,14 +243,18 @@ func (builder *FollowerServiceBuilder) buildFollowerEngine() *FollowerServiceBui heroCacheCollector = metrics.FollowerCacheMetrics(node.MetricsRegisterer) } + packer := hotsignature.NewConsensusSigDataPacker(builder.Committee) + verifier := verification.NewCombinedVerifier(builder.Committee, packer) + val := hotstuffvalidator.New(builder.Committee, verifier) // verifier for HotStuff signature constructs (QCs, TCs, votes) + core, err := follower.NewComplianceCore( node.Logger, node.Metrics.Mempool, heroCacheCollector, - builder.FinalizationDistributor, + builder.FollowerDistributor, builder.FollowerState, builder.FollowerCore, - builder.Validator, + val, builder.SyncCore, node.Tracer, ) @@ -263,12 +270,13 @@ func (builder *FollowerServiceBuilder) buildFollowerEngine() *FollowerServiceBui node.Storage.Headers, builder.Finalized, core, + node.ComplianceConfig, follower.WithChannel(channels.PublicReceiveBlocks), - follower.WithComplianceConfigOpt(compliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold)), ) if err != nil { return nil, fmt.Errorf("could not create follower engine: %w", err) } + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.FollowerEng.OnFinalizedBlock) return builder.FollowerEng, nil }) @@ -276,20 +284,6 @@ func (builder *FollowerServiceBuilder) buildFollowerEngine() *FollowerServiceBui return builder } -func (builder *FollowerServiceBuilder) buildFinalizedHeader() *FollowerServiceBuilder { - builder.Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - finalizedHeader, err := synceng.NewFinalizedHeaderCache(node.Logger, node.State, builder.FinalizationDistributor) - if err != nil { - return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err) - } - builder.FinalizedHeader = finalizedHeader - - return builder.FinalizedHeader, nil - }) - - return builder -} - func (builder *FollowerServiceBuilder) buildSyncEngine() *FollowerServiceBuilder { builder.Component("sync engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { sync, err := synceng.New( @@ -297,16 +291,17 @@ func (builder *FollowerServiceBuilder) buildSyncEngine() *FollowerServiceBuilder node.Metrics.Engine, node.Network, node.Me, + node.State, node.Storage.Blocks, builder.FollowerEng, builder.SyncCore, - builder.FinalizedHeader, builder.SyncEngineParticipantsProviderFactory(), ) if err != nil { return nil, fmt.Errorf("could not create synchronization engine: %w", err) } builder.SyncEng = sync + builder.FollowerDistributor.AddFinalizationConsumer(sync) return builder.SyncEng, nil }) @@ -322,7 +317,6 @@ func (builder *FollowerServiceBuilder) BuildConsensusFollower() cmd.NodeBuilder buildLatestHeader(). buildFollowerCore(). buildFollowerEngine(). - buildFinalizedHeader(). buildSyncEngine() return builder @@ -356,10 +350,10 @@ func FlowConsensusFollowerService(opts ...FollowerOption) *FollowerServiceBuilde ret := &FollowerServiceBuilder{ FollowerServiceConfig: config, // TODO: using RoleAccess here for now. This should be refactored eventually to have its own role type - FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String(), config.baseOptions...), - FinalizationDistributor: pubsub.NewFinalizationDistributor(), + FlowNodeBuilder: cmd.FlowNode(flow.RoleAccess.String(), config.baseOptions...), + FollowerDistributor: pubsub.NewFollowerDistributor(), } - ret.FinalizationDistributor.AddConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) + ret.FollowerDistributor.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(ret.Logger)) // the observer gets a version of the root snapshot file that does not contain any node addresses // hence skip all the root snapshot validations that involved an identity address ret.FlowNodeBuilder.SkipNwAddressBasedValidations = true @@ -375,13 +369,9 @@ func (builder *FollowerServiceBuilder) initNetwork(nodeID module.Local, topology network.Topology, receiveCache *netcache.ReceiveCache, ) (*p2p.Network, error) { - - codec := cborcodec.NewCodec() - - // creates network instance - net, err := p2p.NewNetwork(&p2p.NetworkParameters{ + net, err := p2p.NewNetwork(&p2p.NetworkConfig{ Logger: builder.Logger, - Codec: codec, + Codec: cborcodec.NewCodec(), Me: nodeID, MiddlewareFactory: func() (network.Middleware, error) { return builder.Middleware, nil }, Topology: topology, @@ -389,6 +379,17 @@ func (builder *FollowerServiceBuilder) initNetwork(nodeID module.Local, Metrics: networkMetrics, IdentityProvider: builder.IdentityProvider, ReceiveCache: receiveCache, + ConduitFactory: conduit.NewDefaultConduitFactory(), + AlspCfg: &alspmgr.MisbehaviorReportManagerConfig{ + Logger: builder.Logger, + SpamRecordCacheSize: builder.FlowConfig.NetworkConfig.AlspConfig.SpamRecordCacheSize, + SpamReportQueueSize: builder.FlowConfig.NetworkConfig.AlspConfig.SpamReportQueueSize, + DisablePenalty: builder.FlowConfig.NetworkConfig.AlspConfig.DisablePenalty, + HeartBeatInterval: builder.FlowConfig.NetworkConfig.AlspConfig.HearBeatInterval, + AlspMetrics: builder.Metrics.Network, + HeroCacheMetricsFactory: builder.HeroCacheMetricsFactory(), + NetworkType: network.PublicNetwork, + }, }) if err != nil { return nil, fmt.Errorf("could not initialize network: %w", err) @@ -482,11 +483,11 @@ func (builder *FollowerServiceBuilder) InitIDProviders() { } builder.IDTranslator = translator.NewHierarchicalIDTranslator(idCache, translator.NewPublicNetworkIDTranslator()) - builder.NodeDisallowListDistributor = cmd.BuildDisallowListNotificationDisseminator(builder.DisallowListNotificationCacheSize, builder.MetricsRegisterer, builder.Logger, builder.MetricsEnabled) - // The following wrapper allows to disallow-list byzantine nodes via an admin command: // the wrapper overrides the 'Ejected' flag of the disallow-listed nodes to true - builder.IdentityProvider, err = cache.NewNodeBlocklistWrapper(idCache, node.DB, builder.NodeDisallowListDistributor) + builder.IdentityProvider, err = cache.NewNodeDisallowListWrapper(idCache, node.DB, func() network.DisallowListNotificationConsumer { + return builder.Middleware + }) if err != nil { return fmt.Errorf("could not initialize NodeBlockListWrapper: %w", err) } @@ -517,14 +518,14 @@ func (builder *FollowerServiceBuilder) InitIDProviders() { return nil }) - - builder.Component("disallow list notification distributor", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - // distributor is returned as a component to be started and stopped. - return builder.NodeDisallowListDistributor, nil - }) } func (builder *FollowerServiceBuilder) Initialize() error { + // initialize default flow configuration + if err := config.Unmarshall(&builder.FlowConfig); err != nil { + return fmt.Errorf("failed to initialize flow config for follower builder: %w", err) + } + if err := builder.deriveBootstrapPeerIdentities(); err != nil { return err } @@ -574,7 +575,7 @@ func (builder *FollowerServiceBuilder) validateParams() error { return nil } -// initPublicLibP2PFactory creates the LibP2P factory function for the given node ID and network key for the observer. +// initPublicLibp2pNode creates a libp2p node for the follower service in public (unstaked) network. // The factory function is later passed into the initMiddleware function to eventually instantiate the p2p.LibP2PNode instance // The LibP2P host is created with the following options: // - DHT as client and seeded with the given bootstrap peers @@ -584,69 +585,80 @@ func (builder *FollowerServiceBuilder) validateParams() error { // - No connection manager // - No peer manager // - Default libp2p pubsub options -func (builder *FollowerServiceBuilder) initPublicLibP2PFactory(networkKey crypto.PrivateKey) p2p.LibP2PFactoryFunc { - return func() (p2p.LibP2PNode, error) { - var pis []peer.AddrInfo - - for _, b := range builder.bootstrapIdentities { - pi, err := utils.PeerAddressInfo(*b) - - if err != nil { - return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err) - } - - pis = append(pis, pi) - } - - meshTracer := tracer.NewGossipSubMeshTracer( - builder.Logger, - builder.Metrics.Network, - builder.IdentityProvider, - builder.GossipSubConfig.LocalMeshLogInterval) - - rpcInspectorBuilder := inspector.NewGossipSubInspectorBuilder(builder.Logger, builder.SporkID, builder.GossipSubRPCInspectorsConfig, builder.GossipSubInspectorNotifDistributor) - rpcInspectors, err := rpcInspectorBuilder. - SetPublicNetwork(p2p.PublicNetworkEnabled). - SetMetrics(builder.Metrics.Network, builder.MetricsRegisterer). - SetMetricsEnabled(builder.MetricsEnabled).Build() +// +// Args: +// - networkKey: the private key to use for the libp2p node +// +// Returns: +// - p2p.LibP2PNode: the libp2p node +// - error: if any error occurs. Any error returned from this function is irrecoverable. +func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.PrivateKey) (p2p.LibP2PNode, error) { + var pis []peer.AddrInfo + + for _, b := range builder.bootstrapIdentities { + pi, err := utils.PeerAddressInfo(*b) if err != nil { - return nil, fmt.Errorf("failed to create gossipsub rpc inspectors for public libp2p node: %w", err) + return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err) } - node, err := p2pbuilder.NewNodeBuilder( - builder.Logger, - builder.Metrics.Network, - builder.BaseConfig.BindAddr, - networkKey, - builder.SporkID, - builder.LibP2PResourceManagerConfig). - SetSubscriptionFilter( - subscription.NewRoleBasedFilter( - subscription.UnstakedRole, builder.IdentityProvider, - ), - ). - SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { - return p2pdht.NewDHT(ctx, h, protocols.FlowPublicDHTProtocolID(builder.SporkID), - builder.Logger, - builder.Metrics.Network, - p2pdht.AsClient(), - dht.BootstrapPeers(pis...), - ) - }). - SetStreamCreationRetryInterval(builder.UnicastCreateStreamRetryDelay). - SetGossipSubTracer(meshTracer). - SetGossipSubScoreTracerInterval(builder.GossipSubConfig.ScoreTracerInterval). - SetGossipSubRPCInspectors(rpcInspectors...). - Build() - - if err != nil { - return nil, fmt.Errorf("could not build public libp2p node: %w", err) - } + pis = append(pis, pi) + } - builder.LibP2PNode = node + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: builder.Logger, + Metrics: builder.Metrics.Network, + IDProvider: builder.IdentityProvider, + LoggerInterval: builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, + RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + RpcSentTrackerWorkerQueueCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, + RpcSentTrackerNumOfWorkers: builder.FlowConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, + HeroCacheMetricsFactory: builder.HeroCacheMetricsFactory(), + NetworkingType: network.PublicNetwork, + } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) - return builder.LibP2PNode, nil + node, err := p2pbuilder.NewNodeBuilder( + builder.Logger, + &p2pconfig.MetricsConfig{ + HeroCacheFactory: builder.HeroCacheMetricsFactory(), + Metrics: builder.Metrics.Network, + }, + network.PublicNetwork, + builder.BaseConfig.BindAddr, + networkKey, + builder.SporkID, + builder.IdentityProvider, + &builder.FlowConfig.NetworkConfig.ResourceManagerConfig, + &builder.FlowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig, + p2pconfig.PeerManagerDisableConfig(), // disable peer manager for follower + &p2p.DisallowListCacheConfig{ + MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, + Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), + }). + SetSubscriptionFilter( + subscription.NewRoleBasedFilter( + subscription.UnstakedRole, builder.IdentityProvider, + ), + ). + SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { + return p2pdht.NewDHT(ctx, h, protocols.FlowPublicDHTProtocolID(builder.SporkID), + builder.Logger, + builder.Metrics.Network, + p2pdht.AsClient(), + dht.BootstrapPeers(pis...), + ) + }). + SetStreamCreationRetryInterval(builder.FlowConfig.NetworkConfig.UnicastCreateStreamRetryDelay). + SetGossipSubTracer(meshTracer). + SetGossipSubScoreTracerInterval(builder.FlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval). + Build() + if err != nil { + return nil, fmt.Errorf("could not build public libp2p node: %w", err) } + + builder.LibP2PNode = node + + return builder.LibP2PNode, nil } // initObserverLocal initializes the observer's ID, network key and network address @@ -681,36 +693,30 @@ func (builder *FollowerServiceBuilder) Build() (cmd.Node, error) { // enqueuePublicNetworkInit enqueues the observer network component initialized for the observer func (builder *FollowerServiceBuilder) enqueuePublicNetworkInit() { - var libp2pNode p2p.LibP2PNode + var publicLibp2pNode p2p.LibP2PNode builder. Component("public libp2p node", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - libP2PFactory := builder.initPublicLibP2PFactory(node.NetworkKey) - var err error - libp2pNode, err = libP2PFactory() + publicLibp2pNode, err = builder.initPublicLibp2pNode(node.NetworkKey) if err != nil { return nil, fmt.Errorf("could not create public libp2p node: %w", err) } - return libp2pNode, nil + return publicLibp2pNode, nil }). Component("public network", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - var heroCacheCollector module.HeroCacheMetrics = metrics.NewNoopCollector() - if builder.HeroCacheMetricsEnable { - heroCacheCollector = metrics.NetworkReceiveCacheMetricsFactory(builder.MetricsRegisterer) - } - receiveCache := netcache.NewHeroReceiveCache(builder.NetworkReceivedMessageCacheSize, + receiveCache := netcache.NewHeroReceiveCache(builder.FlowConfig.NetworkConfig.NetworkReceivedMessageCacheSize, builder.Logger, - heroCacheCollector) + metrics.NetworkReceiveCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork)) - err := node.Metrics.Mempool.Register(metrics.ResourceNetworkingReceiveCache, receiveCache.Size) + err := node.Metrics.Mempool.Register(metrics.PrependPublicPrefix(metrics.ResourceNetworkingReceiveCache), receiveCache.Size) if err != nil { return nil, fmt.Errorf("could not register networking receive cache metric: %w", err) } msgValidators := publicNetworkMsgValidators(node.Logger, node.IdentityProvider, node.NodeID) - builder.initMiddleware(node.NodeID, libp2pNode, msgValidators...) + builder.initMiddleware(node.NodeID, publicLibp2pNode, msgValidators...) // topology is nil since it is automatically managed by libp2p net, err := builder.initNetwork(builder.Me, builder.Metrics.Network, builder.Middleware, nil, receiveCache) @@ -747,21 +753,18 @@ func (builder *FollowerServiceBuilder) initMiddleware(nodeID flow.Identifier, libp2pNode p2p.LibP2PNode, validators ...network.MessageValidator, ) network.Middleware { - slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network) - mw := middleware.NewMiddleware( - builder.Logger, - libp2pNode, - nodeID, - builder.Metrics.Bitswap, - builder.SporkID, - middleware.DefaultUnicastTimeout, - builder.IDTranslator, - builder.CodecFactory(), - slashingViolationsConsumer, + mw := middleware.NewMiddleware(&middleware.Config{ + Logger: builder.Logger, + Libp2pNode: libp2pNode, + FlowId: nodeID, + BitSwapMetrics: builder.Metrics.Bitswap, + RootBlockID: builder.SporkID, + UnicastMessageTimeout: middleware.DefaultUnicastTimeout, + IdTranslator: builder.IDTranslator, + Codec: builder.CodecFactory(), + }, middleware.WithMessageValidators(validators...), - // use default identifier provider ) - builder.NodeDisallowListDistributor.AddConsumer(mw) builder.Middleware = mw return builder.Middleware } diff --git a/fvm/README.md b/fvm/README.md index 80c0f733536..b30856d12fa 100644 --- a/fvm/README.md +++ b/fvm/README.md @@ -11,7 +11,7 @@ functionality required by the Flow protocol. import ( "github.com/onflow/cadence/runtime" "github.com/onflow/flow-go/fvm" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/model/flow" ) @@ -26,7 +26,7 @@ ledger := state.NewMapLedger() txIndex := uint32(0) txProc := fvm.Transaction(tx, txIndex) -err := vm.Run(ctx, txProc, ledger) +executionSnapshot, output, err := vm.Run(ctx, txProc, ledger) if err != nil { panic("fatal error during transaction procedure!") } diff --git a/fvm/accounts_test.go b/fvm/accounts_test.go index 2d2315aed37..02613379e73 100644 --- a/fvm/accounts_test.go +++ b/fvm/accounts_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/runtime/format" "github.com/stretchr/testify/assert" @@ -13,13 +14,13 @@ import ( "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/fvm" - "github.com/onflow/flow-go/fvm/storage" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) type errorOnAddressSnapshotWrapper struct { - snapshotTree storage.SnapshotTree + snapshotTree snapshot.SnapshotTree owner flow.Address } @@ -42,9 +43,9 @@ func createAccount( vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) ( - storage.SnapshotTree, + snapshot.SnapshotTree, flow.Address, ) { ctx = fvm.NewContextFromParent( @@ -70,7 +71,7 @@ func createAccount( require.Len(t, accountCreatedEvents, 1) - data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) + data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(t, err) address := flow.ConvertAddress( data.(cadence.Event).Fields[0].(cadence.Address)) @@ -89,11 +90,11 @@ func addAccountKey( t *testing.T, vm fvm.VM, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, address flow.Address, apiVersion accountKeyAPIVersion, ) ( - storage.SnapshotTree, + snapshot.SnapshotTree, flow.AccountPublicKey, ) { @@ -131,9 +132,9 @@ func addAccountCreator( vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, account flow.Address, -) storage.SnapshotTree { +) snapshot.SnapshotTree { script := []byte( fmt.Sprintf(addAccountCreatorTransactionTemplate, chain.ServiceAddress().String(), @@ -160,9 +161,9 @@ func removeAccountCreator( vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, account flow.Address, -) storage.SnapshotTree { +) snapshot.SnapshotTree { script := []byte( fmt.Sprintf( removeAccountCreatorTransactionTemplate, @@ -383,7 +384,7 @@ func TestCreateAccount(t *testing.T) { t.Run("Single account", newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, payer := createAccount( t, vm, @@ -407,7 +408,7 @@ func TestCreateAccount(t *testing.T) { accountCreatedEvents := filterAccountCreatedEvents(output.Events) require.Len(t, accountCreatedEvents, 1) - data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) + data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(t, err) address := flow.ConvertAddress( data.(cadence.Event).Fields[0].(cadence.Address)) @@ -420,7 +421,7 @@ func TestCreateAccount(t *testing.T) { t.Run("Multiple accounts", newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { const count = 3 snapshotTree, payer := createAccount( @@ -450,7 +451,7 @@ func TestCreateAccount(t *testing.T) { } accountCreatedEventCount += 1 - data, err := jsoncdc.Decode(nil, event.Payload) + data, err := ccf.Decode(nil, event.Payload) require.NoError(t, err) address := flow.ConvertAddress( data.(cadence.Event).Fields[0].(cadence.Address)) @@ -475,7 +476,7 @@ func TestCreateAccount_WithRestrictedAccountCreation(t *testing.T) { newVMTest(). withContextOptions(options...). withBootstrapProcedureOptions(fvm.WithRestrictedAccountCreationEnabled(true)). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, payer := createAccount( t, vm, @@ -500,7 +501,7 @@ func TestCreateAccount_WithRestrictedAccountCreation(t *testing.T) { t.Run("Authorized account payer", newVMTest().withContextOptions(options...). withBootstrapProcedureOptions(fvm.WithRestrictedAccountCreationEnabled(true)). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { txBody := flow.NewTransactionBody(). SetScript([]byte(createAccountTransaction)). AddAuthorizer(chain.ServiceAddress()) @@ -518,7 +519,7 @@ func TestCreateAccount_WithRestrictedAccountCreation(t *testing.T) { t.Run("Account payer added to allowlist", newVMTest().withContextOptions(options...). withBootstrapProcedureOptions(fvm.WithRestrictedAccountCreationEnabled(true)). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, payer := createAccount( t, vm, @@ -551,7 +552,7 @@ func TestCreateAccount_WithRestrictedAccountCreation(t *testing.T) { t.Run("Account payer removed from allowlist", newVMTest().withContextOptions(options...). withBootstrapProcedureOptions(fvm.WithRestrictedAccountCreationEnabled(true)). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, payer := createAccount( t, vm, @@ -627,7 +628,7 @@ func TestAddAccountKey(t *testing.T) { t.Run(fmt.Sprintf("Add to empty key list %s", test.apiVersion), newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -675,7 +676,7 @@ func TestAddAccountKey(t *testing.T) { t.Run(fmt.Sprintf("Add to non-empty key list %s", test.apiVersion), newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -737,7 +738,7 @@ func TestAddAccountKey(t *testing.T) { t.Run(fmt.Sprintf("Invalid key %s", test.apiVersion), newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -787,7 +788,7 @@ func TestAddAccountKey(t *testing.T) { for _, test := range multipleKeysTests { t.Run(fmt.Sprintf("Multiple keys %s", test.apiVersion), newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -850,7 +851,7 @@ func TestAddAccountKey(t *testing.T) { t.Run(hashAlgo, newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -941,7 +942,7 @@ func TestRemoveAccountKey(t *testing.T) { t.Run(fmt.Sprintf("Non-existent key %s", test.apiVersion), newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -1001,7 +1002,7 @@ func TestRemoveAccountKey(t *testing.T) { t.Run(fmt.Sprintf("Existing key %s", test.apiVersion), newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -1057,7 +1058,7 @@ func TestRemoveAccountKey(t *testing.T) { t.Run(fmt.Sprintf("Key added by a different api version %s", test.apiVersion), newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -1136,7 +1137,7 @@ func TestRemoveAccountKey(t *testing.T) { for _, test := range multipleKeysTests { t.Run(fmt.Sprintf("Multiple keys %s", test.apiVersion), newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -1202,7 +1203,7 @@ func TestGetAccountKey(t *testing.T) { t.Run("Non-existent key", newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -1252,7 +1253,7 @@ func TestGetAccountKey(t *testing.T) { t.Run("Existing key", newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -1314,7 +1315,7 @@ func TestGetAccountKey(t *testing.T) { t.Run("Key added by a different api version", newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -1378,7 +1379,7 @@ func TestGetAccountKey(t *testing.T) { t.Run("Multiple keys", newVMTest().withContextOptions(options...). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -1459,7 +1460,7 @@ func TestAccountBalanceFields(t *testing.T) { fvm.WithSequenceNumberCheckAndIncrementEnabled(false), fvm.WithCadenceLogging(true), ). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, account := createAccount( t, vm, @@ -1507,7 +1508,7 @@ func TestAccountBalanceFields(t *testing.T) { fvm.WithSequenceNumberCheckAndIncrementEnabled(false), fvm.WithCadenceLogging(true), ). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { nonExistentAddress, err := chain.AddressAtIndex(100) require.NoError(t, err) @@ -1534,7 +1535,7 @@ func TestAccountBalanceFields(t *testing.T) { fvm.WithSequenceNumberCheckAndIncrementEnabled(false), fvm.WithCadenceLogging(true), ). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, address := createAccount( t, vm, @@ -1573,7 +1574,7 @@ func TestAccountBalanceFields(t *testing.T) { ).withBootstrapProcedureOptions( fvm.WithStorageMBPerFLOW(1000_000_000), ). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, account := createAccount( t, vm, @@ -1605,7 +1606,7 @@ func TestAccountBalanceFields(t *testing.T) { _, output, err = vm.Run(ctx, script, snapshotTree) assert.NoError(t, err) assert.NoError(t, output.Err) - assert.Equal(t, cadence.UFix64(9999_3120), output.Value) + assert.Equal(t, cadence.UFix64(99_993_040), output.Value) }), ) @@ -1618,7 +1619,7 @@ func TestAccountBalanceFields(t *testing.T) { ).withBootstrapProcedureOptions( fvm.WithStorageMBPerFLOW(1_000_000_000), ). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { nonExistentAddress, err := chain.AddressAtIndex(100) require.NoError(t, err) @@ -1646,7 +1647,7 @@ func TestAccountBalanceFields(t *testing.T) { fvm.WithAccountCreationFee(100_000), fvm.WithMinimumStorageReservation(100_000), ). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, account := createAccount( t, vm, @@ -1697,7 +1698,7 @@ func TestGetStorageCapacity(t *testing.T) { fvm.WithAccountCreationFee(100_000), fvm.WithMinimumStorageReservation(100_000), ). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { snapshotTree, account := createAccount( t, vm, @@ -1744,7 +1745,7 @@ func TestGetStorageCapacity(t *testing.T) { fvm.WithAccountCreationFee(100_000), fvm.WithMinimumStorageReservation(100_000), ). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { nonExistentAddress, err := chain.AddressAtIndex(100) require.NoError(t, err) @@ -1773,7 +1774,7 @@ func TestGetStorageCapacity(t *testing.T) { fvm.WithAccountCreationFee(100_000), fvm.WithMinimumStorageReservation(100_000), ). - run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { address := chain.ServiceAddress() script := fvm.Script([]byte(fmt.Sprintf(` diff --git a/fvm/blueprints/contracts.go b/fvm/blueprints/contracts.go index dee250b4bac..bbe3ce422ab 100644 --- a/fvm/blueprints/contracts.go +++ b/fvm/blueprints/contracts.go @@ -13,15 +13,15 @@ import ( ) var ContractDeploymentAuthorizedAddressesPath = cadence.Path{ - Domain: common.PathDomainStorage.Identifier(), + Domain: common.PathDomainStorage, Identifier: "authorizedAddressesToDeployContracts", } var ContractRemovalAuthorizedAddressesPath = cadence.Path{ - Domain: common.PathDomainStorage.Identifier(), + Domain: common.PathDomainStorage, Identifier: "authorizedAddressesToRemoveContracts", } var IsContractDeploymentRestrictedPath = cadence.Path{ - Domain: common.PathDomainStorage.Identifier(), + Domain: common.PathDomainStorage, Identifier: "isContractDeploymentRestricted", } diff --git a/fvm/blueprints/fees.go b/fvm/blueprints/fees.go index 72b6f1645d1..486dda41886 100644 --- a/fvm/blueprints/fees.go +++ b/fvm/blueprints/fees.go @@ -15,15 +15,15 @@ import ( ) var TransactionFeesExecutionEffortWeightsPath = cadence.Path{ - Domain: common.PathDomainStorage.Identifier(), + Domain: common.PathDomainStorage, Identifier: "executionEffortWeights", } var TransactionFeesExecutionMemoryWeightsPath = cadence.Path{ - Domain: common.PathDomainStorage.Identifier(), + Domain: common.PathDomainStorage, Identifier: "executionMemoryWeights", } var TransactionFeesExecutionMemoryLimitPath = cadence.Path{ - Domain: common.PathDomainStorage.Identifier(), + Domain: common.PathDomainStorage, Identifier: "executionMemoryLimit", } diff --git a/fvm/blueprints/scripts/deployNodeVersionBeaconTransactionTemplate.cdc b/fvm/blueprints/scripts/deployNodeVersionBeaconTransactionTemplate.cdc new file mode 100644 index 00000000000..24c05ac47c1 --- /dev/null +++ b/fvm/blueprints/scripts/deployNodeVersionBeaconTransactionTemplate.cdc @@ -0,0 +1,5 @@ +transaction(code: String, versionThreshold: UInt64) { + prepare(serviceAccount: AuthAccount) { + serviceAccount.contracts.add(name: "NodeVersionBeacon", code: code.decodeHex(), versionUpdateBuffer: versionThreshold) + } +} \ No newline at end of file diff --git a/fvm/blueprints/scripts/systemChunkTransactionTemplate.cdc b/fvm/blueprints/scripts/systemChunkTransactionTemplate.cdc index 29f790fd098..bdc083bddf2 100644 --- a/fvm/blueprints/scripts/systemChunkTransactionTemplate.cdc +++ b/fvm/blueprints/scripts/systemChunkTransactionTemplate.cdc @@ -1,9 +1,15 @@ import FlowEpoch from 0xEPOCHADDRESS +import NodeVersionBeacon from 0xNODEVERSIONBEACONADDRESS transaction { - prepare(serviceAccount: AuthAccount) { - let heartbeat = serviceAccount.borrow<&FlowEpoch.Heartbeat>(from: FlowEpoch.heartbeatStoragePath) - ?? panic("Could not borrow heartbeat from storage path") - heartbeat.advanceBlock() - } + prepare(serviceAccount: AuthAccount) { + let epochHeartbeat = serviceAccount.borrow<&FlowEpoch.Heartbeat>(from: FlowEpoch.heartbeatStoragePath) + ?? panic("Could not borrow heartbeat from storage path") + epochHeartbeat.advanceBlock() + + let versionBeaconHeartbeat = serviceAccount.borrow<&NodeVersionBeacon.Heartbeat>( + from: NodeVersionBeacon.HeartbeatStoragePath) + ?? panic("Couldn't borrow NodeVersionBeacon.Heartbeat Resource") + versionBeaconHeartbeat.heartbeat() + } } diff --git a/fvm/blueprints/scripts/systemChunkTransactionTemplateDualAuthorizer.cdc b/fvm/blueprints/scripts/systemChunkTransactionTemplateDualAuthorizer.cdc new file mode 100644 index 00000000000..7c5d60d2a97 --- /dev/null +++ b/fvm/blueprints/scripts/systemChunkTransactionTemplateDualAuthorizer.cdc @@ -0,0 +1,18 @@ +import FlowEpoch from 0xEPOCHADDRESS +import NodeVersionBeacon from 0xNODEVERSIONBEACONADDRESS + +transaction { + prepare(serviceAccount: AuthAccount, epochAccount: AuthAccount) { + let epochHeartbeat = + serviceAccount.borrow<&FlowEpoch.Heartbeat>(from: FlowEpoch.heartbeatStoragePath) ?? + epochAccount.borrow<&FlowEpoch.Heartbeat>(from: FlowEpoch.heartbeatStoragePath) ?? + panic("Could not borrow heartbeat from storage path") + epochHeartbeat.advanceBlock() + + let versionBeaconHeartbeat = + serviceAccount.borrow<&NodeVersionBeacon.Heartbeat>(from: NodeVersionBeacon.HeartbeatStoragePath) ?? + epochAccount.borrow<&NodeVersionBeacon.Heartbeat>(from: NodeVersionBeacon.HeartbeatStoragePath) ?? + panic("Couldn't borrow NodeVersionBeacon.Heartbeat Resource") + versionBeaconHeartbeat.heartbeat() + } +} diff --git a/fvm/blueprints/system.go b/fvm/blueprints/system.go index faaa8bf4cdd..f4c6893b34b 100644 --- a/fvm/blueprints/system.go +++ b/fvm/blueprints/system.go @@ -14,24 +14,71 @@ const SystemChunkTransactionGasLimit = 100_000_000 // TODO (Ramtin) after changes to this method are merged into master move them here. +// systemChunkTransactionTemplate looks for the epoch and version beacon heartbeat resources +// and calls them. +// //go:embed scripts/systemChunkTransactionTemplate.cdc var systemChunkTransactionTemplate string -// SystemChunkTransaction creates and returns the transaction corresponding to the system chunk -// for the given chain. +// SystemChunkTransaction creates and returns the transaction corresponding to the +// system chunk for the given chain. func SystemChunkTransaction(chain flow.Chain) (*flow.TransactionBody, error) { - contracts, err := systemcontracts.SystemContractsForChain(chain.ChainID()) if err != nil { return nil, fmt.Errorf("could not get system contracts for chain: %w", err) } + // this is only true for testnet, sandboxnet and mainnet. + if contracts.Epoch.Address != chain.ServiceAddress() { + // Temporary workaround because the heartbeat resources need to be moved + // to the service account: + // - the system chunk will attempt to load both Epoch and VersionBeacon + // resources from either the service account or the staking account + // - the service account committee can then safely move the resources + // at any time + // - once the resources are moved, this workaround should be removed + // after version v0.31.0 + return systemChunkTransactionDualAuthorizers(chain, contracts) + } + + tx := flow.NewTransactionBody(). + SetScript( + []byte(templates.ReplaceAddresses( + systemChunkTransactionTemplate, + templates.Environment{ + EpochAddress: contracts.Epoch.Address.Hex(), + NodeVersionBeaconAddress: contracts.NodeVersionBeacon.Address.Hex(), + }, + )), + ). + AddAuthorizer(contracts.Epoch.Address). + SetGasLimit(SystemChunkTransactionGasLimit) + + return tx, nil +} + +// systemChunkTransactionTemplateDualAuthorizer is the same as systemChunkTransactionTemplate +// but it looks for the heartbeat resources on two different accounts. +// +//go:embed scripts/systemChunkTransactionTemplateDualAuthorizer.cdc +var systemChunkTransactionTemplateDualAuthorizer string + +func systemChunkTransactionDualAuthorizers( + chain flow.Chain, + contracts *systemcontracts.SystemContracts, +) (*flow.TransactionBody, error) { + tx := flow.NewTransactionBody(). - SetScript([]byte(templates.ReplaceAddresses(systemChunkTransactionTemplate, - templates.Environment{ - EpochAddress: contracts.Epoch.Address.Hex(), - })), + SetScript( + []byte(templates.ReplaceAddresses( + systemChunkTransactionTemplateDualAuthorizer, + templates.Environment{ + EpochAddress: contracts.Epoch.Address.Hex(), + NodeVersionBeaconAddress: contracts.NodeVersionBeacon.Address.Hex(), + }, + )), ). + AddAuthorizer(chain.ServiceAddress()). AddAuthorizer(contracts.Epoch.Address). SetGasLimit(SystemChunkTransactionGasLimit) diff --git a/fvm/blueprints/token.go b/fvm/blueprints/token.go index 92cc09e22c3..4058feb6519 100644 --- a/fvm/blueprints/token.go +++ b/fvm/blueprints/token.go @@ -22,6 +22,42 @@ func DeployFungibleTokenContractTransaction(fungibleToken flow.Address) *flow.Tr contractName) } +func DeployNonFungibleTokenContractTransaction(nonFungibleToken flow.Address) *flow.TransactionBody { + contract := contracts.NonFungibleToken() + contractName := "NonFungibleToken" + return DeployContractTransaction( + nonFungibleToken, + contract, + contractName) +} + +func DeployMetadataViewsContractTransaction(fungibleToken, nonFungibleToken flow.Address) *flow.TransactionBody { + contract := contracts.MetadataViews(fungibleToken.HexWithPrefix(), nonFungibleToken.HexWithPrefix()) + contractName := "MetadataViews" + return DeployContractTransaction( + nonFungibleToken, + contract, + contractName) +} + +func DeployViewResolverContractTransaction(nonFungibleToken flow.Address) *flow.TransactionBody { + contract := contracts.ViewResolver() + contractName := "ViewResolver" + return DeployContractTransaction( + nonFungibleToken, + contract, + contractName) +} + +func DeployFungibleTokenMetadataViewsContractTransaction(fungibleToken, nonFungibleToken flow.Address) *flow.TransactionBody { + contract := contracts.FungibleTokenMetadataViews(fungibleToken.Hex(), nonFungibleToken.Hex()) + contractName := "FungibleTokenMetadataViews" + return DeployContractTransaction( + fungibleToken, + contract, + contractName) +} + //go:embed scripts/deployFlowTokenTransactionTemplate.cdc var deployFlowTokenTransactionTemplate string @@ -31,8 +67,8 @@ var createFlowTokenMinterTransactionTemplate string //go:embed scripts/mintFlowTokenTransactionTemplate.cdc var mintFlowTokenTransactionTemplate string -func DeployFlowTokenContractTransaction(service, fungibleToken, flowToken flow.Address) *flow.TransactionBody { - contract := contracts.FlowToken(fungibleToken.HexWithPrefix()) +func DeployFlowTokenContractTransaction(service, fungibleToken, metadataViews, flowToken flow.Address) *flow.TransactionBody { + contract := contracts.FlowToken(fungibleToken.HexWithPrefix(), metadataViews.HexWithPrefix(), metadataViews.HexWithPrefix()) return flow.NewTransactionBody(). SetScript([]byte(deployFlowTokenTransactionTemplate)). diff --git a/fvm/blueprints/version_beacon.go b/fvm/blueprints/version_beacon.go new file mode 100644 index 00000000000..ba3535db728 --- /dev/null +++ b/fvm/blueprints/version_beacon.go @@ -0,0 +1,28 @@ +package blueprints + +import ( + _ "embed" + "encoding/hex" + + "github.com/onflow/cadence" + jsoncdc "github.com/onflow/cadence/encoding/json" + + "github.com/onflow/flow-core-contracts/lib/go/contracts" + + "github.com/onflow/flow-go/model/flow" +) + +//go:embed scripts/deployNodeVersionBeaconTransactionTemplate.cdc +var deployNodeVersionBeaconTransactionTemplate string + +// DeployNodeVersionBeaconTransaction returns the transaction body for the deployment NodeVersionBeacon contract transaction +func DeployNodeVersionBeaconTransaction( + service flow.Address, + versionFreezePeriod cadence.UInt64, +) *flow.TransactionBody { + return flow.NewTransactionBody(). + SetScript([]byte(deployNodeVersionBeaconTransactionTemplate)). + AddArgument(jsoncdc.MustEncode(cadence.String(hex.EncodeToString(contracts.NodeVersionBeacon())))). + AddArgument(jsoncdc.MustEncode(versionFreezePeriod)). + AddAuthorizer(service) +} diff --git a/fvm/bootstrap.go b/fvm/bootstrap.go index a1d503ab7bf..dc938792d0a 100644 --- a/fvm/bootstrap.go +++ b/fvm/bootstrap.go @@ -12,7 +12,6 @@ import ( "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/meter" "github.com/onflow/flow-go/fvm/storage" - "github.com/onflow/flow-go/fvm/storage/derived" "github.com/onflow/flow-go/fvm/storage/logical" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/epochs" @@ -45,6 +44,10 @@ var ( "fee execution effort cost", "0.0"), } + + // DefaultVersionFreezePeriod is the default NodeVersionBeacon freeze period - + // the number of blocks in the future where the version changes are frozen. + DefaultVersionFreezePeriod = cadence.UInt64(1000) ) func mustParseUFix64(name string, valueString string) cadence.UFix64 { @@ -73,6 +76,12 @@ type BootstrapParams struct { storagePerFlow cadence.UFix64 restrictedAccountCreationEnabled cadence.Bool + // versionFreezePeriod is the number of blocks in the future where the version + // changes are frozen. The Node version beacon manages the freeze period, + // but this is the value used when first deploying the contract, during the + // bootstrap procedure. + versionFreezePeriod cadence.UInt64 + // TODO: restrictedContractDeployment should be a bool after RestrictedDeploymentEnabled is removed from the context // restrictedContractDeployment of nil means that the contract deployment is taken from the fvm Context instead of from the state. // This can be used to mimic behaviour on chain before the restrictedContractDeployment is set with a service account transaction. @@ -222,8 +231,9 @@ func Bootstrap( FlowTokenAccountPublicKeys: []flow.AccountPublicKey{serviceAccountPublicKey}, NodeAccountPublicKeys: []flow.AccountPublicKey{serviceAccountPublicKey}, }, - transactionFees: BootstrapProcedureFeeParameters{0, 0, 0}, - epochConfig: epochs.DefaultEpochConfig(), + transactionFees: BootstrapProcedureFeeParameters{0, 0, 0}, + epochConfig: epochs.DefaultEpochConfig(), + versionFreezePeriod: DefaultVersionFreezePeriod, }, } @@ -235,15 +245,11 @@ func Bootstrap( func (b *BootstrapProcedure) NewExecutor( ctx Context, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) ProcedureExecutor { return newBootstrapExecutor(b.BootstrapParams, ctx, txnState) } -func (BootstrapProcedure) SetOutput(output ProcedureOutput) { - // do nothing -} - func (proc *BootstrapProcedure) ComputationLimit(_ Context) uint64 { return math.MaxUint64 } @@ -268,7 +274,7 @@ type bootstrapExecutor struct { BootstrapParams ctx Context - txnState storage.Transaction + txnState storage.TransactionPreparer accountCreator environment.BootstrapAccountCreator } @@ -276,7 +282,7 @@ type bootstrapExecutor struct { func newBootstrapExecutor( params BootstrapParams, ctx Context, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) *bootstrapExecutor { return &bootstrapExecutor{ BootstrapParams: params, @@ -312,7 +318,9 @@ func (b *bootstrapExecutor) Execute() error { service := b.createServiceAccount() fungibleToken := b.deployFungibleToken() - flowToken := b.deployFlowToken(service, fungibleToken) + nonFungibleToken := b.deployNonFungibleToken(service) + b.deployMetadataViews(fungibleToken, nonFungibleToken) + flowToken := b.deployFlowToken(service, fungibleToken, nonFungibleToken) storageFees := b.deployStorageFees(service, fungibleToken, flowToken) feeContract := b.deployFlowFees(service, fungibleToken, flowToken, storageFees) @@ -354,6 +362,8 @@ func (b *bootstrapExecutor) Execute() error { b.deployEpoch(service, fungibleToken, flowToken, feeContract) + b.deployVersionBeacon(service, b.versionFreezePeriod) + // deploy staking proxy contract to the service account b.deployStakingProxyContract(service) @@ -403,7 +413,46 @@ func (b *bootstrapExecutor) deployFungibleToken() flow.Address { return fungibleToken } -func (b *bootstrapExecutor) deployFlowToken(service, fungibleToken flow.Address) flow.Address { +func (b *bootstrapExecutor) deployNonFungibleToken(deployTo flow.Address) flow.Address { + + txError, err := b.invokeMetaTransaction( + b.ctx, + Transaction( + blueprints.DeployNonFungibleTokenContractTransaction(deployTo), + 0), + ) + panicOnMetaInvokeErrf("failed to deploy non-fungible token contract: %s", txError, err) + return deployTo +} + +func (b *bootstrapExecutor) deployMetadataViews(fungibleToken, nonFungibleToken flow.Address) { + + txError, err := b.invokeMetaTransaction( + b.ctx, + Transaction( + blueprints.DeployMetadataViewsContractTransaction(fungibleToken, nonFungibleToken), + 0), + ) + panicOnMetaInvokeErrf("failed to deploy metadata views contract: %s", txError, err) + + txError, err = b.invokeMetaTransaction( + b.ctx, + Transaction( + blueprints.DeployViewResolverContractTransaction(nonFungibleToken), + 0), + ) + panicOnMetaInvokeErrf("failed to deploy view resolver contract: %s", txError, err) + + txError, err = b.invokeMetaTransaction( + b.ctx, + Transaction( + blueprints.DeployFungibleTokenMetadataViewsContractTransaction(fungibleToken, nonFungibleToken), + 0), + ) + panicOnMetaInvokeErrf("failed to deploy fungible token metadata views contract: %s", txError, err) +} + +func (b *bootstrapExecutor) deployFlowToken(service, fungibleToken, metadataViews flow.Address) flow.Address { flowToken := b.createAccount(b.accountKeys.FlowTokenAccountPublicKeys) txError, err := b.invokeMetaTransaction( b.ctx, @@ -411,6 +460,7 @@ func (b *bootstrapExecutor) deployFlowToken(service, fungibleToken flow.Address) blueprints.DeployFlowTokenContractTransaction( service, fungibleToken, + metadataViews, flowToken), 0), ) @@ -598,7 +648,10 @@ func (b *bootstrapExecutor) setupParameters( panicOnMetaInvokeErrf("failed to setup parameters: %s", txError, err) } -func (b *bootstrapExecutor) setupFees(service, flowFees flow.Address, surgeFactor, inclusionEffortCost, executionEffortCost cadence.UFix64) { +func (b *bootstrapExecutor) setupFees( + service, flowFees flow.Address, + surgeFactor, inclusionEffortCost, executionEffortCost cadence.UFix64, +) { txError, err := b.invokeMetaTransaction( b.ctx, Transaction( @@ -704,7 +757,10 @@ func (b *bootstrapExecutor) setupStorageForServiceAccounts( panicOnMetaInvokeErrf("failed to setup storage for service accounts: %s", txError, err) } -func (b *bootstrapExecutor) setStakingAllowlist(service flow.Address, allowedIDs []flow.Identifier) { +func (b *bootstrapExecutor) setStakingAllowlist( + service flow.Address, + allowedIDs []flow.Identifier, +) { txError, err := b.invokeMetaTransaction( b.ctx, @@ -774,8 +830,25 @@ func (b *bootstrapExecutor) deployStakingProxyContract(service flow.Address) { panicOnMetaInvokeErrf("failed to deploy StakingProxy contract: %s", txError, err) } -func (b *bootstrapExecutor) deployLockedTokensContract(service flow.Address, fungibleTokenAddress, - flowTokenAddress flow.Address) { +func (b *bootstrapExecutor) deployVersionBeacon( + service flow.Address, + versionFreezePeriod cadence.UInt64, +) { + tx := blueprints.DeployNodeVersionBeaconTransaction(service, versionFreezePeriod) + txError, err := b.invokeMetaTransaction( + b.ctx, + Transaction( + tx, + 0, + ), + ) + panicOnMetaInvokeErrf("failed to deploy NodeVersionBeacon contract: %s", txError, err) +} + +func (b *bootstrapExecutor) deployLockedTokensContract( + service flow.Address, fungibleTokenAddress, + flowTokenAddress flow.Address, +) { publicKeys, err := flow.EncodeRuntimeAccountPublicKeys(b.accountKeys.ServiceAccountPublicKeys) if err != nil { @@ -800,7 +873,10 @@ func (b *bootstrapExecutor) deployLockedTokensContract(service flow.Address, fun panicOnMetaInvokeErrf("failed to deploy LockedTokens contract: %s", txError, err) } -func (b *bootstrapExecutor) deployStakingCollection(service flow.Address, fungibleTokenAddress, flowTokenAddress flow.Address) { +func (b *bootstrapExecutor) deployStakingCollection( + service flow.Address, + fungibleTokenAddress, flowTokenAddress flow.Address, +) { contract := contracts.FlowStakingCollection( fungibleTokenAddress.Hex(), flowTokenAddress.Hex(), @@ -821,7 +897,10 @@ func (b *bootstrapExecutor) deployStakingCollection(service flow.Address, fungib panicOnMetaInvokeErrf("failed to deploy FlowStakingCollection contract: %s", txError, err) } -func (b *bootstrapExecutor) setContractDeploymentRestrictions(service flow.Address, deployment *bool) { +func (b *bootstrapExecutor) setContractDeploymentRestrictions( + service flow.Address, + deployment *bool, +) { if deployment == nil { return } @@ -884,21 +963,8 @@ func (b *bootstrapExecutor) invokeMetaTransaction( WithComputationLimit(math.MaxUint64), ) - // use new derived transaction data for each meta transaction. - // It's not necessary to cache during bootstrapping and most transactions are contract deploys anyway. - prog, err := derived.NewEmptyDerivedBlockData(). - NewDerivedTransactionData(0, 0) - - if err != nil { - return nil, err - } - - txn := &storage.SerialTransaction{ - NestedTransaction: b.txnState, - DerivedTransactionCommitter: prog, - } - - err = Run(tx.NewExecutor(ctx, txn)) + executor := tx.NewExecutor(ctx, b.txnState) + err := Run(executor) - return tx.Err, err + return executor.Output().Err, err } diff --git a/fvm/context.go b/fvm/context.go index 1fc464cd68e..250955d2082 100644 --- a/fvm/context.go +++ b/fvm/context.go @@ -8,8 +8,8 @@ import ( "github.com/onflow/flow-go/fvm/environment" reusableRuntime "github.com/onflow/flow-go/fvm/runtime" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -160,10 +160,19 @@ func WithEventCollectionSizeLimit(limit uint64) Option { } } +// WithEntropyProvider sets the entropy provider of a virtual machine context. +// +// The VM uses the input to provide entropy to the Cadence runtime randomness functions. +func WithEntropyProvider(source environment.EntropyProvider) Option { + return func(ctx Context) Context { + ctx.EntropyProvider = source + return ctx + } +} + // WithBlockHeader sets the block header for a virtual machine context. // -// The VM uses the header to provide current block information to the Cadence runtime, -// as well as to seed the pseudorandom number generator. +// The VM uses the header to provide current block information to the Cadence runtime. func WithBlockHeader(header *flow.Header) Option { return func(ctx Context) Context { ctx.BlockHeader = header diff --git a/fvm/crypto/hash.go b/fvm/crypto/hash.go index 49d8dd0000b..d1e16c5e2fa 100644 --- a/fvm/crypto/hash.go +++ b/fvm/crypto/hash.go @@ -14,7 +14,7 @@ const tagLength = flow.DomainTagLength // prefixedHashing embeds a crypto hasher and implements // hashing with a prefix : prefixedHashing(data) = hasher(prefix || data) // -// Prefixes are padded tags till 32 bytes to guarantee prefixedHashers are independant +// Prefixes are padded tags till 32 bytes to guarantee prefixedHashers are independent // hashers. // Prefixes are disabled with the particular tag value "" type prefixedHashing struct { diff --git a/fvm/crypto/hash_test.go b/fvm/crypto/hash_test.go index bb9bb64172b..58d15d19b17 100644 --- a/fvm/crypto/hash_test.go +++ b/fvm/crypto/hash_test.go @@ -1,7 +1,7 @@ package crypto_test import ( - "math/rand" + "crypto/rand" "testing" "crypto/sha256" diff --git a/fvm/environment/account_creator.go b/fvm/environment/account_creator.go index a7a0f09294a..07612384d2c 100644 --- a/fvm/environment/account_creator.go +++ b/fvm/environment/account_creator.go @@ -6,7 +6,7 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" @@ -37,12 +37,12 @@ type BootstrapAccountCreator interface { // This ensures cadence can't access unexpected operations while parsing // programs. type ParseRestrictedAccountCreator struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer impl AccountCreator } func NewParseRestrictedAccountCreator( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, creator AccountCreator, ) AccountCreator { return ParseRestrictedAccountCreator{ @@ -88,7 +88,7 @@ func (NoAccountCreator) CreateAccount( // updates the state when next address is called (This secondary functionality // is only used in utility command line). type accountCreator struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer chain flow.Chain accounts Accounts @@ -102,7 +102,7 @@ type accountCreator struct { } func NewAddressGenerator( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, chain flow.Chain, ) AddressGenerator { return &accountCreator{ @@ -112,7 +112,7 @@ func NewAddressGenerator( } func NewBootstrapAccountCreator( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, chain flow.Chain, accounts Accounts, ) BootstrapAccountCreator { @@ -124,7 +124,7 @@ func NewBootstrapAccountCreator( } func NewAccountCreator( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, chain flow.Chain, accounts Accounts, isServiceAccountEnabled bool, diff --git a/fvm/environment/account_creator_test.go b/fvm/environment/account_creator_test.go index 086640d4ed6..b45fef018fa 100644 --- a/fvm/environment/account_creator_test.go +++ b/fvm/environment/account_creator_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/fvm/storage/testutils" "github.com/onflow/flow-go/model/flow" ) @@ -34,7 +34,7 @@ func Test_NewAccountCreator_GeneratingUpdatesState(t *testing.T) { func Test_NewAccountCreator_UsesLedgerState(t *testing.T) { chain := flow.MonotonicEmulator.Chain() txnState := testutils.NewSimpleTransaction( - state.MapStorageSnapshot{ + snapshot.MapStorageSnapshot{ flow.AddressStateRegisterID: flow.HexToAddress("01").Bytes(), }) creator := environment.NewAddressGenerator(txnState, chain) diff --git a/fvm/environment/account_info.go b/fvm/environment/account_info.go index 209239f120d..6af26a1940b 100644 --- a/fvm/environment/account_info.go +++ b/fvm/environment/account_info.go @@ -6,7 +6,7 @@ import ( "github.com/onflow/cadence" "github.com/onflow/cadence/runtime/common" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" @@ -24,12 +24,12 @@ type AccountInfo interface { } type ParseRestrictedAccountInfo struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer impl AccountInfo } func NewParseRestrictedAccountInfo( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, impl AccountInfo, ) AccountInfo { return ParseRestrictedAccountInfo{ diff --git a/fvm/environment/account_key_reader.go b/fvm/environment/account_key_reader.go index dc1eb73ff39..82ee3333cdf 100644 --- a/fvm/environment/account_key_reader.go +++ b/fvm/environment/account_key_reader.go @@ -8,7 +8,7 @@ import ( "github.com/onflow/flow-go/fvm/crypto" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" @@ -32,12 +32,12 @@ type AccountKeyReader interface { } type ParseRestrictedAccountKeyReader struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer impl AccountKeyReader } func NewParseRestrictedAccountKeyReader( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, impl AccountKeyReader, ) AccountKeyReader { return ParseRestrictedAccountKeyReader{ diff --git a/fvm/environment/account_key_updater.go b/fvm/environment/account_key_updater.go index 8cc48f4a962..96c601cb1aa 100644 --- a/fvm/environment/account_key_updater.go +++ b/fvm/environment/account_key_updater.go @@ -12,7 +12,7 @@ import ( fghash "github.com/onflow/flow-go/crypto/hash" "github.com/onflow/flow-go/fvm/crypto" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" @@ -138,12 +138,12 @@ type AccountKeyUpdater interface { } type ParseRestrictedAccountKeyUpdater struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer impl AccountKeyUpdater } func NewParseRestrictedAccountKeyUpdater( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, impl AccountKeyUpdater, ) ParseRestrictedAccountKeyUpdater { return ParseRestrictedAccountKeyUpdater{ @@ -259,7 +259,7 @@ type accountKeyUpdater struct { meter Meter accounts Accounts - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer env Environment } @@ -267,7 +267,7 @@ func NewAccountKeyUpdater( tracer tracing.TracerSpan, meter Meter, accounts Accounts, - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, env Environment, ) *accountKeyUpdater { return &accountKeyUpdater{ diff --git a/fvm/environment/account_key_updater_test.go b/fvm/environment/account_key_updater_test.go index 24c2404b917..61bb4c00d7d 100644 --- a/fvm/environment/account_key_updater_test.go +++ b/fvm/environment/account_key_updater_test.go @@ -220,3 +220,6 @@ func (f FakeAccounts) SetValue(_ flow.RegisterID, _ []byte) error { func (f FakeAccounts) AllocateStorageIndex(_ flow.Address) (atree.StorageIndex, error) { return atree.StorageIndex{}, nil } +func (f FakeAccounts) GenerateAccountLocalID(address flow.Address) (uint64, error) { + return 0, nil +} diff --git a/fvm/environment/account_local_id_generator.go b/fvm/environment/account_local_id_generator.go new file mode 100644 index 00000000000..9a1ba6a35c4 --- /dev/null +++ b/fvm/environment/account_local_id_generator.go @@ -0,0 +1,77 @@ +package environment + +import ( + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/fvm/storage/state" + "github.com/onflow/flow-go/fvm/tracing" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/trace" +) + +type AccountLocalIDGenerator interface { + GenerateAccountID(address common.Address) (uint64, error) +} + +type ParseRestrictedAccountLocalIDGenerator struct { + txnState state.NestedTransactionPreparer + impl AccountLocalIDGenerator +} + +func NewParseRestrictedAccountLocalIDGenerator( + txnState state.NestedTransactionPreparer, + impl AccountLocalIDGenerator, +) AccountLocalIDGenerator { + return ParseRestrictedAccountLocalIDGenerator{ + txnState: txnState, + impl: impl, + } +} + +func (generator ParseRestrictedAccountLocalIDGenerator) GenerateAccountID( + address common.Address, +) (uint64, error) { + return parseRestrict1Arg1Ret( + generator.txnState, + trace.FVMEnvGenerateAccountLocalID, + generator.impl.GenerateAccountID, + address) +} + +type accountLocalIDGenerator struct { + tracer tracing.TracerSpan + meter Meter + accounts Accounts +} + +func NewAccountLocalIDGenerator( + tracer tracing.TracerSpan, + meter Meter, + accounts Accounts, +) AccountLocalIDGenerator { + return &accountLocalIDGenerator{ + tracer: tracer, + meter: meter, + accounts: accounts, + } +} + +func (generator *accountLocalIDGenerator) GenerateAccountID( + runtimeAddress common.Address, +) ( + uint64, + error, +) { + defer generator.tracer.StartExtensiveTracingChildSpan( + trace.FVMEnvGenerateAccountLocalID, + ).End() + + err := generator.meter.MeterComputation(ComputationKindGenerateAccountLocalID, 1) + if err != nil { + return 0, err + } + + return generator.accounts.GenerateAccountLocalID( + flow.ConvertAddress(runtimeAddress), + ) +} diff --git a/fvm/environment/account_local_id_generator_test.go b/fvm/environment/account_local_id_generator_test.go new file mode 100644 index 00000000000..0a2c229226e --- /dev/null +++ b/fvm/environment/account_local_id_generator_test.go @@ -0,0 +1,87 @@ +package environment_test + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/fvm/environment" + envMock "github.com/onflow/flow-go/fvm/environment/mock" + "github.com/onflow/flow-go/fvm/tracing" + "github.com/onflow/flow-go/model/flow" +) + +func Test_accountLocalIDGenerator_GenerateAccountID(t *testing.T) { + address, err := common.HexToAddress("0x1") + require.NoError(t, err) + + t.Run("success", func(t *testing.T) { + meter := envMock.NewMeter(t) + meter.On( + "MeterComputation", + common.ComputationKind(environment.ComputationKindGenerateAccountLocalID), + uint(1), + ).Return(nil) + + accounts := envMock.NewAccounts(t) + accounts.On("GenerateAccountLocalID", flow.ConvertAddress(address)). + Return(uint64(1), nil) + + generator := environment.NewAccountLocalIDGenerator( + tracing.NewMockTracerSpan(), + meter, + accounts, + ) + + id, err := generator.GenerateAccountID(address) + require.NoError(t, err) + require.Equal(t, uint64(1), id) + }) + t.Run("error in meter", func(t *testing.T) { + expectedErr := errors.New("error in meter") + + meter := envMock.NewMeter(t) + meter.On( + "MeterComputation", + common.ComputationKind(environment.ComputationKindGenerateAccountLocalID), + uint(1), + ).Return(expectedErr) + + accounts := envMock.NewAccounts(t) + + generator := environment.NewAccountLocalIDGenerator( + tracing.NewMockTracerSpan(), + meter, + accounts, + ) + + _, err := generator.GenerateAccountID(address) + require.ErrorIs(t, err, expectedErr) + }) + t.Run("err in accounts", func(t *testing.T) { + expectedErr := errors.New("error in accounts") + + meter := envMock.NewMeter(t) + meter.On( + "MeterComputation", + common.ComputationKind(environment.ComputationKindGenerateAccountLocalID), + uint(1), + ).Return(nil) + + accounts := envMock.NewAccounts(t) + accounts.On("GenerateAccountLocalID", flow.ConvertAddress(address)). + Return(uint64(0), expectedErr) + + generator := environment.NewAccountLocalIDGenerator( + tracing.NewMockTracerSpan(), + meter, + accounts, + ) + + _, err := generator.GenerateAccountID(address) + require.ErrorIs(t, err, expectedErr) + }) +} diff --git a/fvm/environment/accounts.go b/fvm/environment/accounts.go index 3879aa71e5e..6af7c24aed5 100644 --- a/fvm/environment/accounts.go +++ b/fvm/environment/accounts.go @@ -12,7 +12,7 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/crypto/hash" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/model/flow" ) @@ -37,15 +37,16 @@ type Accounts interface { GetStorageUsed(address flow.Address) (uint64, error) SetValue(id flow.RegisterID, value flow.RegisterValue) error AllocateStorageIndex(address flow.Address) (atree.StorageIndex, error) + GenerateAccountLocalID(address flow.Address) (uint64, error) } var _ Accounts = &StatefulAccounts{} type StatefulAccounts struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer } -func NewAccounts(txnState state.NestedTransaction) *StatefulAccounts { +func NewAccounts(txnState state.NestedTransactionPreparer) *StatefulAccounts { return &StatefulAccounts{ txnState: txnState, } @@ -698,6 +699,32 @@ func (a *StatefulAccounts) DeleteContract( return a.setContractNames(contractNames, address) } +// GenerateAccountLocalID generates a new account local id for an address +// it is sequential and starts at 1 +// Errors can happen if the account state cannot be read or written to +func (a *StatefulAccounts) GenerateAccountLocalID( + address flow.Address, +) ( + uint64, + error, +) { + as, err := a.getAccountStatus(address) + if err != nil { + return 0, fmt.Errorf("failed to get account local id: %w", err) + } + id := as.AccountIdCounter() + // AccountLocalIDs are defined as non 0 so return the incremented value + // see: https://github.com/onflow/cadence/blob/2081a601106baaf6ae695e3f2a84613160bb2166/runtime/interface.go#L149 + id += 1 + + as.SetAccountIdCounter(id) + err = a.setAccountStatus(address, as) + if err != nil { + return 0, fmt.Errorf("failed to get increment account local id: %w", err) + } + return id, nil +} + func (a *StatefulAccounts) getAccountStatus( address flow.Address, ) ( diff --git a/fvm/environment/accounts_status.go b/fvm/environment/accounts_status.go index c715c80e89e..a420051550f 100644 --- a/fvm/environment/accounts_status.go +++ b/fvm/environment/accounts_status.go @@ -10,20 +10,31 @@ import ( ) const ( - flagSize = 1 - storageUsedSize = 8 - storageIndexSize = 8 - publicKeyCountsSize = 8 - - accountStatusSize = flagSize + + flagSize = 1 + storageUsedSize = 8 + storageIndexSize = 8 + publicKeyCountsSize = 8 + addressIdCounterSize = 8 + + // oldAccountStatusSize is the size of the account status before the address + // id counter was added. After v0.32.0 check if it can be removed as all accounts + // should then have the new status sile len. + oldAccountStatusSize = flagSize + storageUsedSize + storageIndexSize + publicKeyCountsSize - flagIndex = 0 - storageUsedStartIndex = flagIndex + flagSize - storageIndexStartIndex = storageUsedStartIndex + storageUsedSize - publicKeyCountsStartIndex = storageIndexStartIndex + storageIndexSize + accountStatusSize = flagSize + + storageUsedSize + + storageIndexSize + + publicKeyCountsSize + + addressIdCounterSize + + flagIndex = 0 + storageUsedStartIndex = flagIndex + flagSize + storageIndexStartIndex = storageUsedStartIndex + storageUsedSize + publicKeyCountsStartIndex = storageIndexStartIndex + storageIndexSize + addressIdCounterStartIndex = publicKeyCountsStartIndex + publicKeyCountsSize ) // AccountStatus holds meta information about an account @@ -32,7 +43,8 @@ const ( // the first byte captures flags // the next 8 bytes (big-endian) captures storage used by an account // the next 8 bytes (big-endian) captures the storage index of an account -// and the last 8 bytes (big-endian) captures the number of public keys stored on this account +// the next 8 bytes (big-endian) captures the number of public keys stored on this account +// the next 8 bytes (big-endian) captures the current address id counter type AccountStatus [accountStatusSize]byte // NewAccountStatus returns a new AccountStatus @@ -43,13 +55,14 @@ func NewAccountStatus() *AccountStatus { 0, 0, 0, 0, 0, 0, 0, 0, // init value for storage used 0, 0, 0, 0, 0, 0, 0, 1, // init value for storage index 0, 0, 0, 0, 0, 0, 0, 0, // init value for public key counts + 0, 0, 0, 0, 0, 0, 0, 0, // init value for address id counter } } // ToBytes converts AccountStatus to a byte slice // // this has been kept this way in case one day -// we decided to move on to use an struct to represent +// we decided to move on to use a struct to represent // account status. func (a *AccountStatus) ToBytes() []byte { return a[:] @@ -58,6 +71,22 @@ func (a *AccountStatus) ToBytes() []byte { // AccountStatusFromBytes constructs an AccountStatus from the given byte slice func AccountStatusFromBytes(inp []byte) (*AccountStatus, error) { var as AccountStatus + + if len(inp) == oldAccountStatusSize { + // pad the input with zeros + // this is to migrate old account status to new account status on the fly + // TODO: remove this whole block after v0.32.0, when a full migration will + // be made. + sizeIncrease := uint64(accountStatusSize - oldAccountStatusSize) + + // But we also need to fix the storage used by the appropriate size because + // the storage used is part of the account status itself. + copy(as[:], inp) + used := as.StorageUsed() + as.SetStorageUsed(used + sizeIncrease) + return &as, nil + } + if len(inp) != accountStatusSize { return &as, errors.NewValueErrorf(hex.EncodeToString(inp), "invalid account status size") } @@ -96,3 +125,13 @@ func (a *AccountStatus) SetPublicKeyCount(count uint64) { func (a *AccountStatus) PublicKeyCount() uint64 { return binary.BigEndian.Uint64(a[publicKeyCountsStartIndex : publicKeyCountsStartIndex+publicKeyCountsSize]) } + +// SetAccountIdCounter updates id counter of the account +func (a *AccountStatus) SetAccountIdCounter(id uint64) { + binary.BigEndian.PutUint64(a[addressIdCounterStartIndex:addressIdCounterStartIndex+addressIdCounterSize], id) +} + +// AccountIdCounter returns id counter of the account +func (a *AccountStatus) AccountIdCounter() uint64 { + return binary.BigEndian.Uint64(a[addressIdCounterStartIndex : addressIdCounterStartIndex+addressIdCounterSize]) +} diff --git a/fvm/environment/accounts_status_test.go b/fvm/environment/accounts_status_test.go index 5d7a04ddff1..543ee2b05f1 100644 --- a/fvm/environment/accounts_status_test.go +++ b/fvm/environment/accounts_status_test.go @@ -19,11 +19,13 @@ func TestAccountStatus(t *testing.T) { s.SetStorageIndex(index) s.SetPublicKeyCount(34) s.SetStorageUsed(56) + s.SetAccountIdCounter(78) require.Equal(t, uint64(56), s.StorageUsed()) returnedIndex := s.StorageIndex() require.True(t, bytes.Equal(index[:], returnedIndex[:])) require.Equal(t, uint64(34), s.PublicKeyCount()) + require.Equal(t, uint64(78), s.AccountIdCounter()) }) @@ -34,9 +36,31 @@ func TestAccountStatus(t *testing.T) { require.Equal(t, s.StorageIndex(), clone.StorageIndex()) require.Equal(t, s.PublicKeyCount(), clone.PublicKeyCount()) require.Equal(t, s.StorageUsed(), clone.StorageUsed()) + require.Equal(t, s.AccountIdCounter(), clone.AccountIdCounter()) // invalid size bytes _, err = environment.AccountStatusFromBytes([]byte{1, 2}) require.Error(t, err) }) + + t.Run("test serialization - old format", func(t *testing.T) { + // TODO: remove this test when we remove support for the old format + oldBytes := []byte{ + 0, // flags + 0, 0, 0, 0, 0, 0, 0, 7, // storage used + 0, 0, 0, 0, 0, 0, 0, 6, // storage index + 0, 0, 0, 0, 0, 0, 0, 5, // public key counts + } + + // The new format has an extra 8 bytes for the account id counter + // so we need to increase the storage used by 8 bytes while migrating it + increaseInSize := uint64(8) + + migrated, err := environment.AccountStatusFromBytes(oldBytes) + require.NoError(t, err) + require.Equal(t, atree.StorageIndex{0, 0, 0, 0, 0, 0, 0, 6}, migrated.StorageIndex()) + require.Equal(t, uint64(5), migrated.PublicKeyCount()) + require.Equal(t, uint64(7)+increaseInSize, migrated.StorageUsed()) + require.Equal(t, uint64(0), migrated.AccountIdCounter()) + }) } diff --git a/fvm/environment/accounts_test.go b/fvm/environment/accounts_test.go index 7b29dbb125b..7c4302278f2 100644 --- a/fvm/environment/accounts_test.go +++ b/fvm/environment/accounts_test.go @@ -8,7 +8,7 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/fvm/storage/testutils" "github.com/onflow/flow-go/model/flow" ) @@ -68,7 +68,7 @@ func TestAccounts_GetPublicKey(t *testing.T) { for _, value := range [][]byte{{}, nil} { txnState := testutils.NewSimpleTransaction( - state.MapStorageSnapshot{ + snapshot.MapStorageSnapshot{ registerId: value, }) accounts := environment.NewAccounts(txnState) @@ -93,7 +93,7 @@ func TestAccounts_GetPublicKeyCount(t *testing.T) { for _, value := range [][]byte{{}, nil} { txnState := testutils.NewSimpleTransaction( - state.MapStorageSnapshot{ + snapshot.MapStorageSnapshot{ registerId: value, }) accounts := environment.NewAccounts(txnState) @@ -119,7 +119,7 @@ func TestAccounts_GetPublicKeys(t *testing.T) { for _, value := range [][]byte{{}, nil} { txnState := testutils.NewSimpleTransaction( - state.MapStorageSnapshot{ + snapshot.MapStorageSnapshot{ registerId: value, }) @@ -224,6 +224,7 @@ func TestAccounts_SetContracts(t *testing.T) { } func TestAccount_StorageUsed(t *testing.T) { + emptyAccountSize := uint64(48) t.Run("Storage used on account creation is deterministic", func(t *testing.T) { txnState := testutils.NewSimpleTransaction(nil) @@ -235,7 +236,7 @@ func TestAccount_StorageUsed(t *testing.T) { storageUsed, err := accounts.GetStorageUsed(address) require.NoError(t, err) - require.Equal(t, uint64(40), storageUsed) + require.Equal(t, emptyAccountSize, storageUsed) }) t.Run("Storage used on register set increases", func(t *testing.T) { @@ -252,7 +253,7 @@ func TestAccount_StorageUsed(t *testing.T) { storageUsed, err := accounts.GetStorageUsed(address) require.NoError(t, err) - require.Equal(t, uint64(40+32), storageUsed) + require.Equal(t, emptyAccountSize+uint64(32), storageUsed) }) t.Run("Storage used, set twice on same register to same value, stays the same", func(t *testing.T) { @@ -271,7 +272,7 @@ func TestAccount_StorageUsed(t *testing.T) { storageUsed, err := accounts.GetStorageUsed(address) require.NoError(t, err) - require.Equal(t, uint64(40+32), storageUsed) + require.Equal(t, emptyAccountSize+uint64(32), storageUsed) }) t.Run("Storage used, set twice on same register to larger value, increases", func(t *testing.T) { @@ -290,7 +291,7 @@ func TestAccount_StorageUsed(t *testing.T) { storageUsed, err := accounts.GetStorageUsed(address) require.NoError(t, err) - require.Equal(t, uint64(40+33), storageUsed) + require.Equal(t, emptyAccountSize+uint64(33), storageUsed) }) t.Run("Storage used, set twice on same register to smaller value, decreases", func(t *testing.T) { @@ -309,7 +310,7 @@ func TestAccount_StorageUsed(t *testing.T) { storageUsed, err := accounts.GetStorageUsed(address) require.NoError(t, err) - require.Equal(t, uint64(40+31), storageUsed) + require.Equal(t, emptyAccountSize+uint64(31), storageUsed) }) t.Run("Storage used, after register deleted, decreases", func(t *testing.T) { @@ -328,7 +329,7 @@ func TestAccount_StorageUsed(t *testing.T) { storageUsed, err := accounts.GetStorageUsed(address) require.NoError(t, err) - require.Equal(t, uint64(40+0), storageUsed) + require.Equal(t, emptyAccountSize+uint64(0), storageUsed) }) t.Run("Storage used on a complex scenario has correct value", func(t *testing.T) { @@ -359,10 +360,51 @@ func TestAccount_StorageUsed(t *testing.T) { storageUsed, err := accounts.GetStorageUsed(address) require.NoError(t, err) - require.Equal(t, uint64(40+33+42), storageUsed) + require.Equal(t, emptyAccountSize+uint64(33+42), storageUsed) }) } +func TestStatefulAccounts_GenerateAccountLocalID(t *testing.T) { + + // Create 3 accounts + addressA := flow.HexToAddress("0x01") + addressB := flow.HexToAddress("0x02") + addressC := flow.HexToAddress("0x03") + txnState := testutils.NewSimpleTransaction(nil) + a := environment.NewAccounts(txnState) + err := a.Create(nil, addressA) + require.NoError(t, err) + err = a.Create(nil, addressB) + require.NoError(t, err) + err = a.Create(nil, addressC) + require.NoError(t, err) + + // setup some state + _, err = a.GenerateAccountLocalID(addressA) + require.NoError(t, err) + _, err = a.GenerateAccountLocalID(addressA) + require.NoError(t, err) + _, err = a.GenerateAccountLocalID(addressB) + require.NoError(t, err) + + // assert + + // addressA + id, err := a.GenerateAccountLocalID(addressA) + require.NoError(t, err) + require.Equal(t, uint64(3), id) + + // addressB + id, err = a.GenerateAccountLocalID(addressB) + require.NoError(t, err) + require.Equal(t, uint64(2), id) + + // addressC + id, err = a.GenerateAccountLocalID(addressC) + require.NoError(t, err) + require.Equal(t, uint64(1), id) +} + func createByteArray(size int) []byte { bytes := make([]byte, size) for i := range bytes { diff --git a/fvm/environment/block_info.go b/fvm/environment/block_info.go index eddcc542185..9e55a67c649 100644 --- a/fvm/environment/block_info.go +++ b/fvm/environment/block_info.go @@ -6,11 +6,11 @@ import ( "github.com/onflow/cadence/runtime" "github.com/onflow/flow-go/fvm/errors" - storageTxn "github.com/onflow/flow-go/fvm/storage" + "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" - "github.com/onflow/flow-go/storage" + storageErr "github.com/onflow/flow-go/storage" ) type BlockInfo interface { @@ -28,12 +28,12 @@ type BlockInfo interface { } type ParseRestrictedBlockInfo struct { - txnState storageTxn.Transaction + txnState storage.TransactionPreparer impl BlockInfo } func NewParseRestrictedBlockInfo( - txnState storageTxn.Transaction, + txnState storage.TransactionPreparer, impl BlockInfo, ) BlockInfo { return ParseRestrictedBlockInfo{ @@ -145,7 +145,7 @@ func (info *blockInfo) GetBlockAtHeight( header, err := info.blocks.ByHeightFrom(height, info.blockHeader) // TODO (ramtin): remove dependency on storage and move this if condition // to blockfinder - if errors.Is(err, storage.ErrNotFound) { + if errors.Is(err, storageErr.ErrNotFound) { return runtime.Block{}, false, nil } else if err != nil { return runtime.Block{}, false, fmt.Errorf( diff --git a/fvm/environment/contract_updater.go b/fvm/environment/contract_updater.go index 8bc8f6026be..2185b4d09da 100644 --- a/fvm/environment/contract_updater.go +++ b/fvm/environment/contract_updater.go @@ -10,7 +10,7 @@ import ( "github.com/onflow/flow-go/fvm/blueprints" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" @@ -80,12 +80,12 @@ type ContractUpdater interface { } type ParseRestrictedContractUpdater struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer impl ContractUpdater } func NewParseRestrictedContractUpdater( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, impl ContractUpdater, ) ParseRestrictedContractUpdater { return ParseRestrictedContractUpdater{ diff --git a/fvm/environment/crypto_library.go b/fvm/environment/crypto_library.go index 5333630254b..cbb2d24e1f5 100644 --- a/fvm/environment/crypto_library.go +++ b/fvm/environment/crypto_library.go @@ -6,7 +6,7 @@ import ( "github.com/onflow/cadence/runtime" "github.com/onflow/flow-go/fvm/crypto" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/module/trace" ) @@ -54,12 +54,12 @@ type CryptoLibrary interface { } type ParseRestrictedCryptoLibrary struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer impl CryptoLibrary } func NewParseRestrictedCryptoLibrary( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, impl CryptoLibrary, ) CryptoLibrary { return ParseRestrictedCryptoLibrary{ diff --git a/fvm/environment/derived_data_invalidator.go b/fvm/environment/derived_data_invalidator.go index 7229c51ee73..5aa4bf05808 100644 --- a/fvm/environment/derived_data_invalidator.go +++ b/fvm/environment/derived_data_invalidator.go @@ -3,8 +3,8 @@ package environment import ( "github.com/onflow/cadence/runtime/common" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" ) @@ -31,11 +31,10 @@ type DerivedDataInvalidator struct { var _ derived.TransactionInvalidator = DerivedDataInvalidator{} -// TODO(patrick): extract contractKeys from executionSnapshot func NewDerivedDataInvalidator( contractUpdates ContractUpdates, serviceAddress flow.Address, - executionSnapshot *state.ExecutionSnapshot, + executionSnapshot *snapshot.ExecutionSnapshot, ) DerivedDataInvalidator { return DerivedDataInvalidator{ ContractUpdates: contractUpdates, @@ -47,7 +46,7 @@ func NewDerivedDataInvalidator( func meterParamOverridesUpdated( serviceAddress flow.Address, - executionSnapshot *state.ExecutionSnapshot, + executionSnapshot *snapshot.ExecutionSnapshot, ) bool { serviceAccount := string(serviceAddress.Bytes()) storageDomain := common.PathDomainStorage.Identifier() @@ -98,7 +97,7 @@ func (invalidator ProgramInvalidator) ShouldInvalidateEntries() bool { func (invalidator ProgramInvalidator) ShouldInvalidateEntry( location common.AddressLocation, program *derived.Program, - snapshot *state.ExecutionSnapshot, + snapshot *snapshot.ExecutionSnapshot, ) bool { if invalidator.MeterParamOverridesUpdated { // if meter parameters changed we need to invalidate all programs @@ -144,7 +143,7 @@ func (invalidator MeterParamOverridesInvalidator) ShouldInvalidateEntries() bool func (invalidator MeterParamOverridesInvalidator) ShouldInvalidateEntry( _ struct{}, _ derived.MeterParamOverrides, - _ *state.ExecutionSnapshot, + _ *snapshot.ExecutionSnapshot, ) bool { return invalidator.MeterParamOverridesUpdated } diff --git a/fvm/environment/derived_data_invalidator_test.go b/fvm/environment/derived_data_invalidator_test.go index dde9ffc93b0..aa86aaeb258 100644 --- a/fvm/environment/derived_data_invalidator_test.go +++ b/fvm/environment/derived_data_invalidator_test.go @@ -6,13 +6,13 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/meter" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -242,7 +242,7 @@ func TestMeterParamOverridesUpdated(t *testing.T) { memKind: memWeight, } - snapshotTree := storage.NewSnapshotTree(nil) + snapshotTree := snapshot.NewSnapshotTree(nil) ctx := fvm.NewContext(fvm.WithChain(flow.Testnet.Chain())) @@ -257,18 +257,14 @@ func TestMeterParamOverridesUpdated(t *testing.T) { snapshotTree) require.NoError(t, err) - nestedTxn := state.NewTransactionState( - delta.NewDeltaView(snapshotTree.Append(executionSnapshot)), - state.DefaultParameters()) + blockDatabase := storage.NewBlockDatabase( + snapshotTree.Append(executionSnapshot), + 0, + nil) - derivedBlockData := derived.NewEmptyDerivedBlockData() - derivedTxnData, err := derivedBlockData.NewDerivedTransactionData(0, 0) + txnState, err := blockDatabase.NewTransaction(0, state.DefaultParameters()) require.NoError(t, err) - txnState := storage.SerialTransaction{ - NestedTransaction: nestedTxn, - DerivedTransactionCommitter: derivedTxnData, - } computer := fvm.NewMeterParamOverridesComputer(ctx, txnState) overrides, err := computer.Compute(txnState, struct{}{}) @@ -288,7 +284,7 @@ func TestMeterParamOverridesUpdated(t *testing.T) { ctx.TxBody = &flow.TransactionBody{} checkForUpdates := func(id flow.RegisterID, expected bool) { - snapshot := &state.ExecutionSnapshot{ + snapshot := &snapshot.ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ id: flow.RegisterValue("blah"), }, diff --git a/fvm/environment/env.go b/fvm/environment/env.go index 886a82be701..518b49a737a 100644 --- a/fvm/environment/env.go +++ b/fvm/environment/env.go @@ -98,6 +98,8 @@ type EnvironmentParams struct { BlockInfoParams TransactionInfoParams + EntropyProvider + ContractUpdaterParams } diff --git a/fvm/environment/event_emitter.go b/fvm/environment/event_emitter.go index b7bdc1aded6..366c2d81d36 100644 --- a/fvm/environment/event_emitter.go +++ b/fvm/environment/event_emitter.go @@ -6,7 +6,7 @@ import ( "github.com/onflow/cadence" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/convert" @@ -50,12 +50,12 @@ type EventEmitter interface { } type ParseRestrictedEventEmitter struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer impl EventEmitter } func NewParseRestrictedEventEmitter( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, impl EventEmitter, ) EventEmitter { return ParseRestrictedEventEmitter{ @@ -197,6 +197,7 @@ func (emitter *eventEmitter) EmitEvent(event cadence.Event) error { payloadSize) // skip limit if payer is service account + // TODO skip only limit-related errors if !isServiceAccount && eventEmitError != nil { return eventEmitError } diff --git a/fvm/environment/event_emitter_test.go b/fvm/environment/event_emitter_test.go index 76eb5770492..dbbcfe743bb 100644 --- a/fvm/environment/event_emitter_test.go +++ b/fvm/environment/event_emitter_test.go @@ -7,14 +7,13 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/cadence" - jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/cadence/encoding/ccf" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/stdlib" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/meter" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" @@ -155,7 +154,7 @@ func Test_EmitEvent_Limit(t *testing.T) { func createTestEventEmitterWithLimit(chain flow.ChainID, address flow.Address, eventEmitLimit uint64) environment.EventEmitter { txnState := state.NewTransactionState( - delta.NewDeltaView(nil), + nil, state.DefaultParameters().WithMeterParameters( meter.DefaultParameters().WithEventEmitByteLimit(eventEmitLimit), )) @@ -180,7 +179,7 @@ func createTestEventEmitterWithLimit(chain flow.ChainID, address flow.Address, e } func getCadenceEventPayloadByteSize(event cadence.Event) uint64 { - payload, err := jsoncdc.Encode(event) + payload, err := ccf.Encode(event) if err != nil { panic(err) } diff --git a/fvm/environment/event_encoder.go b/fvm/environment/event_encoder.go index 33fdbe20c95..36b1f4bd2cd 100644 --- a/fvm/environment/event_encoder.go +++ b/fvm/environment/event_encoder.go @@ -2,7 +2,7 @@ package environment import ( "github.com/onflow/cadence" - jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/cadence/encoding/ccf" ) type EventEncoder interface { @@ -16,5 +16,5 @@ func NewCadenceEventEncoder() *CadenceEventEncoder { } func (e *CadenceEventEncoder) Encode(event cadence.Event) ([]byte, error) { - return jsoncdc.Encode(event) + return ccf.Encode(event) } diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index 6eb76a6a343..76ac5205725 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -6,11 +6,9 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" - "github.com/onflow/flow-go/engine/execution/state/delta" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage" - "github.com/onflow/flow-go/fvm/storage/derived" - "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" ) @@ -26,7 +24,7 @@ type facadeEnvironment struct { *ProgramLogger EventEmitter - UnsafeRandomGenerator + RandomGenerator CryptoLibrary BlockInfo @@ -38,6 +36,7 @@ type facadeEnvironment struct { *SystemContracts UUIDGenerator + AccountLocalIDGenerator AccountCreator @@ -49,13 +48,13 @@ type facadeEnvironment struct { *Programs accounts Accounts - txnState storage.Transaction + txnState storage.TransactionPreparer } func newFacadeEnvironment( tracer tracing.TracerSpan, params EnvironmentParams, - txnState storage.Transaction, + txnState storage.TransactionPreparer, meter Meter, ) *facadeEnvironment { accounts := NewAccounts(txnState) @@ -76,10 +75,6 @@ func newFacadeEnvironment( ProgramLogger: logger, EventEmitter: NoEventEmitter{}, - UnsafeRandomGenerator: NewUnsafeRandomGenerator( - tracer, - params.BlockHeader, - ), CryptoLibrary: NewCryptoLibrary(tracer, meter), BlockInfo: NewBlockInfo( @@ -107,8 +102,16 @@ func newFacadeEnvironment( UUIDGenerator: NewUUIDGenerator( tracer, + params.Logger, meter, - txnState), + txnState, + params.BlockHeader, + params.TxIndex), + AccountLocalIDGenerator: NewAccountLocalIDGenerator( + tracer, + meter, + accounts, + ), AccountCreator: NoAccountCreator{}, @@ -145,51 +148,37 @@ func newFacadeEnvironment( // testing. func NewScriptEnvironmentFromStorageSnapshot( params EnvironmentParams, - storageSnapshot state.StorageSnapshot, + storageSnapshot snapshot.StorageSnapshot, ) *facadeEnvironment { - derivedBlockData := derived.NewEmptyDerivedBlockData() - derivedTxn, err := derivedBlockData.NewSnapshotReadDerivedTransactionData( - logical.EndOfBlockExecutionTime, - logical.EndOfBlockExecutionTime) - if err != nil { - panic(err) - } - - txn := storage.SerialTransaction{ - NestedTransaction: state.NewTransactionState( - delta.NewDeltaView(storageSnapshot), - state.DefaultParameters()), - DerivedTransactionCommitter: derivedTxn, - } + blockDatabase := storage.NewBlockDatabase(storageSnapshot, 0, nil) return NewScriptEnv( context.Background(), tracing.NewTracerSpan(), params, - txn) + blockDatabase.NewSnapshotReadTransaction(state.DefaultParameters())) } func NewScriptEnv( ctx context.Context, tracer tracing.TracerSpan, params EnvironmentParams, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) *facadeEnvironment { env := newFacadeEnvironment( tracer, params, txnState, NewCancellableMeter(ctx, txnState)) - + env.RandomGenerator = NewDummyRandomGenerator() env.addParseRestrictedChecks() - return env } func NewTransactionEnvironment( tracer tracing.TracerSpan, params EnvironmentParams, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) *facadeEnvironment { env := newFacadeEnvironment( tracer, @@ -237,6 +226,12 @@ func NewTransactionEnvironment( txnState, env) + env.RandomGenerator = NewRandomGenerator( + tracer, + params.EntropyProvider, + params.TxId, + ) + env.addParseRestrictedChecks() return env @@ -277,12 +272,15 @@ func (env *facadeEnvironment) addParseRestrictedChecks() { env.TransactionInfo = NewParseRestrictedTransactionInfo( env.txnState, env.TransactionInfo) - env.UnsafeRandomGenerator = NewParseRestrictedUnsafeRandomGenerator( + env.RandomGenerator = NewParseRestrictedRandomGenerator( env.txnState, - env.UnsafeRandomGenerator) + env.RandomGenerator) env.UUIDGenerator = NewParseRestrictedUUIDGenerator( env.txnState, env.UUIDGenerator) + env.AccountLocalIDGenerator = NewParseRestrictedAccountLocalIDGenerator( + env.txnState, + env.AccountLocalIDGenerator) env.ValueStore = NewParseRestrictedValueStore( env.txnState, env.ValueStore) diff --git a/fvm/environment/generate-wrappers/main.go b/fvm/environment/generate-wrappers/main.go index f7a88676962..53d8cd1ea8b 100644 --- a/fvm/environment/generate-wrappers/main.go +++ b/fvm/environment/generate-wrappers/main.go @@ -15,12 +15,12 @@ package environment import ( "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/module/trace" ) func parseRestricted( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, ) error { if txnState.IsParseRestricted() { @@ -84,7 +84,7 @@ func generateWrapper(numArgs int, numRets int, content *FileContent) { l("](") push() - l("txnState state.NestedTransaction,") + l("txnState state.NestedTransactionPreparer,") l("spanName trace.SpanName,") callbackRet := "error" diff --git a/fvm/environment/meter.go b/fvm/environment/meter.go index 806399aa7a9..895fb2f9151 100644 --- a/fvm/environment/meter.go +++ b/fvm/environment/meter.go @@ -7,7 +7,7 @@ import ( "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/meter" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" ) const ( @@ -46,6 +46,7 @@ const ( ComputationKindBLSAggregateSignatures = 2032 ComputationKindBLSAggregatePublicKeys = 2033 ComputationKindGetOrLoadProgram = 2034 + ComputationKindGenerateAccountLocalID = 2035 ) type Meter interface { @@ -63,10 +64,10 @@ type Meter interface { } type meterImpl struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer } -func NewMeter(txnState state.NestedTransaction) Meter { +func NewMeter(txnState state.NestedTransactionPreparer) Meter { return &meterImpl{ txnState: txnState, } @@ -115,7 +116,7 @@ type cancellableMeter struct { func NewCancellableMeter( ctx context.Context, - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, ) Meter { return &cancellableMeter{ meterImpl: meterImpl{ diff --git a/fvm/environment/mock/account_local_id_generator.go b/fvm/environment/mock/account_local_id_generator.go new file mode 100644 index 00000000000..f6bfac27edb --- /dev/null +++ b/fvm/environment/mock/account_local_id_generator.go @@ -0,0 +1,53 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + common "github.com/onflow/cadence/runtime/common" + + mock "github.com/stretchr/testify/mock" +) + +// AccountLocalIDGenerator is an autogenerated mock type for the AccountLocalIDGenerator type +type AccountLocalIDGenerator struct { + mock.Mock +} + +// GenerateAccountID provides a mock function with given fields: address +func (_m *AccountLocalIDGenerator) GenerateAccountID(address common.Address) (uint64, error) { + ret := _m.Called(address) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(common.Address) (uint64, error)); ok { + return rf(address) + } + if rf, ok := ret.Get(0).(func(common.Address) uint64); ok { + r0 = rf(address) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(common.Address) error); ok { + r1 = rf(address) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewAccountLocalIDGenerator interface { + mock.TestingT + Cleanup(func()) +} + +// NewAccountLocalIDGenerator creates a new instance of AccountLocalIDGenerator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewAccountLocalIDGenerator(t mockConstructorTestingTNewAccountLocalIDGenerator) *AccountLocalIDGenerator { + mock := &AccountLocalIDGenerator{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/fvm/environment/mock/accounts.go b/fvm/environment/mock/accounts.go index 13a8dd34876..ee4656a4be8 100644 --- a/fvm/environment/mock/accounts.go +++ b/fvm/environment/mock/accounts.go @@ -131,6 +131,30 @@ func (_m *Accounts) Exists(address flow.Address) (bool, error) { return r0, r1 } +// GenerateAccountLocalID provides a mock function with given fields: address +func (_m *Accounts) GenerateAccountLocalID(address flow.Address) (uint64, error) { + ret := _m.Called(address) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(flow.Address) (uint64, error)); ok { + return rf(address) + } + if rf, ok := ret.Get(0).(func(flow.Address) uint64); ok { + r0 = rf(address) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(flow.Address) error); ok { + r1 = rf(address) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Get provides a mock function with given fields: address func (_m *Accounts) Get(address flow.Address) (*flow.Account, error) { ret := _m.Called(address) diff --git a/fvm/environment/mock/entropy_provider.go b/fvm/environment/mock/entropy_provider.go new file mode 100644 index 00000000000..cf3f19fb306 --- /dev/null +++ b/fvm/environment/mock/entropy_provider.go @@ -0,0 +1,51 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import mock "github.com/stretchr/testify/mock" + +// EntropyProvider is an autogenerated mock type for the EntropyProvider type +type EntropyProvider struct { + mock.Mock +} + +// RandomSource provides a mock function with given fields: +func (_m *EntropyProvider) RandomSource() ([]byte, error) { + ret := _m.Called() + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func() ([]byte, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() []byte); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewEntropyProvider interface { + mock.TestingT + Cleanup(func()) +} + +// NewEntropyProvider creates a new instance of EntropyProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewEntropyProvider(t mockConstructorTestingTNewEntropyProvider) *EntropyProvider { + mock := &EntropyProvider{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/fvm/environment/mock/environment.go b/fvm/environment/mock/environment.go index 11ee326c3f5..9d860227990 100644 --- a/fvm/environment/mock/environment.go +++ b/fvm/environment/mock/environment.go @@ -466,6 +466,30 @@ func (_m *Environment) FlushPendingUpdates() (environment.ContractUpdates, error return r0, r1 } +// GenerateAccountID provides a mock function with given fields: address +func (_m *Environment) GenerateAccountID(address common.Address) (uint64, error) { + ret := _m.Called(address) + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(common.Address) (uint64, error)); ok { + return rf(address) + } + if rf, ok := ret.Get(0).(func(common.Address) uint64); ok { + r0 = rf(address) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(common.Address) error); ok { + r1 = rf(address) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GenerateUUID provides a mock function with given fields: func (_m *Environment) GenerateUUID() (uint64, error) { ret := _m.Called() diff --git a/fvm/environment/mock/unsafe_random_generator.go b/fvm/environment/mock/random_generator.go similarity index 52% rename from fvm/environment/mock/unsafe_random_generator.go rename to fvm/environment/mock/random_generator.go index c92560981dd..0d0f1cf00e4 100644 --- a/fvm/environment/mock/unsafe_random_generator.go +++ b/fvm/environment/mock/random_generator.go @@ -4,13 +4,13 @@ package mock import mock "github.com/stretchr/testify/mock" -// UnsafeRandomGenerator is an autogenerated mock type for the UnsafeRandomGenerator type -type UnsafeRandomGenerator struct { +// RandomGenerator is an autogenerated mock type for the RandomGenerator type +type RandomGenerator struct { mock.Mock } // UnsafeRandom provides a mock function with given fields: -func (_m *UnsafeRandomGenerator) UnsafeRandom() (uint64, error) { +func (_m *RandomGenerator) UnsafeRandom() (uint64, error) { ret := _m.Called() var r0 uint64 @@ -33,14 +33,14 @@ func (_m *UnsafeRandomGenerator) UnsafeRandom() (uint64, error) { return r0, r1 } -type mockConstructorTestingTNewUnsafeRandomGenerator interface { +type mockConstructorTestingTNewRandomGenerator interface { mock.TestingT Cleanup(func()) } -// NewUnsafeRandomGenerator creates a new instance of UnsafeRandomGenerator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewUnsafeRandomGenerator(t mockConstructorTestingTNewUnsafeRandomGenerator) *UnsafeRandomGenerator { - mock := &UnsafeRandomGenerator{} +// NewRandomGenerator creates a new instance of RandomGenerator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewRandomGenerator(t mockConstructorTestingTNewRandomGenerator) *RandomGenerator { + mock := &RandomGenerator{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) diff --git a/fvm/environment/parse_restricted_checker.go b/fvm/environment/parse_restricted_checker.go index a792788508c..48f38738c4f 100644 --- a/fvm/environment/parse_restricted_checker.go +++ b/fvm/environment/parse_restricted_checker.go @@ -4,12 +4,12 @@ package environment import ( "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/module/trace" ) func parseRestricted( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, ) error { if txnState.IsParseRestricted() { @@ -31,7 +31,7 @@ func parseRestricted( func parseRestrict1Arg[ Arg0T any, ]( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, callback func(Arg0T) error, arg0 Arg0T, @@ -48,7 +48,7 @@ func parseRestrict2Arg[ Arg0T any, Arg1T any, ]( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, callback func(Arg0T, Arg1T) error, arg0 Arg0T, @@ -67,7 +67,7 @@ func parseRestrict3Arg[ Arg1T any, Arg2T any, ]( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, callback func(Arg0T, Arg1T, Arg2T) error, arg0 Arg0T, @@ -85,7 +85,7 @@ func parseRestrict3Arg[ func parseRestrict1Ret[ Ret0T any, ]( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, callback func() (Ret0T, error), ) ( @@ -105,7 +105,7 @@ func parseRestrict1Arg1Ret[ Arg0T any, Ret0T any, ]( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, callback func(Arg0T) (Ret0T, error), arg0 Arg0T, @@ -127,7 +127,7 @@ func parseRestrict2Arg1Ret[ Arg1T any, Ret0T any, ]( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, callback func(Arg0T, Arg1T) (Ret0T, error), arg0 Arg0T, @@ -151,7 +151,7 @@ func parseRestrict3Arg1Ret[ Arg2T any, Ret0T any, ]( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, callback func(Arg0T, Arg1T, Arg2T) (Ret0T, error), arg0 Arg0T, @@ -177,7 +177,7 @@ func parseRestrict4Arg1Ret[ Arg3T any, Ret0T any, ]( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, callback func(Arg0T, Arg1T, Arg2T, Arg3T) (Ret0T, error), arg0 Arg0T, @@ -206,7 +206,7 @@ func parseRestrict6Arg1Ret[ Arg5T any, Ret0T any, ]( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, callback func(Arg0T, Arg1T, Arg2T, Arg3T, Arg4T, Arg5T) (Ret0T, error), arg0 Arg0T, @@ -233,7 +233,7 @@ func parseRestrict1Arg2Ret[ Ret0T any, Ret1T any, ]( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, spanName trace.SpanName, callback func(Arg0T) (Ret0T, Ret1T, error), arg0 Arg0T, diff --git a/fvm/environment/programs.go b/fvm/environment/programs.go index 8aedb0068cc..85c52a843d9 100644 --- a/fvm/environment/programs.go +++ b/fvm/environment/programs.go @@ -11,9 +11,9 @@ import ( "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/module/trace" ) @@ -29,7 +29,7 @@ type Programs struct { meter Meter metrics MetricsReporter - txnState storage.Transaction + txnState storage.TransactionPreparer accounts Accounts // NOTE: non-address programs are not reusable across transactions, hence @@ -45,7 +45,7 @@ func NewPrograms( tracer tracing.TracerSpan, meter Meter, metrics MetricsReporter, - txnState storage.Transaction, + txnState storage.TransactionPreparer, accounts Accounts, ) *Programs { return &Programs{ @@ -93,7 +93,6 @@ func (programs *Programs) getOrLoadAddressProgram( location common.AddressLocation, load func() (*interpreter.Program, error), ) (*interpreter.Program, error) { - top, err := programs.dependencyStack.top() if err != nil { return nil, err @@ -127,7 +126,7 @@ func (programs *Programs) getOrLoadAddressProgram( loader, ) if err != nil { - return nil, fmt.Errorf("error getting program: %w", err) + return nil, fmt.Errorf("error getting program %v: %w", location, err) } // Add dependencies to the stack. @@ -220,7 +219,7 @@ func newProgramLoader( } func (loader *programLoader) Compute( - txState state.NestedTransaction, + _ state.NestedTransactionPreparer, location common.AddressLocation, ) ( *derived.Program, @@ -270,7 +269,7 @@ func (loader *programLoader) loadWithDependencyTracking( error, ) { // this program is not in cache, so we need to load it into the cache. - // tho have proper invalidation, we need to track the dependencies of the program. + // to have proper invalidation, we need to track the dependencies of the program. // If this program depends on another program, // that program will be loaded before this one finishes loading (calls set). // That is why this is a stack. @@ -280,7 +279,13 @@ func (loader *programLoader) loadWithDependencyTracking( // Get collected dependencies of the loaded program. // Pop the dependencies from the stack even if loading errored. - stackLocation, dependencies, depErr := loader.dependencyStack.pop() + // + // In case of an error, the dependencies of the errored program should not be merged + // into the dependencies of the parent program. This is to prevent the parent program + // from thinking that this program was already loaded and is in the cache, + // if it requests it again. + merge := err == nil + stackLocation, dependencies, depErr := loader.dependencyStack.pop(merge) if depErr != nil { err = multierror.Append(err, depErr).ErrorOrNil() } @@ -371,7 +376,9 @@ func (s *dependencyStack) add(dependencies derived.ProgramDependencies) error { } // pop the last dependencies on the stack and return them. -func (s *dependencyStack) pop() (common.Location, derived.ProgramDependencies, error) { +// if merge is false then the dependencies are not merged into the parent tracker. +// this is used to pop the dependencies of a program that errored during loading. +func (s *dependencyStack) pop(merge bool) (common.Location, derived.ProgramDependencies, error) { if len(s.trackers) <= 1 { return nil, derived.NewProgramDependencies(), @@ -384,11 +391,13 @@ func (s *dependencyStack) pop() (common.Location, derived.ProgramDependencies, e tracker := s.trackers[len(s.trackers)-1] s.trackers = s.trackers[:len(s.trackers)-1] - // Add the dependencies of the popped tracker to the parent tracker - // This is an optimisation to avoid having to iterate through the entire stack - // everytime a dependency is pushed or added, instead we add the popped dependencies to the new top of the stack. - // (because if C depends on B which depends on A, A's dependencies include C). - s.trackers[len(s.trackers)-1].dependencies.Merge(tracker.dependencies) + if merge { + // Add the dependencies of the popped tracker to the parent tracker + // This is an optimisation to avoid having to iterate through the entire stack + // everytime a dependency is pushed or added, instead we add the popped dependencies to the new top of the stack. + // (because if C depends on B which depends on A, A's dependencies include C). + s.trackers[len(s.trackers)-1].dependencies.Merge(tracker.dependencies) + } return tracker.location, tracker.dependencies, nil } diff --git a/fvm/environment/programs_test.go b/fvm/environment/programs_test.go index a6c297ca9b8..9cb50d273b5 100644 --- a/fvm/environment/programs_test.go +++ b/fvm/environment/programs_test.go @@ -9,12 +9,12 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/model/flow" ) @@ -44,6 +44,8 @@ var ( contractA0Code = ` pub contract A { + pub struct interface Foo{} + pub fun hello(): String { return "bad version" } @@ -52,6 +54,8 @@ var ( contractACode = ` pub contract A { + pub struct interface Foo{} + pub fun hello(): String { return "hello from A" } @@ -60,16 +64,32 @@ var ( contractA2Code = ` pub contract A2 { + pub struct interface Foo{} + pub fun hello(): String { return "hello from A2" } } ` + contractABreakingCode = ` + pub contract A { + pub struct interface Foo{ + pub fun hello() + } + + pub fun hello(): String { + return "hello from A with breaking change" + } + } + ` + contractBCode = ` import 0xa pub contract B { + pub struct Bar : A.Foo {} + pub fun hello(): String { return "hello from B but also ".concat(A.hello()) } @@ -81,6 +101,8 @@ var ( import A from 0xa pub contract C { + pub struct Bar : A.Foo {} + pub fun hello(): String { return "hello from C, ".concat(B.hello()) } @@ -88,16 +110,14 @@ var ( ` ) -func setupProgramsTest(t *testing.T) storage.SnapshotTree { - txnState := storage.SerialTransaction{ - NestedTransaction: state.NewTransactionState( - delta.NewDeltaView(nil), - state.DefaultParameters()), - } +func setupProgramsTest(t *testing.T) snapshot.SnapshotTree { + blockDatabase := storage.NewBlockDatabase(nil, 0, nil) + txnState, err := blockDatabase.NewTransaction(0, state.DefaultParameters()) + require.NoError(t, err) accounts := environment.NewAccounts(txnState) - err := accounts.Create(nil, addressA) + err = accounts.Create(nil, addressA) require.NoError(t, err) err = accounts.Create(nil, addressB) @@ -109,11 +129,11 @@ func setupProgramsTest(t *testing.T) storage.SnapshotTree { executionSnapshot, err := txnState.FinalizeMainTransaction() require.NoError(t, err) - return storage.NewSnapshotTree(nil).Append(executionSnapshot) + return snapshot.NewSnapshotTree(nil).Append(executionSnapshot) } func getTestContract( - snapshot state.StorageSnapshot, + snapshot snapshot.StorageSnapshot, location common.AddressLocation, ) ( []byte, @@ -127,7 +147,7 @@ func getTestContract( func Test_Programs(t *testing.T) { vm := fvm.NewVirtualMachine() - derivedBlockData := derived.NewEmptyDerivedBlockData() + derivedBlockData := derived.NewEmptyDerivedBlockData(0) mainSnapshot := setupProgramsTest(t) @@ -138,9 +158,9 @@ func Test_Programs(t *testing.T) { fvm.WithCadenceLogging(true), fvm.WithDerivedBlockData(derivedBlockData)) - var contractASnapshot *state.ExecutionSnapshot - var contractBSnapshot *state.ExecutionSnapshot - var txASnapshot *state.ExecutionSnapshot + var contractASnapshot *snapshot.ExecutionSnapshot + var contractBSnapshot *snapshot.ExecutionSnapshot + var txASnapshot *snapshot.ExecutionSnapshot t.Run("contracts can be updated", func(t *testing.T) { retrievedContractA, err := getTestContract( @@ -189,12 +209,12 @@ func Test_Programs(t *testing.T) { }) t.Run("register touches are captured for simple contract A", func(t *testing.T) { - fmt.Println("---------- Real transaction here ------------") + t.Log("---------- Real transaction here ------------") // run a TX using contract A loadedCode := false - execASnapshot := state.NewReadFuncStorageSnapshot( + execASnapshot := snapshot.NewReadFuncStorageSnapshot( func(id flow.RegisterID) (flow.RegisterValue, error) { expectedId := flow.ContractRegisterID( flow.BytesToAddress([]byte(id.Owner)), @@ -239,7 +259,7 @@ func Test_Programs(t *testing.T) { txASnapshot = executionSnapshotA // execute transaction again, this time make sure it doesn't load code - execA2Snapshot := state.NewReadFuncStorageSnapshot( + execA2Snapshot := snapshot.NewReadFuncStorageSnapshot( func(id flow.RegisterID) (flow.RegisterValue, error) { notId := flow.ContractRegisterID( flow.BytesToAddress([]byte(id.Owner)), @@ -340,7 +360,7 @@ func Test_Programs(t *testing.T) { // rerun transaction // execute transaction again, this time make sure it doesn't load code - execB2Snapshot := state.NewReadFuncStorageSnapshot( + execB2Snapshot := snapshot.NewReadFuncStorageSnapshot( func(id flow.RegisterID) (flow.RegisterValue, error) { idA := flow.ContractRegisterID( flow.BytesToAddress([]byte(id.Owner)), @@ -372,7 +392,7 @@ func Test_Programs(t *testing.T) { }) t.Run("deploying new contract A2 invalidates B because of * imports", func(t *testing.T) { - // deploy contract B + // deploy contract A2 executionSnapshot, output, err := vm.Run( context, fvm.Transaction( @@ -444,7 +464,7 @@ func Test_Programs(t *testing.T) { // rerun transaction // execute transaction again, this time make sure it doesn't load code - execB2Snapshot := state.NewReadFuncStorageSnapshot( + execB2Snapshot := snapshot.NewReadFuncStorageSnapshot( func(id flow.RegisterID) (flow.RegisterValue, error) { idA := flow.ContractRegisterID( flow.BytesToAddress([]byte(id.Owner)), @@ -484,7 +504,7 @@ func Test_Programs(t *testing.T) { // at this point programs cache should contain data for contract A // only because contract B has been called - execASnapshot := state.NewReadFuncStorageSnapshot( + execASnapshot := snapshot.NewReadFuncStorageSnapshot( func(id flow.RegisterID) (flow.RegisterValue, error) { notId := flow.ContractRegisterID( flow.BytesToAddress([]byte(id.Owner)), @@ -584,7 +604,7 @@ func Test_ProgramsDoubleCounting(t *testing.T) { snapshotTree := setupProgramsTest(t) vm := fvm.NewVirtualMachine() - derivedBlockData := derived.NewEmptyDerivedBlockData() + derivedBlockData := derived.NewEmptyDerivedBlockData(0) metrics := &metricsReporter{} context := fvm.NewContext( @@ -658,7 +678,7 @@ func Test_ProgramsDoubleCounting(t *testing.T) { require.Equal(t, 0, cached) }) - callC := func(snapshotTree storage.SnapshotTree) storage.SnapshotTree { + callC := func(snapshotTree snapshot.SnapshotTree) snapshot.SnapshotTree { procCallC := fvm.Transaction( flow.NewTransactionBody().SetScript( []byte( @@ -742,6 +762,79 @@ func Test_ProgramsDoubleCounting(t *testing.T) { require.Equal(t, 0, metrics.CacheMisses) }) + t.Run("update A to breaking change and ensure cache state", func(t *testing.T) { + // deploy contract A + executionSnapshot, output, err := vm.Run( + context, + fvm.Transaction( + updateContractTx("A", contractABreakingCode, addressA), + derivedBlockData.NextTxIndexForTestingOnly()), + snapshotTree) + require.NoError(t, err) + require.NoError(t, output.Err) + + snapshotTree = snapshotTree.Append(executionSnapshot) + + entryA := derivedBlockData.GetProgramForTestingOnly(contractALocation) + entryB := derivedBlockData.GetProgramForTestingOnly(contractBLocation) + entryC := derivedBlockData.GetProgramForTestingOnly(contractCLocation) + + require.Nil(t, entryA) + require.Nil(t, entryB) + require.Nil(t, entryC) + + cached := derivedBlockData.CachedPrograms() + require.Equal(t, 1, cached) + }) + + callCAfterItsBroken := func(snapshotTree snapshot.SnapshotTree) snapshot.SnapshotTree { + procCallC := fvm.Transaction( + flow.NewTransactionBody().SetScript( + []byte( + ` + import A from 0xa + import B from 0xb + import C from 0xc + transaction { + prepare() { + log(C.hello()) + } + }`, + )), + derivedBlockData.NextTxIndexForTestingOnly()) + + executionSnapshot, output, err := vm.Run( + context, + procCallC, + snapshotTree) + require.NoError(t, err) + require.Error(t, output.Err) + + entryA := derivedBlockData.GetProgramForTestingOnly(contractALocation) + entryA2 := derivedBlockData.GetProgramForTestingOnly(contractA2Location) + entryB := derivedBlockData.GetProgramForTestingOnly(contractBLocation) + entryC := derivedBlockData.GetProgramForTestingOnly(contractCLocation) + + require.NotNil(t, entryA) + require.NotNil(t, entryA2) // loaded due to "*" import in B + require.Nil(t, entryB) // failed to load + require.Nil(t, entryC) // failed to load + + cached := derivedBlockData.CachedPrograms() + require.Equal(t, 2, cached) + + return snapshotTree.Append(executionSnapshot) + } + + t.Run("Call C when broken", func(t *testing.T) { + metrics.Reset() + snapshotTree = callCAfterItsBroken(snapshotTree) + + // miss A, hit A, hit A2, hit A, hit A2, hit A + require.Equal(t, 5, metrics.CacheHits) + require.Equal(t, 1, metrics.CacheMisses) + }) + } func callTx(name string, address flow.Address) *flow.TransactionBody { @@ -781,7 +874,7 @@ func updateContractTx(name, code string, address flow.Address) *flow.Transaction ).AddAuthorizer(address) } -func compareExecutionSnapshots(t *testing.T, a, b *state.ExecutionSnapshot) { +func compareExecutionSnapshots(t *testing.T, a, b *snapshot.ExecutionSnapshot) { require.Equal(t, a.WriteSet, b.WriteSet) require.Equal(t, a.ReadSet, b.ReadSet) require.Equal(t, a.SpockSecret, b.SpockSecret) diff --git a/fvm/environment/random_generator.go b/fvm/environment/random_generator.go new file mode 100644 index 00000000000..63562ff06bf --- /dev/null +++ b/fvm/environment/random_generator.go @@ -0,0 +1,146 @@ +package environment + +import ( + "encoding/binary" + "fmt" + + "github.com/onflow/flow-go/crypto/random" + "github.com/onflow/flow-go/fvm/errors" + "github.com/onflow/flow-go/fvm/storage/state" + "github.com/onflow/flow-go/fvm/tracing" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state/protocol/prg" +) + +// EntropyProvider represents an entropy (source of randomness) provider +type EntropyProvider interface { + // RandomSource provides a source of entropy that can be + // expanded into randoms (using a pseudo-random generator). + // The returned slice should have at least 128 bits of entropy. + // The function doesn't error in normal operations, any + // error should be treated as an exception. + RandomSource() ([]byte, error) +} + +type RandomGenerator interface { + // UnsafeRandom returns a random uint64 + // The name follows Cadence interface + UnsafeRandom() (uint64, error) +} + +var _ RandomGenerator = (*randomGenerator)(nil) + +// randomGenerator implements RandomGenerator and is used +// for the transactions execution environment +type randomGenerator struct { + tracer tracing.TracerSpan + entropySource EntropyProvider + txId flow.Identifier + prg random.Rand + isPRGCreated bool +} + +type ParseRestrictedRandomGenerator struct { + txnState state.NestedTransactionPreparer + impl RandomGenerator +} + +func NewParseRestrictedRandomGenerator( + txnState state.NestedTransactionPreparer, + impl RandomGenerator, +) RandomGenerator { + return ParseRestrictedRandomGenerator{ + txnState: txnState, + impl: impl, + } +} + +func (gen ParseRestrictedRandomGenerator) UnsafeRandom() ( + uint64, + error, +) { + return parseRestrict1Ret( + gen.txnState, + trace.FVMEnvRandom, + gen.impl.UnsafeRandom) +} + +func NewRandomGenerator( + tracer tracing.TracerSpan, + entropySource EntropyProvider, + txId flow.Identifier, +) RandomGenerator { + gen := &randomGenerator{ + tracer: tracer, + entropySource: entropySource, + txId: txId, + isPRGCreated: false, // PRG is not created + } + + return gen +} + +func (gen *randomGenerator) createPRG() (random.Rand, error) { + // Use the protocol state source of randomness [SoR] for the current block's + // execution + source, err := gen.entropySource.RandomSource() + // `RandomSource` does not error in normal operations. + // Any error should be treated as an exception. + if err != nil { + return nil, fmt.Errorf("reading random source from state failed: %w", err) + } + + // Use the state/protocol PRG derivation from the source of randomness: + // - for the transaction execution case, the PRG used must be a CSPRG + // - use the state/protocol/prg customizer defined for the execution environment + // - use the transaction ID as an extra diversifier of the CSPRG. Although this + // does not add any extra entropy to the output, it allows creating an independent + // PRG for each transaction. + csprg, err := prg.New(source, prg.ExecutionEnvironment, gen.txId[:]) + if err != nil { + return nil, fmt.Errorf("failed to create a CSPRG from source: %w", err) + } + + return csprg, nil +} + +// UnsafeRandom returns a random uint64 using the underlying PRG (currently +// using a crypto-secure one). This function is not thread safe, due to the gen.prg +// instance currently used. This is fine because a +// single transaction has a single RandomGenerator and is run in a single +// thread. +func (gen *randomGenerator) UnsafeRandom() (uint64, error) { + defer gen.tracer.StartExtensiveTracingChildSpan( + trace.FVMEnvRandom).End() + + // PRG creation is only done once. + if !gen.isPRGCreated { + newPRG, err := gen.createPRG() + if err != nil { + return 0, err + } + gen.prg = newPRG + gen.isPRGCreated = true + } + + buf := make([]byte, 8) + gen.prg.Read(buf) // Note: prg.Read does not return error + return binary.LittleEndian.Uint64(buf), nil +} + +var _ RandomGenerator = (*dummyRandomGenerator)(nil) + +// dummyRandomGenerator implements RandomGenerator and is used +// for the scripts execution environment +type dummyRandomGenerator struct{} + +func NewDummyRandomGenerator() RandomGenerator { + return &dummyRandomGenerator{} +} + +// UnsafeRandom() returns an error because executing scripts +// does not support randomness APIs. +func (gen *dummyRandomGenerator) UnsafeRandom() (uint64, error) { + return 0, errors.NewOperationNotSupportedError("Random") +} diff --git a/fvm/environment/random_generator_test.go b/fvm/environment/random_generator_test.go new file mode 100644 index 00000000000..539aa99423f --- /dev/null +++ b/fvm/environment/random_generator_test.go @@ -0,0 +1,80 @@ +package environment_test + +import ( + "math" + mrand "math/rand" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/crypto/random" + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/fvm/environment/mock" + "github.com/onflow/flow-go/fvm/tracing" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestRandomGenerator(t *testing.T) { + entropyProvider := &mock.EntropyProvider{} + entropyProvider.On("RandomSource").Return(unittest.RandomBytes(48), nil) + + getRandoms := func(txId flow.Identifier, N int) []uint64 { + // seed the RG with the same block header + urg := environment.NewRandomGenerator( + tracing.NewTracerSpan(), + entropyProvider, + txId) + numbers := make([]uint64, N) + for i := 0; i < N; i++ { + u, err := urg.UnsafeRandom() + require.NoError(t, err) + numbers[i] = u + } + return numbers + } + + // basic randomness test to check outputs are "uniformly" spread over the + // output space + t.Run("randomness test", func(t *testing.T) { + for i := 0; i < 10; i++ { + txId := unittest.TransactionFixture().ID() + urg := environment.NewRandomGenerator( + tracing.NewTracerSpan(), + entropyProvider, + txId) + + // make sure n is a power of 2 so that there is no bias in the last class + // n is a random power of 2 (from 2 to 2^10) + n := 1 << (1 + mrand.Intn(10)) + classWidth := (math.MaxUint64 / uint64(n)) + 1 + random.BasicDistributionTest(t, uint64(n), uint64(classWidth), urg.UnsafeRandom) + } + }) + + // tests that has deterministic outputs. + t.Run("PRG-based Random", func(t *testing.T) { + for i := 0; i < 10; i++ { + txId := unittest.TransactionFixture().ID() + N := 100 + r1 := getRandoms(txId, N) + r2 := getRandoms(txId, N) + require.Equal(t, r1, r2) + } + }) + + t.Run("transaction specific randomness", func(t *testing.T) { + txns := [][]uint64{} + for i := 0; i < 10; i++ { + txId := unittest.TransactionFixture().ID() + N := 2 + txns = append(txns, getRandoms(txId, N)) + } + + for i, txn := range txns { + for _, otherTxn := range txns[i+1:] { + require.NotEqual(t, txn, otherTxn) + } + } + }) +} diff --git a/fvm/environment/transaction_info.go b/fvm/environment/transaction_info.go index d8a44090263..25cf64baba4 100644 --- a/fvm/environment/transaction_info.go +++ b/fvm/environment/transaction_info.go @@ -4,7 +4,7 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" @@ -48,12 +48,12 @@ type TransactionInfo interface { } type ParseRestrictedTransactionInfo struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer impl TransactionInfo } func NewParseRestrictedTransactionInfo( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, impl TransactionInfo, ) TransactionInfo { return ParseRestrictedTransactionInfo{ diff --git a/fvm/environment/unsafe_random_generator.go b/fvm/environment/unsafe_random_generator.go deleted file mode 100644 index ffb93d31a63..00000000000 --- a/fvm/environment/unsafe_random_generator.go +++ /dev/null @@ -1,144 +0,0 @@ -package environment - -import ( - "crypto/sha256" - "encoding/binary" - "fmt" - "hash" - "sync" - - "golang.org/x/crypto/hkdf" - - "github.com/onflow/flow-go/crypto/random" - "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" - "github.com/onflow/flow-go/fvm/tracing" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/trace" -) - -type UnsafeRandomGenerator interface { - // UnsafeRandom returns a random uint64 - UnsafeRandom() (uint64, error) -} - -type unsafeRandomGenerator struct { - tracer tracing.TracerSpan - - blockHeader *flow.Header - - prg random.Rand - seedOnce sync.Once -} - -type ParseRestrictedUnsafeRandomGenerator struct { - txnState state.NestedTransaction - impl UnsafeRandomGenerator -} - -func NewParseRestrictedUnsafeRandomGenerator( - txnState state.NestedTransaction, - impl UnsafeRandomGenerator, -) UnsafeRandomGenerator { - return ParseRestrictedUnsafeRandomGenerator{ - txnState: txnState, - impl: impl, - } -} - -func (gen ParseRestrictedUnsafeRandomGenerator) UnsafeRandom() ( - uint64, - error, -) { - return parseRestrict1Ret( - gen.txnState, - trace.FVMEnvUnsafeRandom, - gen.impl.UnsafeRandom) -} - -func NewUnsafeRandomGenerator( - tracer tracing.TracerSpan, - blockHeader *flow.Header, -) UnsafeRandomGenerator { - gen := &unsafeRandomGenerator{ - tracer: tracer, - blockHeader: blockHeader, - } - - return gen -} - -// This function abstracts building the PRG seed from the entropy source `randomSource`. -// It does not make assumptions about the quality of the source, nor about -// its length (the source could be a fingerprint of entity, an ID of an entity, -// -// a beacon signature..) -// -// It therefore uses a mechansim to extract the source entropy and expand it into -// the required `seedLen` bytes (this can be a KDF, a MAC, a hash with extended-length output..) -func seedFromEntropySource(randomSource []byte, seedLen int) ([]byte, error) { - // This implementation used HKDF, - // but other promitives with the 2 properties above could also be used. - hkdf := hkdf.New(func() hash.Hash { return sha256.New() }, randomSource, nil, nil) - seed := make([]byte, random.Chacha20SeedLen) - n, err := hkdf.Read(seed) - if n != len(seed) { - return nil, fmt.Errorf("extracting seed with HKDF failed, required %d bytes, got %d", random.Chacha20SeedLen, n) - } - if err != nil { - return nil, fmt.Errorf("extracting seed with HKDF failed: %w", err) - } - return seed, nil -} - -// seed seeds the pseudo-random number generator using the block header ID -// as an entropy source. -// The seed function is currently called for each tranaction, the PRG is used -// to provide all the randoms the transaction needs through UnsafeRandom. -// -// This allows lazy seeding of the random number generator, -// since not a lot of transactions/scripts use it and the time it takes to seed it is not negligible. -func (gen *unsafeRandomGenerator) seed() { - gen.seedOnce.Do(func() { - if gen.blockHeader == nil { - return - } - - // The block header ID is currently used as the entropy source. - // This should evolve to become the beacon signature (safer entropy source than - // the block ID) - // Extract the entropy from the source and expand it into the required seed length. - source := gen.blockHeader.ID() - seed, err := seedFromEntropySource(source[:], random.Chacha20SeedLen) - if err != nil { - return - } - - // initialize a fresh crypto-secure PRG with the seed (here ChaCha20) - // This PRG provides all outputs of Cadence UnsafeRandom. - prg, err := random.NewChacha20PRG(seed, []byte{}) - if err != nil { - return - } - gen.prg = prg - }) -} - -// UnsafeRandom returns a random uint64 using the underlying PRG (currently using a crypto-secure one). -// this is not thread safe, due to the gen.prg instance currently used. -// Its also not thread safe because each thread needs to be deterministically seeded with a different seed. -// This is Ok because a single transaction has a single UnsafeRandomGenerator and is run in a single thread. -func (gen *unsafeRandomGenerator) UnsafeRandom() (uint64, error) { - defer gen.tracer.StartExtensiveTracingChildSpan(trace.FVMEnvUnsafeRandom).End() - - // The internal seeding is only done once. - gen.seed() - - if gen.prg == nil { - return 0, errors.NewOperationNotSupportedError("UnsafeRandom") - } - - buf := make([]byte, 8) - gen.prg.Read(buf) - return binary.LittleEndian.Uint64(buf), nil -} diff --git a/fvm/environment/unsafe_random_generator_test.go b/fvm/environment/unsafe_random_generator_test.go deleted file mode 100644 index 294bd761fd6..00000000000 --- a/fvm/environment/unsafe_random_generator_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package environment_test - -import ( - "fmt" - "math" - mrand "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "gonum.org/v1/gonum/stat" - - "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/tracing" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/unittest" -) - -// TODO: these functions are copied from flow-go/crypto/rand -// Once the new flow-go/crypto/ module version is tagged, flow-go would upgrade -// to the new version and import these functions -func BasicDistributionTest(t *testing.T, n uint64, classWidth uint64, randf func() (uint64, error)) { - // sample size should ideally be a high number multiple of `n` - // but if `n` is too small, we could use a small sample size so that the test - // isn't too slow - sampleSize := 1000 * n - if n < 100 { - sampleSize = (80000 / n) * n // highest multiple of n less than 80000 - } - distribution := make([]float64, n) - // populate the distribution - for i := uint64(0); i < sampleSize; i++ { - r, err := randf() - require.NoError(t, err) - if n*classWidth != 0 { - require.Less(t, r, n*classWidth) - } - distribution[r/classWidth] += 1.0 - } - EvaluateDistributionUniformity(t, distribution) -} - -func EvaluateDistributionUniformity(t *testing.T, distribution []float64) { - tolerance := 0.05 - stdev := stat.StdDev(distribution, nil) - mean := stat.Mean(distribution, nil) - assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed: n: %d, stdev: %v, mean: %v", len(distribution), stdev, mean)) -} - -func TestUnsafeRandomGenerator(t *testing.T) { - // basic randomness test to check outputs are "uniformly" spread over the - // output space - t.Run("randomness test", func(t *testing.T) { - bh := unittest.BlockHeaderFixtureOnChain(flow.Mainnet.Chain().ChainID()) - urg := environment.NewUnsafeRandomGenerator(tracing.NewTracerSpan(), bh) - - // make sure n is a power of 2 so that there is no bias in the last class - // n is a random power of 2 (from 2 to 2^10) - n := 1 << (1 + mrand.Intn(10)) - classWidth := (math.MaxUint64 / uint64(n)) + 1 - BasicDistributionTest(t, uint64(n), uint64(classWidth), urg.UnsafeRandom) - }) - - // tests that unsafeRandom is PRG based and hence has deterministic outputs. - t.Run("PRG-based UnsafeRandom", func(t *testing.T) { - bh := unittest.BlockHeaderFixtureOnChain(flow.Mainnet.Chain().ChainID()) - N := 100 - getRandoms := func() []uint64 { - // seed the RG with the same block header - urg := environment.NewUnsafeRandomGenerator(tracing.NewTracerSpan(), bh) - numbers := make([]uint64, N) - for i := 0; i < N; i++ { - u, err := urg.UnsafeRandom() - require.NoError(t, err) - numbers[i] = u - } - return numbers - } - r1 := getRandoms() - r2 := getRandoms() - require.Equal(t, r1, r2) - }) -} diff --git a/fvm/environment/uuids.go b/fvm/environment/uuids.go index 8c5ca67a3b9..2612ac6d1f8 100644 --- a/fvm/environment/uuids.go +++ b/fvm/environment/uuids.go @@ -1,27 +1,40 @@ package environment import ( + "crypto/sha256" "encoding/binary" "fmt" - "github.com/onflow/flow-go/fvm/state" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" "github.com/onflow/flow-go/utils/slices" ) +const ( + // The max value for any is uuid partition is MaxUint56, since the top + // 8 bits in the uuid are used for partitioning. + MaxUint56 = (uint64(1) << 56) - 1 + + // Start warning when there's only a single high bit left. This should give + // us plenty of time to migrate to larger counters. + Uint56OverflowWarningThreshold = (uint64(1) << 55) - 1 +) + type UUIDGenerator interface { GenerateUUID() (uint64, error) } type ParseRestrictedUUIDGenerator struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer impl UUIDGenerator } func NewParseRestrictedUUIDGenerator( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, impl UUIDGenerator, ) UUIDGenerator { return ParseRestrictedUUIDGenerator{ @@ -39,45 +52,124 @@ func (generator ParseRestrictedUUIDGenerator) GenerateUUID() (uint64, error) { type uUIDGenerator struct { tracer tracing.TracerSpan + log zerolog.Logger meter Meter - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer + + blockHeader *flow.Header + txnIndex uint32 + + initialized bool + partition byte + registerId flow.RegisterID +} + +func uuidPartition(blockId flow.Identifier, txnIndex uint32) byte { + // Partitioning by txnIndex ensures temporally neighboring transactions do + // not share registers / conflict with each other. + // + // Since all blocks will have a transaction at txnIndex 0 but not + // necessarily a transaction at txnIndex 255, if we assign partition based + // only on txnIndex, partition 0's counter (and other low-valued + // partitions' counters) will fill up much more quickly than high-valued + // partitions' counters. Therefore, a deterministically random offset is + // used to ensure the partitioned counters are roughly balanced. Any byte + // in the sha hash is sufficiently random/uniform for this purpose (Note that + // block Id is already a sha hash, but its hash implementation may change + // underneath us). + // + // Note that since partition 0 reuses the legacy counter, its counter is + // much further ahead than the other partitions. If partition 0's counter + // is in danager of overflowing, use variants of "the power of two random + // choices" to shed load to other counters. + // + // The explicit mod is not really needed, but is there for completeness. + partitionOffset := sha256.Sum256(blockId[:])[0] + return byte((uint32(partitionOffset) + txnIndex) % 256) } func NewUUIDGenerator( tracer tracing.TracerSpan, + log zerolog.Logger, meter Meter, - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, + blockHeader *flow.Header, + txnIndex uint32, ) *uUIDGenerator { return &uUIDGenerator{ - tracer: tracer, - meter: meter, - txnState: txnState, + tracer: tracer, + log: log, + meter: meter, + txnState: txnState, + blockHeader: blockHeader, + txnIndex: txnIndex, + initialized: false, } } -// GetUUID reads uint64 byte value for uuid from the state -func (generator *uUIDGenerator) getUUID() (uint64, error) { - stateBytes, err := generator.txnState.Get(flow.UUIDRegisterID) +// getUint64 reads the uint64 value from the partitioned uuid register. +func (generator *uUIDGenerator) getUint64() (uint64, error) { + stateBytes, err := generator.txnState.Get(generator.registerId) if err != nil { - return 0, fmt.Errorf("cannot get uuid byte from state: %w", err) + return 0, fmt.Errorf( + "cannot get uuid partition %d byte from state: %w", + generator.partition, + err) } bytes := slices.EnsureByteSliceSize(stateBytes, 8) return binary.BigEndian.Uint64(bytes), nil } -// SetUUID sets a new uint64 byte value -func (generator *uUIDGenerator) setUUID(uuid uint64) error { +// setUint56 sets a new uint56 value into the partitioned uuid register. +func (generator *uUIDGenerator) setUint56( + value uint64, +) error { + if value > Uint56OverflowWarningThreshold { + if value > MaxUint56 { + return fmt.Errorf( + "uuid partition %d overflowed", + generator.partition) + } + + generator.log.Warn(). + Int("partition", int(generator.partition)). + Uint64("value", value). + Msg("uuid partition is running out of bits") + } + bytes := make([]byte, 8) - binary.BigEndian.PutUint64(bytes, uuid) - err := generator.txnState.Set(flow.UUIDRegisterID, bytes) + binary.BigEndian.PutUint64(bytes, value) + err := generator.txnState.Set(generator.registerId, bytes) if err != nil { - return fmt.Errorf("cannot set uuid byte to state: %w", err) + return fmt.Errorf( + "cannot set uuid %d byte to state: %w", + generator.partition, + err) } return nil } +func (generator *uUIDGenerator) maybeInitializePartition() { + if generator.initialized { + return + } + generator.initialized = true + + // NOTE: block header is not set for scripts. We'll just use partition 0 in + // this case. + if generator.blockHeader == nil { + generator.partition = 0 + } else { + generator.partition = uuidPartition( + generator.blockHeader.ID(), + generator.txnIndex) + } + + generator.registerId = flow.UUIDRegisterID(generator.partition) +} + // GenerateUUID generates a new uuid and persist the data changes into state func (generator *uUIDGenerator) GenerateUUID() (uint64, error) { defer generator.tracer.StartExtensiveTracingChildSpan( @@ -90,14 +182,19 @@ func (generator *uUIDGenerator) GenerateUUID() (uint64, error) { return 0, fmt.Errorf("generate uuid failed: %w", err) } - uuid, err := generator.getUUID() + generator.maybeInitializePartition() + + value, err := generator.getUint64() if err != nil { return 0, fmt.Errorf("cannot generate UUID: %w", err) } - err = generator.setUUID(uuid + 1) + err = generator.setUint56(value + 1) if err != nil { return 0, fmt.Errorf("cannot generate UUID: %w", err) } - return uuid, nil + + // Since the partition counter only goes up to MaxUint56, we can use the + // upper 8 bits to represent which partition was used. + return (uint64(generator.partition) << 56) | value, nil } diff --git a/fvm/environment/uuids_test.go b/fvm/environment/uuids_test.go index 5fa5a4cbde8..b83bd5b1821 100644 --- a/fvm/environment/uuids_test.go +++ b/fvm/environment/uuids_test.go @@ -1,76 +1,351 @@ package environment import ( + "fmt" "testing" + "github.com/rs/zerolog" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/execution/state/delta" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" + "github.com/onflow/flow-go/model/flow" ) -func TestUUIDs_GetAndSetUUID(t *testing.T) { - txnState := state.NewTransactionState( - delta.NewDeltaView(nil), - state.DefaultParameters()) - uuidsA := NewUUIDGenerator( +func TestUUIDPartition(t *testing.T) { + blockHeader := &flow.Header{} + + usedPartitions := map[byte]struct{}{} + + // With enough samples, all partitions should be used. (The first 1500 blocks + // only uses 254 partitions) + for numBlocks := 0; numBlocks < 2000; numBlocks++ { + blockId := blockHeader.ID() + + partition0 := uuidPartition(blockId, 0) + usedPartitions[partition0] = struct{}{} + + for txnIndex := 0; txnIndex < 256; txnIndex++ { + partition := uuidPartition(blockId, uint32(txnIndex)) + + // Ensure neighboring transactions uses neighoring partitions. + require.Equal(t, partition, partition0+byte(txnIndex)) + + // Ensure wrap around. + for i := 0; i < 5; i++ { + require.Equal( + t, + partition, + uuidPartition(blockId, uint32(txnIndex+i*256))) + } + } + + blockHeader.ParentID = blockId + } + + require.Len(t, usedPartitions, 256) +} + +func TestUUIDGeneratorInitializePartitionNoHeader(t *testing.T) { + for txnIndex := uint32(0); txnIndex < 256; txnIndex++ { + uuids := NewUUIDGenerator( + tracing.NewTracerSpan(), + zerolog.Nop(), + nil, + nil, + nil, + txnIndex) + require.False(t, uuids.initialized) + + uuids.maybeInitializePartition() + + require.True(t, uuids.initialized) + require.Equal(t, uuids.partition, byte(0)) + require.Equal(t, uuids.registerId, flow.UUIDRegisterID(byte(0))) + } +} + +func TestUUIDGeneratorInitializePartition(t *testing.T) { + blockHeader := &flow.Header{} + + for numBlocks := 0; numBlocks < 10; numBlocks++ { + blockId := blockHeader.ID() + + for txnIndex := uint32(0); txnIndex < 256; txnIndex++ { + uuids := NewUUIDGenerator( + tracing.NewTracerSpan(), + zerolog.Nop(), + nil, + nil, + blockHeader, + txnIndex) + require.False(t, uuids.initialized) + + uuids.maybeInitializePartition() + + require.True(t, uuids.initialized) + + expectedPartition := uuidPartition(blockId, txnIndex) + + require.Equal(t, uuids.partition, expectedPartition) + require.Equal( + t, + uuids.registerId, + flow.UUIDRegisterID(expectedPartition)) + } + + blockHeader.ParentID = blockId + } +} + +func TestUUIDGeneratorIdGeneration(t *testing.T) { + for txnIndex := uint32(0); txnIndex < 256; txnIndex++ { + testUUIDGenerator(t, &flow.Header{}, txnIndex) + } +} + +func testUUIDGenerator(t *testing.T, blockHeader *flow.Header, txnIndex uint32) { + generator := NewUUIDGenerator( + tracing.NewTracerSpan(), + zerolog.Nop(), + nil, + nil, + blockHeader, + txnIndex) + generator.maybeInitializePartition() + + partition := generator.partition + partitionMinValue := uint64(partition) << 56 + maxUint56 := uint64(72057594037927935) // (1 << 56) - 1 + + t.Run( + fmt.Sprintf("basic get and set uint (partition: %d)", partition), + func(t *testing.T) { + txnState := state.NewTransactionState(nil, state.DefaultParameters()) + uuidsA := NewUUIDGenerator( + tracing.NewTracerSpan(), + zerolog.Nop(), + NewMeter(txnState), + txnState, + blockHeader, + txnIndex) + uuidsA.maybeInitializePartition() + + uuid, err := uuidsA.getUint64() // start from zero + require.NoError(t, err) + require.Equal(t, uint64(0), uuid) + + err = uuidsA.setUint56(5) + require.NoError(t, err) + + // create new UUIDs instance + uuidsB := NewUUIDGenerator( + tracing.NewTracerSpan(), + zerolog.Nop(), + NewMeter(txnState), + txnState, + blockHeader, + txnIndex) + uuidsB.maybeInitializePartition() + + uuid, err = uuidsB.getUint64() // should read saved value + require.NoError(t, err) + + require.Equal(t, uint64(5), uuid) + }) + + t.Run( + fmt.Sprintf("basic id generation (partition: %d)", partition), + func(t *testing.T) { + txnState := state.NewTransactionState(nil, state.DefaultParameters()) + genA := NewUUIDGenerator( + tracing.NewTracerSpan(), + zerolog.Nop(), + NewMeter(txnState), + txnState, + blockHeader, + txnIndex) + + uuidA, err := genA.GenerateUUID() + require.NoError(t, err) + uuidB, err := genA.GenerateUUID() + require.NoError(t, err) + uuidC, err := genA.GenerateUUID() + require.NoError(t, err) + + require.Equal(t, partitionMinValue, uuidA) + require.Equal(t, partitionMinValue+1, uuidB) + require.Equal(t, partitionMinValue|1, uuidB) + require.Equal(t, partitionMinValue+2, uuidC) + require.Equal(t, partitionMinValue|2, uuidC) + + // Create new generator instance from same ledger + genB := NewUUIDGenerator( + tracing.NewTracerSpan(), + zerolog.Nop(), + NewMeter(txnState), + txnState, + blockHeader, + txnIndex) + + uuidD, err := genB.GenerateUUID() + require.NoError(t, err) + uuidE, err := genB.GenerateUUID() + require.NoError(t, err) + uuidF, err := genB.GenerateUUID() + require.NoError(t, err) + + require.Equal(t, partitionMinValue+3, uuidD) + require.Equal(t, partitionMinValue|3, uuidD) + require.Equal(t, partitionMinValue+4, uuidE) + require.Equal(t, partitionMinValue|4, uuidE) + require.Equal(t, partitionMinValue+5, uuidF) + require.Equal(t, partitionMinValue|5, uuidF) + }) + + t.Run( + fmt.Sprintf("setUint56 overflows (partition: %d)", partition), + func(t *testing.T) { + txnState := state.NewTransactionState(nil, state.DefaultParameters()) + uuids := NewUUIDGenerator( + tracing.NewTracerSpan(), + zerolog.Nop(), + NewMeter(txnState), + txnState, + blockHeader, + txnIndex) + uuids.maybeInitializePartition() + + err := uuids.setUint56(maxUint56) + require.NoError(t, err) + + value, err := uuids.getUint64() + require.NoError(t, err) + require.Equal(t, value, maxUint56) + + err = uuids.setUint56(maxUint56 + 1) + require.ErrorContains(t, err, "overflowed") + + value, err = uuids.getUint64() + require.NoError(t, err) + require.Equal(t, value, maxUint56) + }) + + t.Run( + fmt.Sprintf("id generation overflows (partition: %d)", partition), + func(t *testing.T) { + txnState := state.NewTransactionState(nil, state.DefaultParameters()) + uuids := NewUUIDGenerator( + tracing.NewTracerSpan(), + zerolog.Nop(), + NewMeter(txnState), + txnState, + blockHeader, + txnIndex) + uuids.maybeInitializePartition() + + err := uuids.setUint56(maxUint56 - 1) + require.NoError(t, err) + + value, err := uuids.GenerateUUID() + require.NoError(t, err) + require.Equal(t, value, partitionMinValue+maxUint56-1) + require.Equal(t, value, partitionMinValue|(maxUint56-1)) + + value, err = uuids.getUint64() + require.NoError(t, err) + require.Equal(t, value, maxUint56) + + _, err = uuids.GenerateUUID() + require.ErrorContains(t, err, "overflowed") + + value, err = uuids.getUint64() + require.NoError(t, err) + require.Equal(t, value, maxUint56) + }) +} + +func TestUUIDGeneratorHardcodedPartitionIdGeneration(t *testing.T) { + txnState := state.NewTransactionState(nil, state.DefaultParameters()) + uuids := NewUUIDGenerator( tracing.NewTracerSpan(), + zerolog.Nop(), NewMeter(txnState), - txnState) + txnState, + nil, + 0) - uuid, err := uuidsA.getUUID() // start from zero + // Hardcoded the partition to check for exact bytes + uuids.initialized = true + uuids.partition = 0xde + uuids.registerId = flow.UUIDRegisterID(0xde) + + value, err := uuids.GenerateUUID() require.NoError(t, err) - require.Equal(t, uint64(0), uuid) + require.Equal(t, value, uint64(0xde00000000000000)) - err = uuidsA.setUUID(5) + value, err = uuids.getUint64() require.NoError(t, err) + require.Equal(t, value, uint64(1)) - // create new UUIDs instance - uuidsB := NewUUIDGenerator( - tracing.NewTracerSpan(), - NewMeter(txnState), - txnState) - uuid, err = uuidsB.getUUID() // should read saved value + value, err = uuids.GenerateUUID() require.NoError(t, err) + require.Equal(t, value, uint64(0xde00000000000001)) - require.Equal(t, uint64(5), uuid) -} + value, err = uuids.getUint64() + require.NoError(t, err) + require.Equal(t, value, uint64(2)) -func Test_GenerateUUID(t *testing.T) { - txnState := state.NewTransactionState( - delta.NewDeltaView(nil), - state.DefaultParameters()) - genA := NewUUIDGenerator( - tracing.NewTracerSpan(), - NewMeter(txnState), - txnState) + value, err = uuids.GenerateUUID() + require.NoError(t, err) + require.Equal(t, value, uint64(0xde00000000000002)) - uuidA, err := genA.GenerateUUID() + value, err = uuids.getUint64() require.NoError(t, err) - uuidB, err := genA.GenerateUUID() + require.Equal(t, value, uint64(3)) + + // pretend we increamented the counter up to cafBad + cafBad := uint64(0x1c2a3f4b5a6d70) + decafBad := uint64(0xde1c2a3f4b5a6d70) + + err = uuids.setUint56(cafBad) require.NoError(t, err) - uuidC, err := genA.GenerateUUID() + + for i := 0; i < 5; i++ { + value, err = uuids.GenerateUUID() + require.NoError(t, err) + require.Equal(t, value, decafBad+uint64(i)) + } + + value, err = uuids.getUint64() require.NoError(t, err) + require.Equal(t, value, cafBad+uint64(5)) - require.Equal(t, uint64(0), uuidA) - require.Equal(t, uint64(1), uuidB) - require.Equal(t, uint64(2), uuidC) + // pretend we increamented the counter up to overflow - 2 + maxUint56Minus2 := uint64(0xfffffffffffffd) + err = uuids.setUint56(maxUint56Minus2) + require.NoError(t, err) - // Create new generator instance from same ledger - genB := NewUUIDGenerator( - tracing.NewTracerSpan(), - NewMeter(txnState), - txnState) + value, err = uuids.GenerateUUID() + require.NoError(t, err) + require.Equal(t, value, uint64(0xdefffffffffffffd)) - uuidD, err := genB.GenerateUUID() + value, err = uuids.getUint64() require.NoError(t, err) - uuidE, err := genB.GenerateUUID() + require.Equal(t, value, maxUint56Minus2+1) + + value, err = uuids.GenerateUUID() require.NoError(t, err) - uuidF, err := genB.GenerateUUID() + require.Equal(t, value, uint64(0xdefffffffffffffe)) + + value, err = uuids.getUint64() require.NoError(t, err) + require.Equal(t, value, maxUint56Minus2+2) - require.Equal(t, uint64(3), uuidD) - require.Equal(t, uint64(4), uuidE) - require.Equal(t, uint64(5), uuidF) + _, err = uuids.GenerateUUID() + require.ErrorContains(t, err, "overflowed") + + value, err = uuids.getUint64() + require.NoError(t, err) + require.Equal(t, value, maxUint56Minus2+2) } diff --git a/fvm/environment/value_store.go b/fvm/environment/value_store.go index f17f151c51f..8113de6762c 100644 --- a/fvm/environment/value_store.go +++ b/fvm/environment/value_store.go @@ -6,7 +6,7 @@ import ( "github.com/onflow/atree" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" @@ -24,12 +24,12 @@ type ValueStore interface { } type ParseRestrictedValueStore struct { - txnState state.NestedTransaction + txnState state.NestedTransactionPreparer impl ValueStore } func NewParseRestrictedValueStore( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, impl ValueStore, ) ValueStore { return ParseRestrictedValueStore{ diff --git a/fvm/executionParameters.go b/fvm/executionParameters.go index 0475af5fdac..46b64382b73 100644 --- a/fvm/executionParameters.go +++ b/fvm/executionParameters.go @@ -12,11 +12,21 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/meter" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/state" ) +func ProcedureStateParameters( + ctx Context, + proc Procedure, +) state.StateParameters { + return state.DefaultParameters(). + WithMeterParameters(getBasicMeterParameters(ctx, proc)). + WithMaxKeySizeAllowed(ctx.MaxStateKeySize). + WithMaxValueSizeAllowed(ctx.MaxStateValueSize) +} + // getBasicMeterParameters returns the set of meter parameters used for // general procedure execution. Subparts of the procedure execution may // specify custom meter parameters via nested transactions. @@ -45,7 +55,7 @@ func getBasicMeterParameters( func getBodyMeterParameters( ctx Context, proc Procedure, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) ( meter.MeterParameters, error, @@ -84,12 +94,12 @@ func getBodyMeterParameters( type MeterParamOverridesComputer struct { ctx Context - txnState storage.Transaction + txnState storage.TransactionPreparer } func NewMeterParamOverridesComputer( ctx Context, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) MeterParamOverridesComputer { return MeterParamOverridesComputer{ ctx: ctx, @@ -98,7 +108,7 @@ func NewMeterParamOverridesComputer( } func (computer MeterParamOverridesComputer) Compute( - _ state.NestedTransaction, + _ state.NestedTransactionPreparer, _ struct{}, ) ( derived.MeterParamOverrides, diff --git a/fvm/fvm.go b/fvm/fvm.go index ba4a612f810..ab929f174e0 100644 --- a/fvm/fvm.go +++ b/fvm/fvm.go @@ -6,14 +6,13 @@ import ( "github.com/onflow/cadence" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm/environment" errors "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/meter" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage" - "github.com/onflow/flow-go/fvm/storage/derived" "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/model/flow" ) @@ -88,7 +87,7 @@ func Run(executor ProcedureExecutor) error { type Procedure interface { NewExecutor( ctx Context, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) ProcedureExecutor ComputationLimit(ctx Context) uint64 @@ -100,24 +99,27 @@ type Procedure interface { // For transactions, the execution time is TxIndex. For scripts, the // execution time is EndOfBlockExecutionTime. ExecutionTime() logical.Time - - // TODO(patrick): deprecated this. - SetOutput(output ProcedureOutput) } // VM runs procedures type VM interface { + NewExecutor( + Context, + Procedure, + storage.TransactionPreparer, + ) ProcedureExecutor + Run( Context, Procedure, - state.StorageSnapshot, + snapshot.StorageSnapshot, ) ( - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, ProcedureOutput, error, ) - GetAccount(Context, flow.Address, state.StorageSnapshot) (*flow.Account, error) + GetAccount(Context, flow.Address, snapshot.StorageSnapshot) (*flow.Account, error) } var _ VM = (*VirtualMachine)(nil) @@ -130,46 +132,40 @@ func NewVirtualMachine() *VirtualMachine { return &VirtualMachine{} } -// TODO(patrick): rm after updating emulator -func (vm *VirtualMachine) RunV2( +func (vm *VirtualMachine) NewExecutor( ctx Context, proc Procedure, - storageSnapshot state.StorageSnapshot, -) ( - *state.ExecutionSnapshot, - ProcedureOutput, - error, -) { - return vm.Run(ctx, proc, storageSnapshot) + txn storage.TransactionPreparer, +) ProcedureExecutor { + return proc.NewExecutor(ctx, txn) } // Run runs a procedure against a ledger in the given context. func (vm *VirtualMachine) Run( ctx Context, proc Procedure, - storageSnapshot state.StorageSnapshot, + storageSnapshot snapshot.StorageSnapshot, ) ( - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, ProcedureOutput, error, ) { - derivedBlockData := ctx.DerivedBlockData - if derivedBlockData == nil { - derivedBlockData = derived.NewEmptyDerivedBlockDataWithTransactionOffset( - uint32(proc.ExecutionTime())) - } + blockDatabase := storage.NewBlockDatabase( + storageSnapshot, + proc.ExecutionTime(), + ctx.DerivedBlockData) + + stateParameters := ProcedureStateParameters(ctx, proc) - var derivedTxnData derived.DerivedTransactionCommitter + var storageTxn storage.Transaction var err error switch proc.Type() { case ScriptProcedureType: - derivedTxnData, err = derivedBlockData.NewSnapshotReadDerivedTransactionData( - proc.ExecutionTime(), - proc.ExecutionTime()) + storageTxn = blockDatabase.NewSnapshotReadTransaction(stateParameters) case TransactionProcedureType, BootstrapProcedureType: - derivedTxnData, err = derivedBlockData.NewDerivedTransactionData( + storageTxn, err = blockDatabase.NewTransaction( proc.ExecutionTime(), - proc.ExecutionTime()) + stateParameters) default: return nil, ProcedureOutput{}, fmt.Errorf( "invalid proc type: %v", @@ -182,38 +178,18 @@ func (vm *VirtualMachine) Run( err) } - // TODO(patrick): initialize view inside TransactionState - nestedTxn := state.NewTransactionState( - delta.NewDeltaView(storageSnapshot), - state.DefaultParameters(). - WithMeterParameters(getBasicMeterParameters(ctx, proc)). - WithMaxKeySizeAllowed(ctx.MaxStateKeySize). - WithMaxValueSizeAllowed(ctx.MaxStateValueSize)) - - txnState := &storage.SerialTransaction{ - NestedTransaction: nestedTxn, - DerivedTransactionCommitter: derivedTxnData, - } - - executor := proc.NewExecutor(ctx, txnState) + executor := proc.NewExecutor(ctx, storageTxn) err = Run(executor) if err != nil { return nil, ProcedureOutput{}, err } - // Note: it is safe to skip committing derived data for non-normal - // transactions (i.e., bootstrap and script) since these do not invalidate - // derived data entries. - if proc.Type() == TransactionProcedureType { - // NOTE: It is not safe to ignore derivedTxnData' commit error for - // transactions that trigger derived data invalidation. - err = derivedTxnData.Commit() - if err != nil { - return nil, ProcedureOutput{}, err - } + err = storageTxn.Finalize() + if err != nil { + return nil, ProcedureOutput{}, err } - executionSnapshot, err := txnState.FinalizeMainTransaction() + executionSnapshot, err := storageTxn.Commit() if err != nil { return nil, ProcedureOutput{}, err } @@ -225,14 +201,17 @@ func (vm *VirtualMachine) Run( func (vm *VirtualMachine) GetAccount( ctx Context, address flow.Address, - storageSnapshot state.StorageSnapshot, + storageSnapshot snapshot.StorageSnapshot, ) ( *flow.Account, error, ) { - nestedTxn := state.NewTransactionState( - // TODO(patrick): initialize view inside TransactionState - delta.NewDeltaView(storageSnapshot), + blockDatabase := storage.NewBlockDatabase( + storageSnapshot, + 0, + ctx.DerivedBlockData) + + storageTxn := blockDatabase.NewSnapshotReadTransaction( state.DefaultParameters(). WithMaxKeySizeAllowed(ctx.MaxStateKeySize). WithMaxValueSizeAllowed(ctx.MaxStateValueSize). @@ -240,30 +219,11 @@ func (vm *VirtualMachine) GetAccount( meter.DefaultParameters(). WithStorageInteractionLimit(ctx.MaxStateInteractionSize))) - derivedBlockData := ctx.DerivedBlockData - if derivedBlockData == nil { - derivedBlockData = derived.NewEmptyDerivedBlockData() - } - - derivedTxnData, err := derivedBlockData.NewSnapshotReadDerivedTransactionData( - logical.EndOfBlockExecutionTime, - logical.EndOfBlockExecutionTime) - if err != nil { - return nil, fmt.Errorf( - "error creating derived transaction data for GetAccount: %w", - err) - } - - txnState := &storage.SerialTransaction{ - NestedTransaction: nestedTxn, - DerivedTransactionCommitter: derivedTxnData, - } - env := environment.NewScriptEnv( context.Background(), ctx.TracerSpan, ctx.EnvironmentParams, - txnState) + storageTxn) account, err := env.GetAccount(address) if err != nil { if errors.IsLedgerFailure(err) { diff --git a/fvm/fvm_bench_test.go b/fvm/fvm_bench_test.go index 51f02f0e2f0..276c8cb69b8 100644 --- a/fvm/fvm_bench_test.go +++ b/fvm/fvm_bench_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/runtime" @@ -31,8 +32,8 @@ import ( "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/fvm" reusableRuntime "github.com/onflow/flow-go/fvm/runtime" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" completeLedger "github.com/onflow/flow-go/ledger/complete" "github.com/onflow/flow-go/ledger/complete/wal/fixtures" "github.com/onflow/flow-go/model/flow" @@ -88,7 +89,7 @@ func (account *TestBenchAccount) DeployContract(b *testing.B, blockExec TestBenc require.NoError(b, err) computationResult := blockExec.ExecuteCollections(b, [][]*flow.TransactionBody{{txBody}}) - require.Empty(b, computationResult.TransactionResults[0].ErrorMessage) + require.Empty(b, computationResult.AllTransactionResults()[0].ErrorMessage) } func (account *TestBenchAccount) AddArrayToStorage(b *testing.B, blockExec TestBenchBlockExecutor, list []string) { @@ -125,14 +126,14 @@ func (account *TestBenchAccount) AddArrayToStorage(b *testing.B, blockExec TestB require.NoError(b, err) computationResult := blockExec.ExecuteCollections(b, [][]*flow.TransactionBody{{txBody}}) - require.Empty(b, computationResult.TransactionResults[0].ErrorMessage) + require.Empty(b, computationResult.AllTransactionResults()[0].ErrorMessage) } // BasicBlockExecutor executes blocks in sequence and applies all changes (not fork aware) type BasicBlockExecutor struct { blockComputer computer.BlockComputer derivedChainData *derived.DerivedChainData - activeSnapshot state.StorageSnapshot + activeSnapshot snapshot.SnapshotTree activeStateCommitment flow.StateCommitment chain flow.Chain serviceAccount *TestBenchAccount @@ -208,6 +209,8 @@ func NewBasicBlockExecutor(tb testing.TB, chain flow.Chain, logger zerolog.Logge ) me := new(moduleMock.Local) + me.On("NodeID").Return(unittest.IdentifierFixture()) + me.On("Sign", mock.Anything, mock.Anything).Return(nil, nil) me.On("SignFunc", mock.Anything, mock.Anything, mock.Anything). Return(nil, nil) @@ -221,10 +224,13 @@ func NewBasicBlockExecutor(tb testing.TB, chain flow.Chain, logger zerolog.Logge ledgerCommitter, me, prov, - nil) + nil, + nil, + 1) // We're interested in fvm's serial execution time require.NoError(tb, err) - snapshot := exeState.NewLedgerStorageSnapshot(ledger, initialCommit) + activeSnapshot := snapshot.NewSnapshotTree( + exeState.NewLedgerStorageSnapshot(ledger, initialCommit)) derivedChainData, err := derived.NewDerivedChainData( derived.DefaultDerivedDataCacheSize) @@ -234,7 +240,7 @@ func NewBasicBlockExecutor(tb testing.TB, chain flow.Chain, logger zerolog.Logge blockComputer: blockComputer, derivedChainData: derivedChainData, activeStateCommitment: initialCommit, - activeSnapshot: snapshot, + activeSnapshot: activeSnapshot, chain: chain, serviceAccount: serviceAccount, onStopFunc: onStopFunc, @@ -265,7 +271,11 @@ func (b *BasicBlockExecutor) ExecuteCollections(tb testing.TB, collections [][]* derivedBlockData) require.NoError(tb, err) - b.activeStateCommitment = computationResult.EndState + b.activeStateCommitment = computationResult.CurrentEndState() + + for _, snapshot := range computationResult.AllExecutionSnapshots() { + b.activeSnapshot = b.activeSnapshot.Append(snapshot) + } return computationResult } @@ -295,21 +305,19 @@ func (b *BasicBlockExecutor) SetupAccounts(tb testing.TB, privateKeys []flow.Acc require.NoError(tb, err) computationResult := b.ExecuteCollections(tb, [][]*flow.TransactionBody{{txBody}}) - require.Empty(tb, computationResult.TransactionResults[0].ErrorMessage) + require.Empty(tb, computationResult.AllTransactionResults()[0].ErrorMessage) var addr flow.Address - for _, eventList := range computationResult.Events { - for _, event := range eventList { - if event.Type == flow.EventAccountCreated { - data, err := jsoncdc.Decode(nil, event.Payload) - if err != nil { - tb.Fatal("setup account failed, error decoding events") - } - addr = flow.ConvertAddress( - data.(cadence.Event).Fields[0].(cadence.Address)) - break + for _, event := range computationResult.AllEvents() { + if event.Type == flow.EventAccountCreated { + data, err := ccf.Decode(nil, event.Payload) + if err != nil { + tb.Fatal("setup account failed, error decoding events") } + addr = flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) + break } } if addr == flow.EmptyAddress { @@ -441,10 +449,12 @@ func BenchmarkRuntimeTransaction(b *testing.B) { computationResult := blockExecutor.ExecuteCollections(b, [][]*flow.TransactionBody{transactions}) totalInteractionUsed := uint64(0) totalComputationUsed := uint64(0) - for j := 0; j < transactionsPerBlock; j++ { - require.Empty(b, computationResult.TransactionResults[j].ErrorMessage) - totalInteractionUsed += logE.InteractionUsed[computationResult.TransactionResults[j].ID().String()] - totalComputationUsed += computationResult.TransactionResults[j].ComputationUsed + results := computationResult.AllTransactionResults() + // not interested in the system transaction + for _, txRes := range results[0 : len(results)-1] { + require.Empty(b, txRes.ErrorMessage) + totalInteractionUsed += logE.InteractionUsed[txRes.ID().String()] + totalComputationUsed += txRes.ComputationUsed } b.ReportMetric(float64(totalInteractionUsed/uint64(transactionsPerBlock)), "interactions") b.ReportMetric(float64(totalComputationUsed/uint64(transactionsPerBlock)), "computation") @@ -686,8 +696,10 @@ func BenchRunNFTBatchTransfer(b *testing.B, } computationResult = blockExecutor.ExecuteCollections(b, [][]*flow.TransactionBody{transactions}) - for j := 0; j < transactionsPerBlock; j++ { - require.Empty(b, computationResult.TransactionResults[j].ErrorMessage) + results := computationResult.AllTransactionResults() + // not interested in the system transaction + for _, txRes := range results[0 : len(results)-1] { + require.Empty(b, txRes.ErrorMessage) } } } @@ -727,7 +739,7 @@ func setupReceiver(b *testing.B, be TestBenchBlockExecutor, nftAccount, batchNFT require.NoError(b, err) computationResult := be.ExecuteCollections(b, [][]*flow.TransactionBody{{txBody}}) - require.Empty(b, computationResult.TransactionResults[0].ErrorMessage) + require.Empty(b, computationResult.AllTransactionResults()[0].ErrorMessage) } func mintNFTs(b *testing.B, be TestBenchBlockExecutor, batchNFTAccount *TestBenchAccount, size int) { @@ -763,7 +775,7 @@ func mintNFTs(b *testing.B, be TestBenchBlockExecutor, batchNFTAccount *TestBenc require.NoError(b, err) computationResult := be.ExecuteCollections(b, [][]*flow.TransactionBody{{txBody}}) - require.Empty(b, computationResult.TransactionResults[0].ErrorMessage) + require.Empty(b, computationResult.AllTransactionResults()[0].ErrorMessage) } func fundAccounts(b *testing.B, be TestBenchBlockExecutor, value cadence.UFix64, accounts ...flow.Address) { @@ -780,7 +792,7 @@ func fundAccounts(b *testing.B, be TestBenchBlockExecutor, value cadence.UFix64, require.NoError(b, err) computationResult := be.ExecuteCollections(b, [][]*flow.TransactionBody{{txBody}}) - require.Empty(b, computationResult.TransactionResults[0].ErrorMessage) + require.Empty(b, computationResult.AllTransactionResults()[0].ErrorMessage) } } diff --git a/fvm/fvm_blockcontext_test.go b/fvm/fvm_blockcontext_test.go index f933d3db642..e4148fcc5c7 100644 --- a/fvm/fvm_blockcontext_test.go +++ b/fvm/fvm_blockcontext_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/runtime" "github.com/stretchr/testify/mock" @@ -22,8 +23,7 @@ import ( "github.com/onflow/flow-go/fvm/blueprints" envMock "github.com/onflow/flow-go/fvm/environment/mock" errors "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" - "github.com/onflow/flow-go/fvm/storage" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -946,7 +946,7 @@ func TestBlockContext_ExecuteTransaction_StorageLimit(t *testing.T) { t.Run("Storing too much data fails", newVMTest().withBootstrapProcedureOptions(bootstrapOptions...). run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // this test requires storage limits to be enforced ctx.LimitAccountStorage = true @@ -993,7 +993,7 @@ func TestBlockContext_ExecuteTransaction_StorageLimit(t *testing.T) { })) t.Run("Increasing storage capacity works", newVMTest().withBootstrapProcedureOptions(bootstrapOptions...). run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { ctx.LimitAccountStorage = true // this test requires storage limits to be enforced // Create an account private key. @@ -1080,7 +1080,7 @@ func TestBlockContext_ExecuteTransaction_InteractionLimitReached(t *testing.T) { t.Run("Using to much interaction fails", newVMTest().withBootstrapProcedureOptions(bootstrapOptions...). withContextOptions(fvm.WithTransactionFeesEnabled(true)). run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { ctx.MaxStateInteractionSize = 500_000 // Create an account private key. @@ -1161,7 +1161,7 @@ func TestBlockContext_ExecuteTransaction_InteractionLimitReached(t *testing.T) { t.Run("Using to much interaction but not failing because of service account", newVMTest().withBootstrapProcedureOptions(bootstrapOptions...). withContextOptions(fvm.WithTransactionFeesEnabled(true)). run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { ctx.MaxStateInteractionSize = 500_000 // Create an account private key. @@ -1209,7 +1209,7 @@ func TestBlockContext_ExecuteTransaction_InteractionLimitReached(t *testing.T) { fvm.WithSequenceNumberCheckAndIncrementEnabled(false), ). run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { ctx.MaxStateInteractionSize = 50_000 // Create an account private key. @@ -1614,7 +1614,7 @@ func TestBlockContext_GetAccount(t *testing.T) { // read the address of the account created (e.g. "0x01" and convert it // to flow.address) - data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) + data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(t, err) address := flow.ConvertAddress( data.(cadence.Event).Fields[0].(cadence.Address)) @@ -1664,17 +1664,19 @@ func TestBlockContext_GetAccount(t *testing.T) { }) } -func TestBlockContext_UnsafeRandom(t *testing.T) { +func TestBlockContext_Random(t *testing.T) { t.Parallel() chain, vm := createChainAndVm(flow.Mainnet) header := &flow.Header{Height: 42} + source := testutil.EntropyProviderFixture(nil) ctx := fvm.NewContext( fvm.WithChain(chain), fvm.WithBlockHeader(header), + fvm.WithEntropyProvider(source), fvm.WithCadenceLogging(true), ) @@ -1701,9 +1703,10 @@ func TestBlockContext_UnsafeRandom(t *testing.T) { require.Len(t, output.Logs, 1) - num, err := strconv.ParseUint(output.Logs[0], 10, 64) + // output cannot be deterministic because transaction signature is not deterministic + // (which makes the tx hash and the PRG seed used by the execution not deterministic) + _, err = strconv.ParseUint(output.Logs[0], 10, 64) require.NoError(t, err) - require.Equal(t, uint64(0x7515f254adc6f8af), num) }) } @@ -1735,7 +1738,7 @@ func TestBlockContext_ExecuteTransaction_CreateAccount_WithMonotonicAddresses(t require.Len(t, accountCreatedEvents, 1) - data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) + data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(t, err) address := flow.ConvertAddress( data.(cadence.Event).Fields[0].(cadence.Address)) @@ -1748,7 +1751,7 @@ func TestBlockContext_ExecuteTransaction_FailingTransactions(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - storageSnapshot state.StorageSnapshot, + storageSnapshot snapshot.StorageSnapshot, address flow.Address, ) uint64 { @@ -1781,7 +1784,7 @@ func TestBlockContext_ExecuteTransaction_FailingTransactions(t *testing.T) { fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW), fvm.WithExecutionMemoryLimit(math.MaxUint64), ).run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { ctx.LimitAccountStorage = true // this test requires storage limits to be enforced // Create an account private key. @@ -1843,7 +1846,7 @@ func TestBlockContext_ExecuteTransaction_FailingTransactions(t *testing.T) { fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW), fvm.WithExecutionMemoryLimit(math.MaxUint64), ).run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { ctx.LimitAccountStorage = true // this test requires storage limits to be enforced // Create an account private key. @@ -1908,7 +1911,7 @@ func TestBlockContext_ExecuteTransaction_FailingTransactions(t *testing.T) { fvm.WithAccountCreationFee(fvm.DefaultAccountCreationFee), ). run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // this test requires storage limits to be enforced ctx.LimitAccountStorage = true @@ -1967,7 +1970,7 @@ func TestBlockContext_ExecuteTransaction_FailingTransactions(t *testing.T) { fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW), ). run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // this test requires storage limits to be enforced ctx.LimitAccountStorage = true diff --git a/fvm/fvm_fuzz_test.go b/fvm/fvm_fuzz_test.go index 18bc7685ea0..8877540b362 100644 --- a/fvm/fvm_fuzz_test.go +++ b/fvm/fvm_fuzz_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" "github.com/onflow/flow-go/engine/execution/testutil" @@ -15,7 +16,7 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/meter" - "github.com/onflow/flow-go/fvm/storage" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -32,7 +33,7 @@ func FuzzTransactionComputationLimit(f *testing.F) { tt := fuzzTransactionTypes[transactionType] - vmt.run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + vmt.run(func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // create the transaction txBody := tt.createTxBody(t, tctx) // set the computation limit @@ -223,7 +224,7 @@ func getDeductedFees(tb testing.TB, tctx transactionTypeContext, results fuzzRes var feesDeductedEvent cadence.Event for _, e := range results.output.Events { if string(e.Type) == fmt.Sprintf("A.%s.FlowFees.FeesDeducted", environment.FlowFeesAddress(tctx.chain)) { - data, err := jsoncdc.Decode(nil, e.Payload) + data, err := ccf.Decode(nil, e.Payload) require.NoError(tb, err) feesDeductedEvent, ok = data.(cadence.Event) require.True(tb, ok, "Event payload should be of type cadence event.") @@ -232,8 +233,15 @@ func getDeductedFees(tb testing.TB, tctx transactionTypeContext, results fuzzRes if feesDeductedEvent.Type() == nil { return 0, false } - fees, ok = feesDeductedEvent.Fields[0].(cadence.UFix64) - require.True(tb, ok, "FeesDeducted[0] event should be of type cadence.UFix64.") + + for i, f := range feesDeductedEvent.Type().(*cadence.EventType).Fields { + if f.Identifier == "amount" { + fees, ok = feesDeductedEvent.Fields[i].(cadence.UFix64) + require.True(tb, ok, "FeesDeducted event amount field should be of type cadence.UFix64.") + break + } + } + return fees, true } @@ -254,7 +262,7 @@ func bootstrapFuzzStateAndTxContext(tb testing.TB) (bootstrappedVmTest, transact ).withContextOptions( fvm.WithTransactionFeesEnabled(true), fvm.WithAccountStorageLimit(true), - ).bootstrapWith(func(vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) (storage.SnapshotTree, error) { + ).bootstrapWith(func(vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) (snapshot.SnapshotTree, error) { // ==== Create an account ==== var txBody *flow.TransactionBody privateKey, txBody = testutil.CreateAccountCreationTransaction(tb, chain) @@ -276,7 +284,7 @@ func bootstrapFuzzStateAndTxContext(tb testing.TB) (bootstrappedVmTest, transact accountCreatedEvents := filterAccountCreatedEvents(output.Events) // read the address of the account created (e.g. "0x01" and convert it to flow.address) - data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) + data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(tb, err) address = flow.ConvertAddress( diff --git a/fvm/fvm_signature_test.go b/fvm/fvm_signature_test.go index a32076de063..6a4e20ad284 100644 --- a/fvm/fvm_signature_test.go +++ b/fvm/fvm_signature_test.go @@ -16,7 +16,7 @@ import ( "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/fvm" fvmCrypto "github.com/onflow/flow-go/fvm/crypto" - "github.com/onflow/flow-go/fvm/storage" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" msig "github.com/onflow/flow-go/module/signature" ) @@ -162,7 +162,7 @@ func TestKeyListSignature(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) { privateKey, publicKey := createKey() signableMessage, message := createMessage("foo") @@ -258,7 +258,7 @@ func TestKeyListSignature(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) { privateKeyA, publicKeyA := createKey() privateKeyB, publicKeyB := createKey() @@ -394,7 +394,7 @@ func TestBLSMultiSignature(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) { code := func(signatureAlgorithm signatureAlgorithm) []byte { @@ -505,7 +505,7 @@ func TestBLSMultiSignature(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) { code := []byte( @@ -628,7 +628,7 @@ func TestBLSMultiSignature(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) { code := func(signatureAlgorithm signatureAlgorithm) []byte { @@ -752,7 +752,7 @@ func TestBLSMultiSignature(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) { message, cadenceMessage := createMessage("random_message") diff --git a/fvm/fvm_test.go b/fvm/fvm_test.go index c034115be27..e93c19c575a 100644 --- a/fvm/fvm_test.go +++ b/fvm/fvm_test.go @@ -9,9 +9,11 @@ import ( "testing" "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/runtime/tests/utils" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/crypto" @@ -24,8 +26,7 @@ import ( errors "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/meter" reusableRuntime "github.com/onflow/flow-go/fvm/runtime" - "github.com/onflow/flow-go/fvm/state" - "github.com/onflow/flow-go/fvm/storage" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -64,7 +65,7 @@ func createChainAndVm(chainID flow.ChainID) (flow.Chain, fvm.VM) { } func (vmt vmTest) run( - f func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree), + f func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree), ) func(t *testing.T) { return func(t *testing.T) { baseOpts := []fvm.Option{ @@ -78,7 +79,7 @@ func (vmt vmTest) run( chain := ctx.Chain vm := fvm.NewVirtualMachine() - snapshotTree := storage.NewSnapshotTree(nil) + snapshotTree := snapshot.NewSnapshotTree(nil) baseBootstrapOpts := []fvm.BootstrapProcedureOption{ fvm.WithInitialTokenSupply(unittest.GenesisTokenSupply), @@ -101,7 +102,7 @@ func (vmt vmTest) run( // bootstrapWith executes the bootstrap procedure and the custom bootstrap function // and returns a prepared bootstrappedVmTest with all the state needed func (vmt vmTest) bootstrapWith( - bootstrap func(vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) (storage.SnapshotTree, error), + bootstrap func(vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) (snapshot.SnapshotTree, error), ) (bootstrappedVmTest, error) { baseOpts := []fvm.Option{ @@ -115,7 +116,7 @@ func (vmt vmTest) bootstrapWith( chain := ctx.Chain vm := fvm.NewVirtualMachine() - snapshotTree := storage.NewSnapshotTree(nil) + snapshotTree := snapshot.NewSnapshotTree(nil) baseBootstrapOpts := []fvm.BootstrapProcedureOption{ fvm.WithInitialTokenSupply(unittest.GenesisTokenSupply), @@ -144,12 +145,12 @@ func (vmt vmTest) bootstrapWith( type bootstrappedVmTest struct { chain flow.Chain ctx fvm.Context - snapshotTree storage.SnapshotTree + snapshotTree snapshot.SnapshotTree } // run Runs a test from the bootstrapped state, without changing the bootstrapped state func (vmt bootstrappedVmTest) run( - f func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree), + f func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree), ) func(t *testing.T) { return func(t *testing.T) { f(t, fvm.NewVirtualMachine(), vmt.chain, vmt.ctx, vmt.snapshotTree) @@ -419,7 +420,7 @@ func TestWithServiceAccount(t *testing.T) { fvm.WithSequenceNumberCheckAndIncrementEnabled(false), ) - snapshotTree := storage.NewSnapshotTree(nil) + snapshotTree := snapshot.NewSnapshotTree(nil) txBody := flow.NewTransactionBody(). SetScript([]byte(`transaction { prepare(signer: AuthAccount) { AuthAccount(payer: signer) } }`)). @@ -557,7 +558,7 @@ func TestEventLimits(t *testing.T) { func TestHappyPathTransactionSigning(t *testing.T) { newVMTest().run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // Create an account private key. privateKey, err := testutil.GenerateAccountPrivateKey() require.NoError(t, err) @@ -595,7 +596,7 @@ func TestHappyPathTransactionSigning(t *testing.T) { } func TestTransactionFeeDeduction(t *testing.T) { - getBalance := func(vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree, address flow.Address) uint64 { + getBalance := func(vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree, address flow.Address) uint64 { code := []byte(fmt.Sprintf(` import FungibleToken from 0x%s @@ -695,16 +696,30 @@ func TestTransactionFeeDeduction(t *testing.T) { unittest.EnsureEventsIndexSeq(t, output.Events, chain.ChainID()) require.NotEmpty(t, feeDeduction.Payload) - payload, err := jsoncdc.Decode(nil, feeDeduction.Payload) + payload, err := ccf.Decode(nil, feeDeduction.Payload) require.NoError(t, err) event := payload.(cadence.Event) - require.Equal(t, txFees, event.Fields[0].ToGoValue()) + var actualTXFees any + var actualInclusionEffort any + var actualExecutionEffort any + for i, f := range event.EventType.Fields { + switch f.Identifier { + case "amount": + actualTXFees = event.Fields[i].ToGoValue() + case "executionEffort": + actualExecutionEffort = event.Fields[i].ToGoValue() + case "inclusionEffort": + actualInclusionEffort = event.Fields[i].ToGoValue() + } + } + + require.Equal(t, txFees, actualTXFees) // Inclusion effort should be equivalent to 1.0 UFix64 - require.Equal(t, uint64(100_000_000), event.Fields[1].ToGoValue()) + require.Equal(t, uint64(100_000_000), actualInclusionEffort) // Execution effort should be non-0 - require.Greater(t, event.Fields[2].ToGoValue(), uint64(0)) + require.Greater(t, actualExecutionEffort, uint64(0)) }, }, @@ -910,8 +925,8 @@ func TestTransactionFeeDeduction(t *testing.T) { }, } - runTx := func(tc testCase) func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { - return func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + runTx := func(tc testCase) func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { + return func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // ==== Create an account ==== privateKey, txBody := testutil.CreateAccountCreationTransaction(t, chain) @@ -935,7 +950,7 @@ func TestTransactionFeeDeduction(t *testing.T) { require.Len(t, accountCreatedEvents, 1) // read the address of the account created (e.g. "0x01" and convert it to flow.address) - data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) + data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(t, err) address := flow.ConvertAddress( data.(cadence.Event).Fields[0].(cadence.Address)) @@ -1052,7 +1067,7 @@ func TestSettingExecutionWeights(t *testing.T) { }, ), ).run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { txBody := flow.NewTransactionBody(). SetScript([]byte(` @@ -1100,7 +1115,7 @@ func TestSettingExecutionWeights(t *testing.T) { ).withContextOptions( fvm.WithMemoryLimit(10_000_000_000), ).run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // Create an account private key. privateKeys, err := testutil.GenerateAccountPrivateKeys(1) require.NoError(t, err) @@ -1150,7 +1165,7 @@ func TestSettingExecutionWeights(t *testing.T) { ).withContextOptions( fvm.WithMemoryLimit(10_000_000_000), ).run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { txBody := flow.NewTransactionBody(). SetScript([]byte(` @@ -1194,7 +1209,7 @@ func TestSettingExecutionWeights(t *testing.T) { memoryWeights, ), ).run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { privateKeys, err := testutil.GenerateAccountPrivateKeys(1) require.NoError(t, err) @@ -1261,7 +1276,7 @@ func TestSettingExecutionWeights(t *testing.T) { }, ), ).run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { txBody := flow.NewTransactionBody(). SetScript([]byte(` transaction { @@ -1297,7 +1312,7 @@ func TestSettingExecutionWeights(t *testing.T) { }, ), ).run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { txBody := flow.NewTransactionBody(). SetScript([]byte(` @@ -1334,7 +1349,7 @@ func TestSettingExecutionWeights(t *testing.T) { }, ), ).run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { txBody := flow.NewTransactionBody(). SetScript([]byte(` transaction { @@ -1377,7 +1392,7 @@ func TestSettingExecutionWeights(t *testing.T) { fvm.WithTransactionFeesEnabled(true), fvm.WithMemoryLimit(math.MaxUint64), ).run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // Use the maximum amount of computation so that the transaction still passes. loops := uint64(997) maxExecutionEffort := uint64(997) @@ -1432,12 +1447,21 @@ func TestSettingExecutionWeights(t *testing.T) { for _, event := range output.Events { // the fee deduction event should only contain the max gas worth of execution effort. if strings.Contains(string(event.Type), "FlowFees.FeesDeducted") { - ev, err := jsoncdc.Decode(nil, event.Payload) + v, err := ccf.Decode(nil, event.Payload) require.NoError(t, err) + + ev := v.(cadence.Event) + var actualExecutionEffort any + for i, f := range ev.Type().(*cadence.EventType).Fields { + if f.Identifier == "executionEffort" { + actualExecutionEffort = ev.Fields[i].ToGoValue() + } + } + require.Equal( t, maxExecutionEffort, - ev.(cadence.Event).Fields[2].ToGoValue().(uint64)) + actualExecutionEffort) } } unittest.EnsureEventsIndexSeq(t, output.Events, chain.ChainID()) @@ -1494,7 +1518,7 @@ func TestStorageUsed(t *testing.T) { _, output, err := vm.Run( ctx, fvm.Script(code), - state.MapStorageSnapshot{ + snapshot.MapStorageSnapshot{ accountStatusId: status.ToBytes(), }) require.NoError(t, err) @@ -1629,7 +1653,7 @@ func TestStorageCapacity(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) { service := chain.ServiceAddress() snapshotTree, signer := createAccount( @@ -1730,7 +1754,7 @@ func TestScriptContractMutationsFailure(t *testing.T) { t.Run("contract additions are not committed", newVMTest().run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // Create an account private key. privateKeys, err := testutil.GenerateAccountPrivateKeys(1) require.NoError(t, err) @@ -1771,7 +1795,7 @@ func TestScriptContractMutationsFailure(t *testing.T) { t.Run("contract removals are not committed", newVMTest().run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // Create an account private key. privateKeys, err := testutil.GenerateAccountPrivateKeys(1) privateKey := privateKeys[0] @@ -1841,7 +1865,7 @@ func TestScriptContractMutationsFailure(t *testing.T) { t.Run("contract updates are not committed", newVMTest().run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // Create an account private key. privateKeys, err := testutil.GenerateAccountPrivateKeys(1) privateKey := privateKeys[0] @@ -1914,7 +1938,7 @@ func TestScriptAccountKeyMutationsFailure(t *testing.T) { t.Run("Account key additions are not committed", newVMTest().run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // Create an account private key. privateKeys, err := testutil.GenerateAccountPrivateKeys(1) require.NoError(t, err) @@ -1961,7 +1985,7 @@ func TestScriptAccountKeyMutationsFailure(t *testing.T) { t.Run("Account key removals are not committed", newVMTest().run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // Create an account private key. privateKeys, err := testutil.GenerateAccountPrivateKeys(1) require.NoError(t, err) @@ -1999,6 +2023,87 @@ func TestScriptAccountKeyMutationsFailure(t *testing.T) { ) } +func TestScriptExecutionLimit(t *testing.T) { + + t.Parallel() + + script := fvm.Script([]byte(` + pub fun main() { + var s: Int256 = 1024102410241024 + var i: Int256 = 0 + var a: Int256 = 7 + var b: Int256 = 5 + var c: Int256 = 2 + + while i < 150000 { + s = s * a + s = s / b + s = s / c + i = i + 1 + } + } + `)) + + bootstrapProcedureOptions := []fvm.BootstrapProcedureOption{ + fvm.WithTransactionFee(fvm.DefaultTransactionFees), + fvm.WithExecutionMemoryLimit(math.MaxUint32), + fvm.WithExecutionEffortWeights(map[common.ComputationKind]uint64{ + common.ComputationKindStatement: 1569, + common.ComputationKindLoop: 1569, + common.ComputationKindFunctionInvocation: 1569, + environment.ComputationKindGetValue: 808, + environment.ComputationKindCreateAccount: 2837670, + environment.ComputationKindSetValue: 765, + }), + fvm.WithExecutionMemoryWeights(meter.DefaultMemoryWeights), + fvm.WithMinimumStorageReservation(fvm.DefaultMinimumStorageReservation), + fvm.WithAccountCreationFee(fvm.DefaultAccountCreationFee), + fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW), + } + + t.Run("Exceeding computation limit", + newVMTest().withBootstrapProcedureOptions( + bootstrapProcedureOptions..., + ).withContextOptions( + fvm.WithTransactionFeesEnabled(true), + fvm.WithAccountStorageLimit(true), + fvm.WithComputationLimit(10000), + ).run( + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { + scriptCtx := fvm.NewContextFromParent(ctx) + + _, output, err := vm.Run(scriptCtx, script, snapshotTree) + require.NoError(t, err) + require.Error(t, output.Err) + require.True(t, errors.IsComputationLimitExceededError(output.Err)) + require.ErrorContains(t, output.Err, "computation exceeds limit (10000)") + require.GreaterOrEqual(t, output.ComputationUsed, uint64(10000)) + require.GreaterOrEqual(t, output.MemoryEstimate, uint64(548020260)) + }, + ), + ) + + t.Run("Sufficient computation limit", + newVMTest().withBootstrapProcedureOptions( + bootstrapProcedureOptions..., + ).withContextOptions( + fvm.WithTransactionFeesEnabled(true), + fvm.WithAccountStorageLimit(true), + fvm.WithComputationLimit(20000), + ).run( + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { + scriptCtx := fvm.NewContextFromParent(ctx) + + _, output, err := vm.Run(scriptCtx, script, snapshotTree) + require.NoError(t, err) + require.NoError(t, output.Err) + require.GreaterOrEqual(t, output.ComputationUsed, uint64(17955)) + require.GreaterOrEqual(t, output.MemoryEstimate, uint64(984017413)) + }, + ), + ) +} + func TestInteractionLimit(t *testing.T) { type testCase struct { name string @@ -2058,7 +2163,7 @@ func TestInteractionLimit(t *testing.T) { fvm.WithTransactionFeesEnabled(true), fvm.WithAccountStorageLimit(true), ).bootstrapWith( - func(vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) (storage.SnapshotTree, error) { + func(vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) (snapshot.SnapshotTree, error) { // ==== Create an account ==== var txBody *flow.TransactionBody privateKey, txBody = testutil.CreateAccountCreationTransaction(t, chain) @@ -2085,7 +2190,7 @@ func TestInteractionLimit(t *testing.T) { accountCreatedEvents := filterAccountCreatedEvents(output.Events) // read the address of the account created (e.g. "0x01" and convert it to flow.address) - data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) + data, err := ccf.Decode(nil, accountCreatedEvents[0].Payload) if err != nil { return snapshotTree, err } @@ -2125,7 +2230,7 @@ func TestInteractionLimit(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, vmt.run( - func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree storage.SnapshotTree) { + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { // ==== Transfer funds with lowe interaction limit ==== txBody := transferTokensTx(chain). AddAuthorizer(address). @@ -2183,7 +2288,7 @@ func TestAuthAccountCapabilities(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) { // Create an account private key. privateKeys, err := testutil.GenerateAccountPrivateKeys(1) @@ -2276,7 +2381,7 @@ func TestAuthAccountCapabilities(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) { // Create two private keys privateKeys, err := testutil.GenerateAccountPrivateKeys(2) @@ -2410,7 +2515,7 @@ func TestAttachments(t *testing.T) { vm fvm.VM, chain flow.Chain, ctx fvm.Context, - snapshotTree storage.SnapshotTree, + snapshotTree snapshot.SnapshotTree, ) { script := fvm.Script([]byte(` @@ -2449,3 +2554,263 @@ func TestAttachments(t *testing.T) { test(t, false) }) } + +func TestCapabilityControllers(t *testing.T) { + test := func(t *testing.T, capabilityControllersEnabled bool) { + newVMTest(). + withBootstrapProcedureOptions(). + withContextOptions( + fvm.WithReusableCadenceRuntimePool( + reusableRuntime.NewReusableCadenceRuntimePool( + 1, + runtime.Config{ + CapabilityControllersEnabled: capabilityControllersEnabled, + }, + ), + ), + ). + run(func( + t *testing.T, + vm fvm.VM, + chain flow.Chain, + ctx fvm.Context, + snapshotTree snapshot.SnapshotTree, + ) { + txBody := flow.NewTransactionBody(). + SetScript([]byte(` + transaction { + prepare(signer: AuthAccount) { + let cap = signer.capabilities.storage.issue<&Int>(/storage/foo) + assert(cap.id == 1) + + let cap2 = signer.capabilities.storage.issue<&String>(/storage/bar) + assert(cap2.id == 2) + } + } + `)). + SetProposalKey(chain.ServiceAddress(), 0, 0). + AddAuthorizer(chain.ServiceAddress()). + SetPayer(chain.ServiceAddress()) + + err := testutil.SignTransactionAsServiceAccount(txBody, 0, chain) + require.NoError(t, err) + + _, output, err := vm.Run( + ctx, + fvm.Transaction(txBody, 0), + snapshotTree) + require.NoError(t, err) + + if capabilityControllersEnabled { + require.NoError(t, output.Err) + } else { + require.Error(t, output.Err) + require.ErrorContains( + t, + output.Err, + "`AuthAccount` has no member `capabilities`") + } + }, + )(t) + } + + t.Run("enabled", func(t *testing.T) { + test(t, true) + }) + + t.Run("disabled", func(t *testing.T) { + test(t, false) + }) +} + +func TestStorageIterationWithBrokenValues(t *testing.T) { + + t.Parallel() + + newVMTest(). + withBootstrapProcedureOptions(). + withContextOptions( + fvm.WithReusableCadenceRuntimePool( + reusableRuntime.NewReusableCadenceRuntimePool( + 1, + runtime.Config{ + AccountLinkingEnabled: true, + }, + ), + ), + fvm.WithContractDeploymentRestricted(false), + ). + run( + func( + t *testing.T, + vm fvm.VM, + chain flow.Chain, + ctx fvm.Context, + snapshotTree snapshot.SnapshotTree, + ) { + // Create a private key + privateKeys, err := testutil.GenerateAccountPrivateKeys(1) + require.NoError(t, err) + + // Bootstrap a ledger, creating an account with the provided private key and the root account. + snapshotTree, accounts, err := testutil.CreateAccounts( + vm, + snapshotTree, + privateKeys, + chain, + ) + require.NoError(t, err) + + contractA := ` + pub contract A { + pub struct interface Foo{} + } + ` + + updatedContractA := ` + pub contract A { + pub struct interface Foo{ + pub fun hello() + } + } + ` + + contractB := fmt.Sprintf(` + import A from %s + + pub contract B { + pub struct Bar : A.Foo {} + + pub struct interface Foo2{} + }`, + accounts[0].HexWithPrefix(), + ) + + contractC := fmt.Sprintf(` + import B from %s + import A from %s + + pub contract C { + pub struct Bar : A.Foo, B.Foo2 {} + + pub struct interface Foo3{} + }`, + accounts[0].HexWithPrefix(), + accounts[0].HexWithPrefix(), + ) + + contractD := fmt.Sprintf(` + import C from %s + import B from %s + import A from %s + + pub contract D { + pub struct Bar : A.Foo, B.Foo2, C.Foo3 {} + }`, + accounts[0].HexWithPrefix(), + accounts[0].HexWithPrefix(), + accounts[0].HexWithPrefix(), + ) + + var sequenceNumber uint64 = 0 + + runTransaction := func(code []byte) { + txBody := flow.NewTransactionBody(). + SetScript(code). + SetPayer(chain.ServiceAddress()). + SetProposalKey(chain.ServiceAddress(), 0, sequenceNumber). + AddAuthorizer(accounts[0]) + + _ = testutil.SignPayload(txBody, accounts[0], privateKeys[0]) + _ = testutil.SignEnvelope(txBody, chain.ServiceAddress(), unittest.ServiceAccountPrivateKey) + + executionSnapshot, output, err := vm.Run( + ctx, + fvm.Transaction(txBody, 0), + snapshotTree, + ) + require.NoError(t, err) + require.NoError(t, output.Err) + + snapshotTree = snapshotTree.Append(executionSnapshot) + + // increment sequence number + sequenceNumber++ + } + + // Deploy `A` + runTransaction(utils.DeploymentTransaction( + "A", + []byte(contractA), + )) + + // Deploy `B` + runTransaction(utils.DeploymentTransaction( + "B", + []byte(contractB), + )) + + // Deploy `C` + runTransaction(utils.DeploymentTransaction( + "C", + []byte(contractC), + )) + + // Deploy `D` + runTransaction(utils.DeploymentTransaction( + "D", + []byte(contractD), + )) + + // Store values + runTransaction([]byte(fmt.Sprintf( + ` + import D from %s + import C from %s + import B from %s + + transaction { + prepare(signer: AuthAccount) { + signer.save("Hello, World!", to: /storage/first) + signer.save(["one", "two", "three"], to: /storage/second) + signer.save(D.Bar(), to: /storage/third) + signer.save(C.Bar(), to: /storage/fourth) + signer.save(B.Bar(), to: /storage/fifth) + + signer.link<&String>(/private/a, target:/storage/first) + signer.link<&[String]>(/private/b, target:/storage/second) + signer.link<&D.Bar>(/private/c, target:/storage/third) + signer.link<&C.Bar>(/private/d, target:/storage/fourth) + signer.link<&B.Bar>(/private/e, target:/storage/fifth) + } + }`, + accounts[0].HexWithPrefix(), + accounts[0].HexWithPrefix(), + accounts[0].HexWithPrefix(), + ))) + + // Update `A`. `B`, `C` and `D` are now broken. + runTransaction(utils.UpdateTransaction( + "A", + []byte(updatedContractA), + )) + + // Iterate stored values + runTransaction([]byte( + ` + transaction { + prepare(account: AuthAccount) { + var total = 0 + account.forEachPrivate(fun (path: PrivatePath, type: Type): Bool { + account.getCapability<&AnyStruct>(path).borrow()! + total = total + 1 + return true + }) + + assert(total == 2, message:"found ".concat(total.toString())) + } + }`, + )) + }, + )(t) +} diff --git a/fvm/meter/memory_meter.go b/fvm/meter/memory_meter.go index dfc93b43301..c20fab45a4b 100644 --- a/fvm/meter/memory_meter.go +++ b/fvm/meter/memory_meter.go @@ -21,26 +21,29 @@ var ( common.MemoryKindNumberValue: 8, // weights for these values include the cost of the Go struct itself (first number) // as well as the overhead for creation of the underlying atree (second number) - common.MemoryKindArrayValueBase: 57 + 48, - common.MemoryKindDictionaryValueBase: 33 + 96, - common.MemoryKindCompositeValueBase: 233 + 96, - common.MemoryKindSimpleCompositeValue: 73, - common.MemoryKindSimpleCompositeValueBase: 89, - common.MemoryKindOptionalValue: 41, - common.MemoryKindTypeValue: 17, - common.MemoryKindPathValue: 24, - common.MemoryKindStorageCapabilityValue: 1, - common.MemoryKindPathLinkValue: 1, - common.MemoryKindAccountLinkValue: 1, - common.MemoryKindAccountReferenceValue: 1, - common.MemoryKindPublishedValue: 1, - common.MemoryKindStorageReferenceValue: 41, - common.MemoryKindEphemeralReferenceValue: 41, - common.MemoryKindInterpretedFunctionValue: 128, - common.MemoryKindHostFunctionValue: 41, - common.MemoryKindBoundFunctionValue: 25, - common.MemoryKindBigInt: 50, - common.MemoryKindVoidExpression: 1, + common.MemoryKindArrayValueBase: 57 + 48, + common.MemoryKindDictionaryValueBase: 33 + 96, + common.MemoryKindCompositeValueBase: 233 + 96, + common.MemoryKindSimpleCompositeValue: 73, + common.MemoryKindSimpleCompositeValueBase: 89, + common.MemoryKindOptionalValue: 41, + common.MemoryKindTypeValue: 17, + common.MemoryKindPathValue: 24, + common.MemoryKindPathCapabilityValue: 1, + common.MemoryKindIDCapabilityValue: 1, + common.MemoryKindPathLinkValue: 1, + common.MemoryKindStorageCapabilityControllerValue: 32, + common.MemoryKindAccountCapabilityControllerValue: 32, + common.MemoryKindAccountLinkValue: 1, + common.MemoryKindAccountReferenceValue: 1, + common.MemoryKindPublishedValue: 1, + common.MemoryKindStorageReferenceValue: 41, + common.MemoryKindEphemeralReferenceValue: 41, + common.MemoryKindInterpretedFunctionValue: 128, + common.MemoryKindHostFunctionValue: 41, + common.MemoryKindBoundFunctionValue: 25, + common.MemoryKindBigInt: 50, + common.MemoryKindVoidExpression: 1, // Atree @@ -69,38 +72,41 @@ var ( // Cadence Values - common.MemoryKindCadenceVoidValue: 1, - common.MemoryKindCadenceOptionalValue: 17, - common.MemoryKindCadenceBoolValue: 8, - common.MemoryKindCadenceStringValue: 16, - common.MemoryKindCadenceCharacterValue: 16, - common.MemoryKindCadenceAddressValue: 8, - common.MemoryKindCadenceIntValue: 50, - common.MemoryKindCadenceNumberValue: 1, - common.MemoryKindCadenceArrayValueBase: 41, - common.MemoryKindCadenceArrayValueLength: 16, - common.MemoryKindCadenceDictionaryValue: 41, - common.MemoryKindCadenceKeyValuePair: 33, - common.MemoryKindCadenceStructValueBase: 33, - common.MemoryKindCadenceStructValueSize: 16, - common.MemoryKindCadenceResourceValueBase: 33, - common.MemoryKindCadenceResourceValueSize: 16, - common.MemoryKindCadenceEventValueBase: 33, - common.MemoryKindCadenceEventValueSize: 16, - common.MemoryKindCadenceContractValueBase: 33, - common.MemoryKindCadenceContractValueSize: 16, - common.MemoryKindCadenceEnumValueBase: 33, - common.MemoryKindCadenceEnumValueSize: 16, - common.MemoryKindCadencePathLinkValue: 1, - common.MemoryKindCadencePathValue: 33, - common.MemoryKindCadenceTypeValue: 17, - common.MemoryKindCadenceStorageCapabilityValue: 1, - common.MemoryKindCadenceFunctionValue: 1, - common.MemoryKindCadenceAttachmentValueBase: 33, - common.MemoryKindCadenceAttachmentValueSize: 16, + common.MemoryKindCadenceVoidValue: 1, + common.MemoryKindCadenceOptionalValue: 17, + common.MemoryKindCadenceBoolValue: 8, + common.MemoryKindCadenceStringValue: 16, + common.MemoryKindCadenceCharacterValue: 16, + common.MemoryKindCadenceAddressValue: 8, + common.MemoryKindCadenceIntValue: 50, + common.MemoryKindCadenceNumberValue: 1, + common.MemoryKindCadenceArrayValueBase: 41, + common.MemoryKindCadenceArrayValueLength: 16, + common.MemoryKindCadenceDictionaryValue: 41, + common.MemoryKindCadenceKeyValuePair: 33, + common.MemoryKindCadenceStructValueBase: 33, + common.MemoryKindCadenceStructValueSize: 16, + common.MemoryKindCadenceResourceValueBase: 33, + common.MemoryKindCadenceResourceValueSize: 16, + common.MemoryKindCadenceEventValueBase: 33, + common.MemoryKindCadenceEventValueSize: 16, + common.MemoryKindCadenceContractValueBase: 33, + common.MemoryKindCadenceContractValueSize: 16, + common.MemoryKindCadenceEnumValueBase: 33, + common.MemoryKindCadenceEnumValueSize: 16, + common.MemoryKindCadencePathLinkValue: 1, + common.MemoryKindCadenceAccountLinkValue: 1, + common.MemoryKindCadencePathValue: 33, + common.MemoryKindCadenceTypeValue: 17, + common.MemoryKindCadencePathCapabilityValue: 1, + common.MemoryKindCadenceIDCapabilityValue: 1, + common.MemoryKindCadenceFunctionValue: 1, + common.MemoryKindCadenceAttachmentValueBase: 33, + common.MemoryKindCadenceAttachmentValueSize: 16, // Cadence Types + common.MemoryKindCadenceTypeParameter: 17, common.MemoryKindCadenceOptionalType: 17, common.MemoryKindCadenceVariableSizedArrayType: 17, common.MemoryKindCadenceConstantSizedArrayType: 25, diff --git a/fvm/mock/procedure.go b/fvm/mock/procedure.go index 6b3e7bb98fd..f4c2929490f 100644 --- a/fvm/mock/procedure.go +++ b/fvm/mock/procedure.go @@ -58,11 +58,11 @@ func (_m *Procedure) MemoryLimit(ctx fvm.Context) uint64 { } // NewExecutor provides a mock function with given fields: ctx, txnState -func (_m *Procedure) NewExecutor(ctx fvm.Context, txnState storage.Transaction) fvm.ProcedureExecutor { +func (_m *Procedure) NewExecutor(ctx fvm.Context, txnState storage.TransactionPreparer) fvm.ProcedureExecutor { ret := _m.Called(ctx, txnState) var r0 fvm.ProcedureExecutor - if rf, ok := ret.Get(0).(func(fvm.Context, storage.Transaction) fvm.ProcedureExecutor); ok { + if rf, ok := ret.Get(0).(func(fvm.Context, storage.TransactionPreparer) fvm.ProcedureExecutor); ok { r0 = rf(ctx, txnState) } else { if ret.Get(0) != nil { @@ -73,11 +73,6 @@ func (_m *Procedure) NewExecutor(ctx fvm.Context, txnState storage.Transaction) return r0 } -// SetOutput provides a mock function with given fields: output -func (_m *Procedure) SetOutput(output fvm.ProcedureOutput) { - _m.Called(output) -} - // ShouldDisableMemoryAndInteractionLimits provides a mock function with given fields: ctx func (_m *Procedure) ShouldDisableMemoryAndInteractionLimits(ctx fvm.Context) bool { ret := _m.Called(ctx) diff --git a/fvm/mock/vm.go b/fvm/mock/vm.go index 6a70e4ef083..1f836fd9836 100644 --- a/fvm/mock/vm.go +++ b/fvm/mock/vm.go @@ -8,7 +8,9 @@ import ( mock "github.com/stretchr/testify/mock" - state "github.com/onflow/flow-go/fvm/state" + snapshot "github.com/onflow/flow-go/fvm/storage/snapshot" + + storage "github.com/onflow/flow-go/fvm/storage" ) // VM is an autogenerated mock type for the VM type @@ -17,15 +19,15 @@ type VM struct { } // GetAccount provides a mock function with given fields: _a0, _a1, _a2 -func (_m *VM) GetAccount(_a0 fvm.Context, _a1 flow.Address, _a2 state.StorageSnapshot) (*flow.Account, error) { +func (_m *VM) GetAccount(_a0 fvm.Context, _a1 flow.Address, _a2 snapshot.StorageSnapshot) (*flow.Account, error) { ret := _m.Called(_a0, _a1, _a2) var r0 *flow.Account var r1 error - if rf, ok := ret.Get(0).(func(fvm.Context, flow.Address, state.StorageSnapshot) (*flow.Account, error)); ok { + if rf, ok := ret.Get(0).(func(fvm.Context, flow.Address, snapshot.StorageSnapshot) (*flow.Account, error)); ok { return rf(_a0, _a1, _a2) } - if rf, ok := ret.Get(0).(func(fvm.Context, flow.Address, state.StorageSnapshot) *flow.Account); ok { + if rf, ok := ret.Get(0).(func(fvm.Context, flow.Address, snapshot.StorageSnapshot) *flow.Account); ok { r0 = rf(_a0, _a1, _a2) } else { if ret.Get(0) != nil { @@ -33,7 +35,7 @@ func (_m *VM) GetAccount(_a0 fvm.Context, _a1 flow.Address, _a2 state.StorageSna } } - if rf, ok := ret.Get(1).(func(fvm.Context, flow.Address, state.StorageSnapshot) error); ok { + if rf, ok := ret.Get(1).(func(fvm.Context, flow.Address, snapshot.StorageSnapshot) error); ok { r1 = rf(_a0, _a1, _a2) } else { r1 = ret.Error(1) @@ -42,31 +44,47 @@ func (_m *VM) GetAccount(_a0 fvm.Context, _a1 flow.Address, _a2 state.StorageSna return r0, r1 } +// NewExecutor provides a mock function with given fields: _a0, _a1, _a2 +func (_m *VM) NewExecutor(_a0 fvm.Context, _a1 fvm.Procedure, _a2 storage.TransactionPreparer) fvm.ProcedureExecutor { + ret := _m.Called(_a0, _a1, _a2) + + var r0 fvm.ProcedureExecutor + if rf, ok := ret.Get(0).(func(fvm.Context, fvm.Procedure, storage.TransactionPreparer) fvm.ProcedureExecutor); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(fvm.ProcedureExecutor) + } + } + + return r0 +} + // Run provides a mock function with given fields: _a0, _a1, _a2 -func (_m *VM) Run(_a0 fvm.Context, _a1 fvm.Procedure, _a2 state.StorageSnapshot) (*state.ExecutionSnapshot, fvm.ProcedureOutput, error) { +func (_m *VM) Run(_a0 fvm.Context, _a1 fvm.Procedure, _a2 snapshot.StorageSnapshot) (*snapshot.ExecutionSnapshot, fvm.ProcedureOutput, error) { ret := _m.Called(_a0, _a1, _a2) - var r0 *state.ExecutionSnapshot + var r0 *snapshot.ExecutionSnapshot var r1 fvm.ProcedureOutput var r2 error - if rf, ok := ret.Get(0).(func(fvm.Context, fvm.Procedure, state.StorageSnapshot) (*state.ExecutionSnapshot, fvm.ProcedureOutput, error)); ok { + if rf, ok := ret.Get(0).(func(fvm.Context, fvm.Procedure, snapshot.StorageSnapshot) (*snapshot.ExecutionSnapshot, fvm.ProcedureOutput, error)); ok { return rf(_a0, _a1, _a2) } - if rf, ok := ret.Get(0).(func(fvm.Context, fvm.Procedure, state.StorageSnapshot) *state.ExecutionSnapshot); ok { + if rf, ok := ret.Get(0).(func(fvm.Context, fvm.Procedure, snapshot.StorageSnapshot) *snapshot.ExecutionSnapshot); ok { r0 = rf(_a0, _a1, _a2) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*state.ExecutionSnapshot) + r0 = ret.Get(0).(*snapshot.ExecutionSnapshot) } } - if rf, ok := ret.Get(1).(func(fvm.Context, fvm.Procedure, state.StorageSnapshot) fvm.ProcedureOutput); ok { + if rf, ok := ret.Get(1).(func(fvm.Context, fvm.Procedure, snapshot.StorageSnapshot) fvm.ProcedureOutput); ok { r1 = rf(_a0, _a1, _a2) } else { r1 = ret.Get(1).(fvm.ProcedureOutput) } - if rf, ok := ret.Get(2).(func(fvm.Context, fvm.Procedure, state.StorageSnapshot) error); ok { + if rf, ok := ret.Get(2).(func(fvm.Context, fvm.Procedure, snapshot.StorageSnapshot) error); ok { r2 = rf(_a0, _a1, _a2) } else { r2 = ret.Error(2) diff --git a/fvm/runtime/wrapped_cadence_runtime.go b/fvm/runtime/wrapped_cadence_runtime.go index 9e8c695d0a5..48f7488162f 100644 --- a/fvm/runtime/wrapped_cadence_runtime.go +++ b/fvm/runtime/wrapped_cadence_runtime.go @@ -68,6 +68,7 @@ func (wr WrappedCadenceRuntime) ReadStored(address common.Address, path cadence. } func (wr WrappedCadenceRuntime) ReadLinked(address common.Address, path cadence.Path, context runtime.Context) (cadence.Value, error) { + //nolint:staticcheck v, err := wr.Runtime.ReadLinked(address, path, context) return v, errors.HandleRuntimeError(err) } diff --git a/fvm/script.go b/fvm/script.go index 5371c413845..c979fb309f5 100644 --- a/fvm/script.go +++ b/fvm/script.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/hashicorp/go-multierror" "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" @@ -20,9 +21,6 @@ type ScriptProcedure struct { Script []byte Arguments [][]byte RequestContext context.Context - - // TODO(patrick): remove - ProcedureOutput } func Script(code []byte) *ScriptProcedure { @@ -71,15 +69,11 @@ func NewScriptWithContextAndArgs( func (proc *ScriptProcedure) NewExecutor( ctx Context, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) ProcedureExecutor { return newScriptExecutor(ctx, proc, txnState) } -func (proc *ScriptProcedure) SetOutput(output ProcedureOutput) { - proc.ProcedureOutput = output -} - func (proc *ScriptProcedure) ComputationLimit(ctx Context) uint64 { computationLimit := ctx.ComputationLimit // if ctx.ComputationLimit is also zero, fallback to the default computation limit @@ -115,7 +109,7 @@ func (proc *ScriptProcedure) ExecutionTime() logical.Time { type scriptExecutor struct { ctx Context proc *ScriptProcedure - txnState storage.Transaction + txnState storage.TransactionPreparer env environment.Environment @@ -125,7 +119,7 @@ type scriptExecutor struct { func newScriptExecutor( ctx Context, proc *ScriptProcedure, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) *scriptExecutor { return &scriptExecutor{ ctx: ctx, @@ -205,11 +199,13 @@ func (executor *scriptExecutor) executeScript() error { Source: executor.proc.Script, Arguments: executor.proc.Arguments, }, - common.ScriptLocation(executor.proc.ID)) + common.ScriptLocation(executor.proc.ID), + ) + populateErr := executor.output.PopulateEnvironmentValues(executor.env) if err != nil { - return err + return multierror.Append(err, populateErr) } executor.output.Value = value - return executor.output.PopulateEnvironmentValues(executor.env) + return populateErr } diff --git a/fvm/storage/block_database.go b/fvm/storage/block_database.go new file mode 100644 index 00000000000..3d1922825d9 --- /dev/null +++ b/fvm/storage/block_database.go @@ -0,0 +1,118 @@ +package storage + +import ( + "fmt" + + "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/primary" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" +) + +// BlockDatabase packages the primary index (BlockData) and secondary indices +// (DerivedBlockData) into a single database via 2PC. +type BlockDatabase struct { + *primary.BlockData + *derived.DerivedBlockData +} + +type transaction struct { + *primary.TransactionData + *derived.DerivedTransactionData +} + +// NOTE: storageSnapshot must be thread safe. +func NewBlockDatabase( + storageSnapshot snapshot.StorageSnapshot, + snapshotTime logical.Time, + cachedDerivedBlockData *derived.DerivedBlockData, // optional +) *BlockDatabase { + derivedBlockData := cachedDerivedBlockData + if derivedBlockData == nil { + derivedBlockData = derived.NewEmptyDerivedBlockData(snapshotTime) + } + + return &BlockDatabase{ + BlockData: primary.NewBlockData(storageSnapshot, snapshotTime), + DerivedBlockData: derivedBlockData, + } +} + +func (database *BlockDatabase) NewTransaction( + executionTime logical.Time, + parameters state.StateParameters, +) ( + Transaction, + error, +) { + primaryTxn, err := database.BlockData.NewTransactionData( + executionTime, + parameters) + if err != nil { + return nil, fmt.Errorf("failed to create primary transaction: %w", err) + } + + derivedTxn, err := database.DerivedBlockData.NewDerivedTransactionData( + primaryTxn.SnapshotTime(), + executionTime) + if err != nil { + return nil, fmt.Errorf("failed to create dervied transaction: %w", err) + } + + return &transaction{ + TransactionData: primaryTxn, + DerivedTransactionData: derivedTxn, + }, nil +} + +func (database *BlockDatabase) NewSnapshotReadTransaction( + parameters state.StateParameters, +) Transaction { + + return &transaction{ + TransactionData: database.BlockData. + NewSnapshotReadTransactionData(parameters), + DerivedTransactionData: database.DerivedBlockData. + NewSnapshotReadDerivedTransactionData(), + } +} + +func (txn *transaction) Validate() error { + err := txn.DerivedTransactionData.Validate() + if err != nil { + return fmt.Errorf("derived indices validate failed: %w", err) + } + + // NOTE: Since the primary txn's SnapshotTime() is exposed to the user, + // the primary txn should be validated last to prevent primary txn' + // snapshot time advancement in case of derived txn validation failure. + err = txn.TransactionData.Validate() + if err != nil { + return fmt.Errorf("primary index validate failed: %w", err) + } + + return nil +} + +func (txn *transaction) Finalize() error { + // NOTE: DerivedTransactionData does not need to be finalized. + return txn.TransactionData.Finalize() +} + +func (txn *transaction) Commit() (*snapshot.ExecutionSnapshot, error) { + err := txn.DerivedTransactionData.Commit() + if err != nil { + return nil, fmt.Errorf("derived indices commit failed: %w", err) + } + + // NOTE: Since the primary txn's SnapshotTime() is exposed to the user, + // the primary txn should be committed last to prevent primary txn' + // snapshot time advancement in case of derived txn commit failure. + executionSnapshot, err := txn.TransactionData.Commit() + if err != nil { + return nil, fmt.Errorf("primary index commit failed: %w", err) + } + + return executionSnapshot, nil +} diff --git a/fvm/storage/derived/derived_block_data.go b/fvm/storage/derived/derived_block_data.go index 993399e13ef..f39c3a1553a 100644 --- a/fvm/storage/derived/derived_block_data.go +++ b/fvm/storage/derived/derived_block_data.go @@ -6,13 +6,13 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/state" ) -type DerivedTransaction interface { +type DerivedTransactionPreparer interface { GetOrComputeProgram( - txState state.NestedTransaction, + txState state.NestedTransactionPreparer, addressLocation common.AddressLocation, programComputer ValueComputer[common.AddressLocation, *Program], ) ( @@ -22,7 +22,7 @@ type DerivedTransaction interface { GetProgram(location common.AddressLocation) (*Program, bool) GetMeterParamOverrides( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, getMeterParamOverrides ValueComputer[struct{}, MeterParamOverrides], ) ( MeterParamOverrides, @@ -32,13 +32,6 @@ type DerivedTransaction interface { AddInvalidator(invalidator TransactionInvalidator) } -type DerivedTransactionCommitter interface { - DerivedTransaction - - Validate() error - Commit() error -} - type Program struct { *interpreter.Program @@ -66,31 +59,18 @@ type DerivedTransactionData struct { meterParamOverrides *TableTransaction[struct{}, MeterParamOverrides] } -func NewEmptyDerivedBlockData() *DerivedBlockData { +func NewEmptyDerivedBlockData( + initialSnapshotTime logical.Time, +) *DerivedBlockData { return &DerivedBlockData{ programs: NewEmptyTable[ common.AddressLocation, *Program, - ](), + ](initialSnapshotTime), meterParamOverrides: NewEmptyTable[ struct{}, MeterParamOverrides, - ](), - } -} - -// This variant is needed by the chunk verifier, which does not start at the -// beginning of the block. -func NewEmptyDerivedBlockDataWithTransactionOffset(offset uint32) *DerivedBlockData { - return &DerivedBlockData{ - programs: NewEmptyTableWithOffset[ - common.AddressLocation, - *Program, - ](offset), - meterParamOverrides: NewEmptyTableWithOffset[ - struct{}, - MeterParamOverrides, - ](offset), + ](initialSnapshotTime), } } @@ -101,38 +81,22 @@ func (block *DerivedBlockData) NewChildDerivedBlockData() *DerivedBlockData { } } -func (block *DerivedBlockData) NewSnapshotReadDerivedTransactionData( - snapshotTime logical.Time, - executionTime logical.Time, -) ( - DerivedTransactionCommitter, - error, -) { - txnPrograms, err := block.programs.NewSnapshotReadTableTransaction( - snapshotTime, - executionTime) - if err != nil { - return nil, err - } +func (block *DerivedBlockData) NewSnapshotReadDerivedTransactionData() *DerivedTransactionData { + txnPrograms := block.programs.NewSnapshotReadTableTransaction() - txnMeterParamOverrides, err := block.meterParamOverrides.NewSnapshotReadTableTransaction( - snapshotTime, - executionTime) - if err != nil { - return nil, err - } + txnMeterParamOverrides := block.meterParamOverrides.NewSnapshotReadTableTransaction() return &DerivedTransactionData{ programs: txnPrograms, meterParamOverrides: txnMeterParamOverrides, - }, nil + } } func (block *DerivedBlockData) NewDerivedTransactionData( snapshotTime logical.Time, executionTime logical.Time, ) ( - DerivedTransactionCommitter, + *DerivedTransactionData, error, ) { txnPrograms, err := block.programs.NewTableTransaction( @@ -174,7 +138,7 @@ func (block *DerivedBlockData) CachedPrograms() int { } func (transaction *DerivedTransactionData) GetOrComputeProgram( - txState state.NestedTransaction, + txState state.NestedTransactionPreparer, addressLocation common.AddressLocation, programComputer ValueComputer[common.AddressLocation, *Program], ) ( @@ -213,7 +177,7 @@ func (transaction *DerivedTransactionData) AddInvalidator( } func (transaction *DerivedTransactionData) GetMeterParamOverrides( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, getMeterParamOverrides ValueComputer[struct{}, MeterParamOverrides], ) ( MeterParamOverrides, diff --git a/fvm/storage/derived/derived_chain_data.go b/fvm/storage/derived/derived_chain_data.go index 18d55eae5d2..a3ec9a488df 100644 --- a/fvm/storage/derived/derived_chain_data.go +++ b/fvm/storage/derived/derived_chain_data.go @@ -72,7 +72,7 @@ func (chain *DerivedChainData) GetOrCreateDerivedBlockData( if ok { current = parentEntry.(*DerivedBlockData).NewChildDerivedBlockData() } else { - current = NewEmptyDerivedBlockData() + current = NewEmptyDerivedBlockData(0) } chain.lru.Add(currentBlockId, current) @@ -87,5 +87,5 @@ func (chain *DerivedChainData) NewDerivedBlockDataForScript( return block.NewChildDerivedBlockData() } - return NewEmptyDerivedBlockData() + return NewEmptyDerivedBlockData(0) } diff --git a/fvm/storage/derived/derived_chain_data_test.go b/fvm/storage/derived/derived_chain_data_test.go index b45e2f232f8..0c79af2f603 100644 --- a/fvm/storage/derived/derived_chain_data_test.go +++ b/fvm/storage/derived/derived_chain_data_test.go @@ -8,8 +8,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/execution/state/delta" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/model/flow" ) @@ -47,12 +46,11 @@ func TestDerivedChainData(t *testing.T) { txn, err := block1.NewDerivedTransactionData(0, 0) require.NoError(t, err) - view := delta.NewDeltaView(nil) - txState := state.NewTransactionState(view, state.DefaultParameters()) + txState := state.NewTransactionState(nil, state.DefaultParameters()) _, err = txn.GetOrComputeProgram(txState, loc1, newProgramLoader( func( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, key common.AddressLocation, ) (*Program, error) { return prog1, nil @@ -83,12 +81,11 @@ func TestDerivedChainData(t *testing.T) { txn, err = block2.NewDerivedTransactionData(0, 0) require.NoError(t, err) - view = delta.NewDeltaView(nil) - txState = state.NewTransactionState(view, state.DefaultParameters()) + txState = state.NewTransactionState(nil, state.DefaultParameters()) _, err = txn.GetOrComputeProgram(txState, loc2, newProgramLoader( func( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, key common.AddressLocation, ) (*Program, error) { return prog2, nil @@ -185,7 +182,7 @@ func TestDerivedChainData(t *testing.T) { type programLoader struct { f func( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, key common.AddressLocation, ) (*Program, error) } @@ -194,7 +191,7 @@ var _ ValueComputer[common.AddressLocation, *Program] = &programLoader{} func newProgramLoader( f func( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, key common.AddressLocation, ) (*Program, error), ) *programLoader { @@ -204,7 +201,7 @@ func newProgramLoader( } func (p *programLoader) Compute( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, key common.AddressLocation, ) (*Program, error) { return p.f(txnState, key) diff --git a/fvm/storage/derived/error.go b/fvm/storage/derived/error.go deleted file mode 100644 index a07840eb532..00000000000 --- a/fvm/storage/derived/error.go +++ /dev/null @@ -1,34 +0,0 @@ -package derived - -import ( - "fmt" -) - -type RetryableError interface { - error - IsRetryable() bool -} - -type retryableError struct { - error - - isRetryable bool -} - -func newRetryableError(msg string, vals ...interface{}) RetryableError { - return retryableError{ - error: fmt.Errorf(msg, vals...), - isRetryable: true, - } -} - -func newNotRetryableError(msg string, vals ...interface{}) RetryableError { - return retryableError{ - error: fmt.Errorf(msg, vals...), - isRetryable: false, - } -} - -func (err retryableError) IsRetryable() bool { - return err.isRetryable -} diff --git a/fvm/storage/derived/table.go b/fvm/storage/derived/table.go index 663a4276b99..91d7153dcb4 100644 --- a/fvm/storage/derived/table.go +++ b/fvm/storage/derived/table.go @@ -6,19 +6,21 @@ import ( "github.com/hashicorp/go-multierror" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/errors" "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" ) // ValueComputer is used by DerivedDataTable's GetOrCompute to compute the // derived value when the value is not in DerivedDataTable (i.e., "cache miss"). type ValueComputer[TKey any, TVal any] interface { - Compute(txnState state.NestedTransaction, key TKey) (TVal, error) + Compute(txnState state.NestedTransactionPreparer, key TKey) (TVal, error) } type invalidatableEntry[TVal any] struct { - Value TVal // immutable after initialization. - ExecutionSnapshot *state.ExecutionSnapshot // immutable after initialization. + Value TVal // immutable after initialization. + ExecutionSnapshot *snapshot.ExecutionSnapshot // immutable after initialization. isInvalid bool // Guarded by DerivedDataTable' lock. } @@ -77,31 +79,19 @@ type TableTransaction[TKey comparable, TVal any] struct { invalidators chainedTableInvalidators[TKey, TVal] } -func newEmptyTable[TKey comparable, TVal any]( - latestCommit logical.Time, +func NewEmptyTable[ + TKey comparable, + TVal any, +]( + initialSnapshotTime logical.Time, ) *DerivedDataTable[TKey, TVal] { return &DerivedDataTable[TKey, TVal]{ items: map[TKey]*invalidatableEntry[TVal]{}, - latestCommitExecutionTime: latestCommit, + latestCommitExecutionTime: initialSnapshotTime - 1, invalidators: nil, } } -func NewEmptyTable[TKey comparable, TVal any]() *DerivedDataTable[TKey, TVal] { - return newEmptyTable[TKey, TVal](logical.ParentBlockTime) -} - -// This variant is needed by the chunk verifier, which does not start at the -// beginning of the block. -func NewEmptyTableWithOffset[ - TKey comparable, - TVal any, -]( - offset uint32, -) *DerivedDataTable[TKey, TVal] { - return newEmptyTable[TKey, TVal](logical.Time(offset) - 1) -} - func (table *DerivedDataTable[TKey, TVal]) NewChildTable() *DerivedDataTable[TKey, TVal] { table.lock.RLock() defer table.lock.RUnlock() @@ -177,16 +167,16 @@ func (table *DerivedDataTable[TKey, TVal]) get( func (table *DerivedDataTable[TKey, TVal]) unsafeValidate( txn *TableTransaction[TKey, TVal], -) RetryableError { +) error { if txn.isSnapshotReadTransaction && txn.invalidators.ShouldInvalidateEntries() { - return newNotRetryableError( + return fmt.Errorf( "invalid TableTransaction: snapshot read can't invalidate") } if table.latestCommitExecutionTime >= txn.executionTime { - return newNotRetryableError( + return fmt.Errorf( "invalid TableTransaction: non-increasing time (%v >= %v)", table.latestCommitExecutionTime, txn.executionTime) @@ -194,8 +184,15 @@ func (table *DerivedDataTable[TKey, TVal]) unsafeValidate( for _, entry := range txn.readSet { if entry.isInvalid { - return newRetryableError( - "invalid TableTransactions. outdated read set") + if txn.snapshotTime == txn.executionTime { + // This should never happen since the transaction is + // sequentially executed. + return fmt.Errorf( + "invalid TableTransaction: unrecoverable outdated read set") + } + + return errors.NewRetryableConflictError( + "invalid TableTransaction: outdated read set") } } @@ -208,8 +205,16 @@ func (table *DerivedDataTable[TKey, TVal]) unsafeValidate( entry.Value, entry.ExecutionSnapshot) { - return newRetryableError( - "invalid TableTransactions. outdated write set") + if txn.snapshotTime == txn.executionTime { + // This should never happen since the transaction is + // sequentially executed. + return fmt.Errorf( + "invalid TableTransaction: unrecoverable outdated " + + "write set") + } + + return errors.NewRetryableConflictError( + "invalid TableTransaction: outdated write set") } } } @@ -221,7 +226,7 @@ func (table *DerivedDataTable[TKey, TVal]) unsafeValidate( func (table *DerivedDataTable[TKey, TVal]) validate( txn *TableTransaction[TKey, TVal], -) RetryableError { +) error { table.lock.RLock() defer table.lock.RUnlock() @@ -230,15 +235,14 @@ func (table *DerivedDataTable[TKey, TVal]) validate( func (table *DerivedDataTable[TKey, TVal]) commit( txn *TableTransaction[TKey, TVal], -) RetryableError { +) error { table.lock.Lock() defer table.lock.Unlock() - if table.latestCommitExecutionTime+1 < txn.snapshotTime && - (!txn.isSnapshotReadTransaction || - txn.snapshotTime != logical.EndOfBlockExecutionTime) { + if !txn.isSnapshotReadTransaction && + table.latestCommitExecutionTime+1 < txn.snapshotTime { - return newNotRetryableError( + return fmt.Errorf( "invalid TableTransaction: missing commit range [%v, %v)", table.latestCommitExecutionTime+1, txn.snapshotTime) @@ -251,6 +255,12 @@ func (table *DerivedDataTable[TKey, TVal]) commit( return err } + // Don't perform actual commit for snapshot read transaction. This is + // safe since all values are derived from the primary source. + if txn.isSnapshotReadTransaction { + return nil + } + for key, entry := range txn.writeSet { _, ok := table.items[key] if ok { @@ -280,38 +290,15 @@ func (table *DerivedDataTable[TKey, TVal]) commit( txn.invalidators...) } - // NOTE: We cannot advance commit time when we encounter a snapshot read - // (aka script) transaction since these transactions don't generate new - // snapshots. It is safe to commit the entries since snapshot read - // transactions never invalidate entries. - if !txn.isSnapshotReadTransaction { - table.latestCommitExecutionTime = txn.executionTime - } + table.latestCommitExecutionTime = txn.executionTime return nil } func (table *DerivedDataTable[TKey, TVal]) newTableTransaction( - upperBoundExecutionTime logical.Time, snapshotTime logical.Time, executionTime logical.Time, isSnapshotReadTransaction bool, -) ( - *TableTransaction[TKey, TVal], - error, -) { - if executionTime < 0 || executionTime > upperBoundExecutionTime { - return nil, fmt.Errorf( - "invalid TableTransactions: execution time out of bound: %v", - executionTime) - } - - if snapshotTime > executionTime { - return nil, fmt.Errorf( - "invalid TableTransactions: snapshot > execution: %v > %v", - snapshotTime, - executionTime) - } - +) *TableTransaction[TKey, TVal] { return &TableTransaction[TKey, TVal]{ table: table, snapshotTime: snapshotTime, @@ -320,20 +307,13 @@ func (table *DerivedDataTable[TKey, TVal]) newTableTransaction( readSet: map[TKey]*invalidatableEntry[TVal]{}, writeSet: map[TKey]*invalidatableEntry[TVal]{}, isSnapshotReadTransaction: isSnapshotReadTransaction, - }, nil + } } -func (table *DerivedDataTable[TKey, TVal]) NewSnapshotReadTableTransaction( - snapshotTime logical.Time, - executionTime logical.Time, -) ( - *TableTransaction[TKey, TVal], - error, -) { +func (table *DerivedDataTable[TKey, TVal]) NewSnapshotReadTableTransaction() *TableTransaction[TKey, TVal] { return table.newTableTransaction( - logical.LargestSnapshotReadTransactionExecutionTime, - snapshotTime, - executionTime, + logical.EndOfBlockExecutionTime, + logical.EndOfBlockExecutionTime, true) } @@ -344,17 +324,31 @@ func (table *DerivedDataTable[TKey, TVal]) NewTableTransaction( *TableTransaction[TKey, TVal], error, ) { + if executionTime < 0 || + executionTime > logical.LargestNormalTransactionExecutionTime { + + return nil, fmt.Errorf( + "invalid TableTransactions: execution time out of bound: %v", + executionTime) + } + + if snapshotTime > executionTime { + return nil, fmt.Errorf( + "invalid TableTransactions: snapshot > execution: %v > %v", + snapshotTime, + executionTime) + } + return table.newTableTransaction( - logical.LargestNormalTransactionExecutionTime, snapshotTime, executionTime, - false) + false), nil } // Note: use GetOrCompute instead of Get/Set whenever possible. func (txn *TableTransaction[TKey, TVal]) get(key TKey) ( TVal, - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, bool, ) { @@ -380,7 +374,7 @@ func (txn *TableTransaction[TKey, TVal]) get(key TKey) ( func (txn *TableTransaction[TKey, TVal]) GetForTestingOnly(key TKey) ( TVal, - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, bool, ) { return txn.get(key) @@ -389,7 +383,7 @@ func (txn *TableTransaction[TKey, TVal]) GetForTestingOnly(key TKey) ( func (txn *TableTransaction[TKey, TVal]) set( key TKey, value TVal, - snapshot *state.ExecutionSnapshot, + snapshot *snapshot.ExecutionSnapshot, ) { txn.writeSet[key] = &invalidatableEntry[TVal]{ Value: value, @@ -405,7 +399,7 @@ func (txn *TableTransaction[TKey, TVal]) set( func (txn *TableTransaction[TKey, TVal]) SetForTestingOnly( key TKey, value TVal, - snapshot *state.ExecutionSnapshot, + snapshot *snapshot.ExecutionSnapshot, ) { txn.set(key, value, snapshot) } @@ -418,7 +412,7 @@ func (txn *TableTransaction[TKey, TVal]) SetForTestingOnly( // Note: valFunc must be an idempotent function and it must not modify // txnState's values. func (txn *TableTransaction[TKey, TVal]) GetOrCompute( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, key TKey, computer ValueComputer[TKey, TVal], ) ( @@ -478,11 +472,11 @@ func (txn *TableTransaction[TKey, TVal]) AddInvalidator( }) } -func (txn *TableTransaction[TKey, TVal]) Validate() RetryableError { +func (txn *TableTransaction[TKey, TVal]) Validate() error { return txn.table.validate(txn) } -func (txn *TableTransaction[TKey, TVal]) Commit() RetryableError { +func (txn *TableTransaction[TKey, TVal]) Commit() error { return txn.table.commit(txn) } diff --git a/fvm/storage/derived/table_invalidator.go b/fvm/storage/derived/table_invalidator.go index 93e15769802..d0a8cc8ef0f 100644 --- a/fvm/storage/derived/table_invalidator.go +++ b/fvm/storage/derived/table_invalidator.go @@ -1,8 +1,8 @@ package derived import ( - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" ) type TableInvalidator[TKey comparable, TVal any] interface { @@ -10,7 +10,7 @@ type TableInvalidator[TKey comparable, TVal any] interface { ShouldInvalidateEntries() bool // This returns true if the table entry should be invalidated. - ShouldInvalidateEntry(TKey, TVal, *state.ExecutionSnapshot) bool + ShouldInvalidateEntry(TKey, TVal, *snapshot.ExecutionSnapshot) bool } type tableInvalidatorAtTime[TKey comparable, TVal any] struct { @@ -50,7 +50,7 @@ func (chained chainedTableInvalidators[TKey, TVal]) ShouldInvalidateEntries() bo func (chained chainedTableInvalidators[TKey, TVal]) ShouldInvalidateEntry( key TKey, value TVal, - snapshot *state.ExecutionSnapshot, + snapshot *snapshot.ExecutionSnapshot, ) bool { for _, invalidator := range chained { if invalidator.ShouldInvalidateEntry(key, value, snapshot) { diff --git a/fvm/storage/derived/table_invalidator_test.go b/fvm/storage/derived/table_invalidator_test.go index 98d69724eef..6fa4d7940d2 100644 --- a/fvm/storage/derived/table_invalidator_test.go +++ b/fvm/storage/derived/table_invalidator_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" ) type testInvalidator struct { @@ -22,7 +22,7 @@ func (invalidator testInvalidator) ShouldInvalidateEntries() bool { func (invalidator *testInvalidator) ShouldInvalidateEntry( key string, value *string, - snapshot *state.ExecutionSnapshot, + snapshot *snapshot.ExecutionSnapshot, ) bool { invalidator.callCount += 1 return invalidator.invalidateAll || diff --git a/fvm/storage/derived/table_test.go b/fvm/storage/derived/table_test.go index 6f5f7511793..2d131c0f500 100644 --- a/fvm/storage/derived/table_test.go +++ b/fvm/storage/derived/table_test.go @@ -7,18 +7,19 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/execution/state/delta" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/errors" "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/model/flow" ) func newEmptyTestBlock() *DerivedDataTable[string, *string] { - return NewEmptyTable[string, *string]() + return NewEmptyTable[string, *string](0) } func TestDerivedDataTableWithTransactionOffset(t *testing.T) { - block := NewEmptyTableWithOffset[string, *string](18) + block := NewEmptyTable[string, *string](18) require.Equal( t, @@ -60,30 +61,8 @@ func TestDerivedDataTableNormalTransactionInvalidSnapshotTime(t *testing.T) { require.NoError(t, err) } -func TestDerivedDataTableSnapshotReadTransactionInvalidExecutionTimeBound( - t *testing.T, -) { - block := newEmptyTestBlock() - - _, err := block.NewSnapshotReadTableTransaction( - logical.ParentBlockTime, - logical.ParentBlockTime) - require.ErrorContains(t, err, "execution time out of bound") - - _, err = block.NewSnapshotReadTableTransaction(logical.ParentBlockTime, 0) - require.NoError(t, err) - - _, err = block.NewSnapshotReadTableTransaction(0, logical.ChildBlockTime) - require.ErrorContains(t, err, "execution time out of bound") - - _, err = block.NewSnapshotReadTableTransaction( - 0, - logical.EndOfBlockExecutionTime) - require.NoError(t, err) -} - func TestDerivedDataTableToValidateTime(t *testing.T) { - block := NewEmptyTableWithOffset[string, *string](8) + block := NewEmptyTable[string, *string](8) require.Equal( t, logical.Time(7), @@ -292,7 +271,7 @@ func TestDerivedDataTableValidateRejectOutOfOrderCommit(t *testing.T) { validateErr = testTxn.Validate() require.ErrorContains(t, validateErr, "non-increasing time") - require.False(t, validateErr.IsRetryable()) + require.False(t, errors.IsRetryableConflictError(validateErr)) } func TestDerivedDataTableValidateRejectNonIncreasingExecutionTime(t *testing.T) { @@ -309,7 +288,7 @@ func TestDerivedDataTableValidateRejectNonIncreasingExecutionTime(t *testing.T) validateErr := testTxn.Validate() require.ErrorContains(t, validateErr, "non-increasing time") - require.False(t, validateErr.IsRetryable()) + require.False(t, errors.IsRetryableConflictError(validateErr)) } func TestDerivedDataTableValidateRejectOutdatedReadSet(t *testing.T) { @@ -327,7 +306,7 @@ func TestDerivedDataTableValidateRejectOutdatedReadSet(t *testing.T) { key := "abc" valueString := "value" expectedValue := &valueString - expectedSnapshot := &state.ExecutionSnapshot{} + expectedSnapshot := &snapshot.ExecutionSnapshot{} testSetupTxn1.SetForTestingOnly(key, expectedValue, expectedSnapshot) @@ -354,7 +333,7 @@ func TestDerivedDataTableValidateRejectOutdatedReadSet(t *testing.T) { validateErr = testTxn.Validate() require.ErrorContains(t, validateErr, "outdated read set") - require.True(t, validateErr.IsRetryable()) + require.True(t, errors.IsRetryableConflictError(validateErr)) } func TestDerivedDataTableValidateRejectOutdatedWriteSet(t *testing.T) { @@ -374,11 +353,11 @@ func TestDerivedDataTableValidateRejectOutdatedWriteSet(t *testing.T) { require.NoError(t, err) value := "value" - testTxn.SetForTestingOnly("key", &value, &state.ExecutionSnapshot{}) + testTxn.SetForTestingOnly("key", &value, &snapshot.ExecutionSnapshot{}) validateErr := testTxn.Validate() require.ErrorContains(t, validateErr, "outdated write set") - require.True(t, validateErr.IsRetryable()) + require.True(t, errors.IsRetryableConflictError(validateErr)) } func TestDerivedDataTableValidateIgnoreInvalidatorsOlderThanSnapshot(t *testing.T) { @@ -397,60 +376,12 @@ func TestDerivedDataTableValidateIgnoreInvalidatorsOlderThanSnapshot(t *testing. require.NoError(t, err) value := "value" - testTxn.SetForTestingOnly("key", &value, &state.ExecutionSnapshot{}) + testTxn.SetForTestingOnly("key", &value, &snapshot.ExecutionSnapshot{}) err = testTxn.Validate() require.NoError(t, err) } -func TestDerivedDataTableCommitEndOfBlockSnapshotRead(t *testing.T) { - block := newEmptyTestBlock() - - commitTime := logical.Time(5) - testSetupTxn, err := block.NewTableTransaction(0, commitTime) - require.NoError(t, err) - - err = testSetupTxn.Commit() - require.NoError(t, err) - - require.Equal(t, commitTime, block.LatestCommitExecutionTimeForTestingOnly()) - - testTxn, err := block.NewSnapshotReadTableTransaction( - logical.EndOfBlockExecutionTime, - logical.EndOfBlockExecutionTime) - require.NoError(t, err) - - err = testTxn.Commit() - require.NoError(t, err) - - require.Equal(t, commitTime, block.LatestCommitExecutionTimeForTestingOnly()) -} - -func TestDerivedDataTableCommitSnapshotReadDontAdvanceTime(t *testing.T) { - block := newEmptyTestBlock() - - commitTime := logical.Time(71) - testSetupTxn, err := block.NewTableTransaction(0, commitTime) - require.NoError(t, err) - - err = testSetupTxn.Commit() - require.NoError(t, err) - - repeatedTime := commitTime + 1 - for i := 0; i < 10; i++ { - txn, err := block.NewSnapshotReadTableTransaction(0, repeatedTime) - require.NoError(t, err) - - err = txn.Commit() - require.NoError(t, err) - } - - require.Equal( - t, - commitTime, - block.LatestCommitExecutionTimeForTestingOnly()) -} - func TestDerivedDataTableCommitWriteOnlyTransactionNoInvalidation(t *testing.T) { block := newEmptyTestBlock() @@ -466,7 +397,7 @@ func TestDerivedDataTableCommitWriteOnlyTransactionNoInvalidation(t *testing.T) valueString := "stuff" expectedValue := &valueString - expectedSnapshot := &state.ExecutionSnapshot{} + expectedSnapshot := &snapshot.ExecutionSnapshot{} testTxn.SetForTestingOnly(key, expectedValue, expectedSnapshot) @@ -515,7 +446,7 @@ func TestDerivedDataTableCommitWriteOnlyTransactionWithInvalidation(t *testing.T valueString := "blah" expectedValue := &valueString - expectedSnapshot := &state.ExecutionSnapshot{} + expectedSnapshot := &snapshot.ExecutionSnapshot{} testTxn.SetForTestingOnly(key, expectedValue, expectedSnapshot) @@ -563,7 +494,7 @@ func TestDerivedDataTableCommitUseOriginalEntryOnDuplicateWriteEntries(t *testin key := "17" valueString := "foo" expectedValue := &valueString - expectedSnapshot := &state.ExecutionSnapshot{} + expectedSnapshot := &snapshot.ExecutionSnapshot{} testSetupTxn.SetForTestingOnly(key, expectedValue, expectedSnapshot) @@ -578,7 +509,7 @@ func TestDerivedDataTableCommitUseOriginalEntryOnDuplicateWriteEntries(t *testin otherString := "other" otherValue := &otherString - otherSnapshot := &state.ExecutionSnapshot{} + otherSnapshot := &snapshot.ExecutionSnapshot{} testTxn.SetForTestingOnly(key, otherValue, otherSnapshot) @@ -611,14 +542,14 @@ func TestDerivedDataTableCommitReadOnlyTransactionNoInvalidation(t *testing.T) { key1 := "key1" valStr1 := "value1" expectedValue1 := &valStr1 - expectedSnapshot1 := &state.ExecutionSnapshot{} + expectedSnapshot1 := &snapshot.ExecutionSnapshot{} testSetupTxn.SetForTestingOnly(key1, expectedValue1, expectedSnapshot1) key2 := "key2" valStr2 := "value2" expectedValue2 := &valStr2 - expectedSnapshot2 := &state.ExecutionSnapshot{} + expectedSnapshot2 := &snapshot.ExecutionSnapshot{} testSetupTxn.SetForTestingOnly(key2, expectedValue2, expectedSnapshot2) @@ -695,14 +626,14 @@ func TestDerivedDataTableCommitReadOnlyTransactionWithInvalidation(t *testing.T) key1 := "key1" valStr1 := "v1" expectedValue1 := &valStr1 - expectedSnapshot1 := &state.ExecutionSnapshot{} + expectedSnapshot1 := &snapshot.ExecutionSnapshot{} testSetupTxn2.SetForTestingOnly(key1, expectedValue1, expectedSnapshot1) key2 := "key2" valStr2 := "v2" expectedValue2 := &valStr2 - expectedSnapshot2 := &state.ExecutionSnapshot{} + expectedSnapshot2 := &snapshot.ExecutionSnapshot{} testSetupTxn2.SetForTestingOnly(key2, expectedValue2, expectedSnapshot2) @@ -768,7 +699,7 @@ func TestDerivedDataTableCommitValidateError(t *testing.T) { commitErr := testTxn.Commit() require.ErrorContains(t, commitErr, "non-increasing time") - require.False(t, commitErr.IsRetryable()) + require.False(t, errors.IsRetryableConflictError(commitErr)) } func TestDerivedDataTableCommitRejectCommitGapForNormalTxn(t *testing.T) { @@ -794,68 +725,42 @@ func TestDerivedDataTableCommitRejectCommitGapForNormalTxn(t *testing.T) { commitErr := testTxn.Commit() require.ErrorContains(t, commitErr, "missing commit range [6, 10)") - require.False(t, commitErr.IsRetryable()) + require.False(t, errors.IsRetryableConflictError(commitErr)) } -func TestDerivedDataTableCommitRejectCommitGapForSnapshotRead(t *testing.T) { +func TestDerivedDataTableCommitSnapshotReadDontAdvanceTime(t *testing.T) { block := newEmptyTestBlock() - commitTime := logical.Time(5) + commitTime := logical.Time(71) testSetupTxn, err := block.NewTableTransaction(0, commitTime) require.NoError(t, err) err = testSetupTxn.Commit() require.NoError(t, err) - require.Equal( - t, - commitTime, - block.LatestCommitExecutionTimeForTestingOnly()) - - testTxn, err := block.NewSnapshotReadTableTransaction(10, 10) - require.NoError(t, err) - - err = testTxn.Validate() - require.NoError(t, err) - - commitErr := testTxn.Commit() - require.ErrorContains(t, commitErr, "missing commit range [6, 10)") - require.False(t, commitErr.IsRetryable()) -} - -func TestDerivedDataTableCommitSnapshotReadDoesNotAdvanceCommitTime(t *testing.T) { - block := newEmptyTestBlock() - - expectedTime := logical.Time(10) - testSetupTxn, err := block.NewTableTransaction(0, expectedTime) - require.NoError(t, err) - - err = testSetupTxn.Commit() - require.NoError(t, err) - - testTxn, err := block.NewSnapshotReadTableTransaction(0, 11) - require.NoError(t, err) + for i := 0; i < 10; i++ { + txn := block.NewSnapshotReadTableTransaction() - err = testTxn.Commit() - require.NoError(t, err) + err = txn.Commit() + require.NoError(t, err) + } require.Equal( t, - expectedTime, + commitTime, block.LatestCommitExecutionTimeForTestingOnly()) } func TestDerivedDataTableCommitBadSnapshotReadInvalidator(t *testing.T) { block := newEmptyTestBlock() - testTxn, err := block.NewSnapshotReadTableTransaction(0, 42) - require.NoError(t, err) + testTxn := block.NewSnapshotReadTableTransaction() testTxn.AddInvalidator(&testInvalidator{invalidateAll: true}) commitErr := testTxn.Commit() require.ErrorContains(t, commitErr, "snapshot read can't invalidate") - require.False(t, commitErr.IsRetryable()) + require.False(t, errors.IsRetryableConflictError(commitErr)) } func TestDerivedDataTableCommitFineGrainInvalidation(t *testing.T) { @@ -869,12 +774,12 @@ func TestDerivedDataTableCommitFineGrainInvalidation(t *testing.T) { readKey1 := "read-key-1" readValStr1 := "read-value-1" readValue1 := &readValStr1 - readSnapshot1 := &state.ExecutionSnapshot{} + readSnapshot1 := &snapshot.ExecutionSnapshot{} readKey2 := "read-key-2" readValStr2 := "read-value-2" readValue2 := &readValStr2 - readSnapshot2 := &state.ExecutionSnapshot{} + readSnapshot2 := &snapshot.ExecutionSnapshot{} testSetupTxn.SetForTestingOnly(readKey1, readValue1, readSnapshot1) testSetupTxn.SetForTestingOnly(readKey2, readValue2, readSnapshot2) @@ -902,12 +807,12 @@ func TestDerivedDataTableCommitFineGrainInvalidation(t *testing.T) { writeKey1 := "write key 1" writeValStr1 := "write value 1" writeValue1 := &writeValStr1 - writeSnapshot1 := &state.ExecutionSnapshot{} + writeSnapshot1 := &snapshot.ExecutionSnapshot{} writeKey2 := "write key 2" writeValStr2 := "write value 2" writeValue2 := &writeValStr2 - writeSnapshot2 := &state.ExecutionSnapshot{} + writeSnapshot2 := &snapshot.ExecutionSnapshot{} testTxn.SetForTestingOnly(writeKey1, writeValue1, writeSnapshot1) testTxn.SetForTestingOnly(writeKey2, writeValue2, writeSnapshot2) @@ -988,7 +893,7 @@ func TestDerivedDataTableNewChildDerivedBlockData(t *testing.T) { key := "foo bar" valStr := "zzz" value := &valStr - state := &state.ExecutionSnapshot{} + state := &snapshot.ExecutionSnapshot{} txn.SetForTestingOnly(key, value, state) @@ -1042,7 +947,7 @@ type testValueComputer struct { } func (computer *testValueComputer) Compute( - txnState state.NestedTransaction, + txnState state.NestedTransactionPreparer, key flow.RegisterID, ) ( int, @@ -1058,14 +963,15 @@ func (computer *testValueComputer) Compute( } func TestDerivedDataTableGetOrCompute(t *testing.T) { - blockDerivedData := NewEmptyTable[flow.RegisterID, int]() + blockDerivedData := NewEmptyTable[flow.RegisterID, int](0) key := flow.NewRegisterID("addr", "key") value := 12345 t.Run("compute value", func(t *testing.T) { - view := delta.NewDeltaView(nil) - txnState := state.NewTransactionState(view, state.DefaultParameters()) + txnState := state.NewTransactionState( + nil, + state.DefaultParameters()) txnDerivedData, err := blockDerivedData.NewTableTransaction(0, 0) assert.NoError(t, err) @@ -1101,8 +1007,9 @@ func TestDerivedDataTableGetOrCompute(t *testing.T) { }) t.Run("get value", func(t *testing.T) { - view := delta.NewDeltaView(nil) - txnState := state.NewTransactionState(view, state.DefaultParameters()) + txnState := state.NewTransactionState( + nil, + state.DefaultParameters()) txnDerivedData, err := blockDerivedData.NewTableTransaction(1, 1) assert.NoError(t, err) diff --git a/fvm/storage/errors/errors.go b/fvm/storage/errors/errors.go new file mode 100644 index 00000000000..4f6fca25015 --- /dev/null +++ b/fvm/storage/errors/errors.go @@ -0,0 +1,58 @@ +package errors + +import ( + stdErrors "errors" + "fmt" +) + +type Unwrappable interface { + Unwrap() error +} + +type RetryableConflictError interface { + IsRetryableConflict() bool + + Unwrappable + error +} + +func IsRetryableConflictError(originalErr error) bool { + if originalErr == nil { + return false + } + + currentErr := originalErr + for { + var retryable RetryableConflictError + if !stdErrors.As(currentErr, &retryable) { + return false + } + + if retryable.IsRetryableConflict() { + return true + } + + currentErr = retryable.Unwrap() + } +} + +type retryableConflictError struct { + error +} + +func NewRetryableConflictError( + msg string, + vals ...interface{}, +) error { + return &retryableConflictError{ + error: fmt.Errorf(msg, vals...), + } +} + +func (retryableConflictError) IsRetryableConflict() bool { + return true +} + +func (err *retryableConflictError) Unwrap() error { + return err.error +} diff --git a/fvm/storage/errors/errors_test.go b/fvm/storage/errors/errors_test.go new file mode 100644 index 00000000000..6791315c4d0 --- /dev/null +++ b/fvm/storage/errors/errors_test.go @@ -0,0 +1,17 @@ +package errors + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsRetryablelConflictError(t *testing.T) { + require.False(t, IsRetryableConflictError(fmt.Errorf("generic error"))) + + err := NewRetryableConflictError("bad %s", "conflict") + require.True(t, IsRetryableConflictError(err)) + + require.True(t, IsRetryableConflictError(fmt.Errorf("wrapped: %w", err))) +} diff --git a/fvm/storage/logical/time.go b/fvm/storage/logical/time.go index ae33c5e377d..b7fe4c6dc15 100644 --- a/fvm/storage/logical/time.go +++ b/fvm/storage/logical/time.go @@ -41,10 +41,6 @@ const ( // such as during script execution. EndOfBlockExecutionTime = ChildBlockTime - 1 - // A snapshot read transaction may occur at any time within the range - // [0, EndOfBlockExecutionTime] - LargestSnapshotReadTransactionExecutionTime = EndOfBlockExecutionTime - // A normal transaction cannot commit to EndOfBlockExecutionTime. // // Note that we can assign the time to any value in the range diff --git a/fvm/storage/primary/block_data.go b/fvm/storage/primary/block_data.go new file mode 100644 index 00000000000..bf5c3d7aa58 --- /dev/null +++ b/fvm/storage/primary/block_data.go @@ -0,0 +1,232 @@ +package primary + +import ( + "fmt" + "sync" + + "github.com/onflow/flow-go/fvm/storage/errors" + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" + "github.com/onflow/flow-go/model/flow" +) + +const ( + conflictErrorTemplate = "invalid transaction: committed txn %d conflicts " + + "with executing txn %d with snapshot at %d (Conflicting register: %v)" +) + +// BlockData is a rudimentary in-memory MVCC database for storing (RegisterID, +// RegisterValue) pairs for a particular block. The database enforces +// atomicity, consistency, and isolation, but not durability (The transactions +// are made durable by the block computer using aggregated execution snapshots). +type BlockData struct { + mutex sync.RWMutex + + latestSnapshot timestampedSnapshotTree // Guarded by mutex +} + +type TransactionData struct { + block *BlockData + + executionTime logical.Time + isSnapshotReadTransaction bool + + snapshot *rebaseableTimestampedSnapshotTree + + state.NestedTransactionPreparer + + finalizedExecutionSnapshot *snapshot.ExecutionSnapshot +} + +// Note: storageSnapshot must be thread safe. +func NewBlockData( + storageSnapshot snapshot.StorageSnapshot, + snapshotTime logical.Time, +) *BlockData { + return &BlockData{ + latestSnapshot: newTimestampedSnapshotTree( + storageSnapshot, + logical.Time(snapshotTime)), + } +} + +func (block *BlockData) LatestSnapshot() timestampedSnapshotTree { + block.mutex.RLock() + defer block.mutex.RUnlock() + + return block.latestSnapshot +} + +func (block *BlockData) newTransactionData( + isSnapshotReadTransaction bool, + executionTime logical.Time, + parameters state.StateParameters, +) *TransactionData { + snapshot := newRebaseableTimestampedSnapshotTree(block.LatestSnapshot()) + return &TransactionData{ + block: block, + executionTime: executionTime, + snapshot: snapshot, + isSnapshotReadTransaction: isSnapshotReadTransaction, + NestedTransactionPreparer: state.NewTransactionState( + snapshot, + parameters), + } +} + +func (block *BlockData) NewTransactionData( + executionTime logical.Time, + parameters state.StateParameters, +) ( + *TransactionData, + error, +) { + if executionTime < 0 || + executionTime > logical.LargestNormalTransactionExecutionTime { + + return nil, fmt.Errorf( + "invalid tranaction: execution time out of bound") + } + + txn := block.newTransactionData( + false, + executionTime, + parameters) + + if txn.SnapshotTime() > executionTime { + return nil, fmt.Errorf( + "invalid transaction: snapshot > execution: %v > %v", + txn.SnapshotTime(), + executionTime) + } + + return txn, nil +} + +func (block *BlockData) NewSnapshotReadTransactionData( + parameters state.StateParameters, +) *TransactionData { + return block.newTransactionData( + true, + logical.EndOfBlockExecutionTime, + parameters) +} + +func (txn *TransactionData) SnapshotTime() logical.Time { + return txn.snapshot.SnapshotTime() +} + +func (txn *TransactionData) validate( + latestSnapshot timestampedSnapshotTree, +) error { + validatedSnapshotTime := txn.SnapshotTime() + + if latestSnapshot.SnapshotTime() <= validatedSnapshotTime { + // transaction's snapshot is up-to-date. + return nil + } + + var readSet map[flow.RegisterID]struct{} + if txn.finalizedExecutionSnapshot != nil { + readSet = txn.finalizedExecutionSnapshot.ReadSet + } else { + readSet = txn.InterimReadSet() + } + + updates, err := latestSnapshot.UpdatesSince(validatedSnapshotTime) + if err != nil { + return fmt.Errorf("invalid transaction: %w", err) + } + + for i, writeSet := range updates { + hasConflict, registerId := intersect(writeSet, readSet) + if hasConflict { + return errors.NewRetryableConflictError( + conflictErrorTemplate, + validatedSnapshotTime+logical.Time(i), + txn.executionTime, + validatedSnapshotTime, + registerId) + } + } + + txn.snapshot.Rebase(latestSnapshot) + return nil +} + +func (txn *TransactionData) Validate() error { + return txn.validate(txn.block.LatestSnapshot()) +} + +func (txn *TransactionData) Finalize() error { + executionSnapshot, err := txn.FinalizeMainTransaction() + if err != nil { + return err + } + + // NOTE: Since cadence does not support the notion of read only execution, + // snapshot read transaction execution can inadvertently produce a non-empty + // write set. We'll just drop these updates. + if txn.isSnapshotReadTransaction { + executionSnapshot.WriteSet = nil + } + + txn.finalizedExecutionSnapshot = executionSnapshot + return nil +} + +func (block *BlockData) commit(txn *TransactionData) error { + if txn.finalizedExecutionSnapshot == nil { + return fmt.Errorf("invalid transaction: transaction not finalized.") + } + + block.mutex.Lock() + defer block.mutex.Unlock() + + err := txn.validate(block.latestSnapshot) + if err != nil { + return err + } + + // Don't perform actual commit for snapshot read transaction since they + // do not advance logical time. + if txn.isSnapshotReadTransaction { + return nil + } + + latestSnapshotTime := block.latestSnapshot.SnapshotTime() + + if latestSnapshotTime < txn.executionTime { + // i.e., transactions are committed out-of-order. + return fmt.Errorf( + "invalid transaction: missing commit range [%v, %v)", + latestSnapshotTime, + txn.executionTime) + } + + if block.latestSnapshot.SnapshotTime() > txn.executionTime { + // i.e., re-commiting an already committed transaction. + return fmt.Errorf( + "invalid transaction: non-increasing time (%v >= %v)", + latestSnapshotTime-1, + txn.executionTime) + } + + block.latestSnapshot = block.latestSnapshot.Append( + txn.finalizedExecutionSnapshot) + + return nil +} + +func (txn *TransactionData) Commit() ( + *snapshot.ExecutionSnapshot, + error, +) { + err := txn.block.commit(txn) + if err != nil { + return nil, err + } + + return txn.finalizedExecutionSnapshot, nil +} diff --git a/fvm/storage/primary/block_data_test.go b/fvm/storage/primary/block_data_test.go new file mode 100644 index 00000000000..8c20e301b0b --- /dev/null +++ b/fvm/storage/primary/block_data_test.go @@ -0,0 +1,661 @@ +package primary + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/fvm/storage/errors" + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" + "github.com/onflow/flow-go/model/flow" +) + +func TestBlockDataWithTransactionOffset(t *testing.T) { + key := flow.RegisterID{ + Owner: "", + Key: "key", + } + expectedValue := flow.RegisterValue([]byte("value")) + + snapshotTime := logical.Time(18) + + block := NewBlockData( + snapshot.MapStorageSnapshot{ + key: expectedValue, + }, + snapshotTime) + + snapshot := block.LatestSnapshot() + require.Equal(t, snapshotTime, snapshot.SnapshotTime()) + + value, err := snapshot.Get(key) + require.NoError(t, err) + require.Equal(t, expectedValue, value) +} + +func TestBlockDataNormalTransactionInvalidExecutionTime(t *testing.T) { + snapshotTime := logical.Time(5) + block := NewBlockData(nil, snapshotTime) + + txn, err := block.NewTransactionData(-1, state.DefaultParameters()) + require.ErrorContains(t, err, "execution time out of bound") + require.Nil(t, txn) + + txn, err = block.NewTransactionData( + logical.EndOfBlockExecutionTime, + state.DefaultParameters()) + require.ErrorContains(t, err, "execution time out of bound") + require.Nil(t, txn) + + txn, err = block.NewTransactionData( + snapshotTime-1, + state.DefaultParameters()) + require.ErrorContains(t, err, "snapshot > execution: 5 > 4") + require.Nil(t, txn) +} + +func testBlockDataValidate( + t *testing.T, + shouldFinalize bool, +) { + baseSnapshotTime := logical.Time(11) + block := NewBlockData(nil, baseSnapshotTime) + + // Commit a key before the actual test txn (which read the same key). + + testSetupTxn, err := block.NewTransactionData( + baseSnapshotTime, + state.DefaultParameters()) + require.NoError(t, err) + + registerId1 := flow.RegisterID{ + Owner: "", + Key: "key1", + } + expectedValue1 := flow.RegisterValue([]byte("value1")) + + err = testSetupTxn.Set(registerId1, expectedValue1) + require.NoError(t, err) + + err = testSetupTxn.Finalize() + require.NoError(t, err) + + _, err = testSetupTxn.Commit() + require.NoError(t, err) + + require.Equal( + t, + baseSnapshotTime+1, + block.LatestSnapshot().SnapshotTime()) + + value, err := block.LatestSnapshot().Get(registerId1) + require.NoError(t, err) + require.Equal(t, expectedValue1, value) + + // Start the test transaction at an "older" snapshot to ensure valdiate + // works as expected. + + testTxn, err := block.NewTransactionData( + baseSnapshotTime+3, + state.DefaultParameters()) + require.NoError(t, err) + + // Commit a bunch of unrelated transactions. + + testSetupTxn, err = block.NewTransactionData( + baseSnapshotTime+1, + state.DefaultParameters()) + require.NoError(t, err) + + registerId2 := flow.RegisterID{ + Owner: "", + Key: "key2", + } + expectedValue2 := flow.RegisterValue([]byte("value2")) + + err = testSetupTxn.Set(registerId2, expectedValue2) + require.NoError(t, err) + + err = testSetupTxn.Finalize() + require.NoError(t, err) + + _, err = testSetupTxn.Commit() + require.NoError(t, err) + + testSetupTxn, err = block.NewTransactionData( + baseSnapshotTime+2, + state.DefaultParameters()) + require.NoError(t, err) + + registerId3 := flow.RegisterID{ + Owner: "", + Key: "key3", + } + expectedValue3 := flow.RegisterValue([]byte("value3")) + + err = testSetupTxn.Set(registerId3, expectedValue3) + require.NoError(t, err) + + err = testSetupTxn.Finalize() + require.NoError(t, err) + + _, err = testSetupTxn.Commit() + require.NoError(t, err) + + // Actual test + + _, err = testTxn.Get(registerId1) + require.NoError(t, err) + + if shouldFinalize { + err = testTxn.Finalize() + require.NoError(t, err) + + require.NotNil(t, testTxn.finalizedExecutionSnapshot) + } else { + require.Nil(t, testTxn.finalizedExecutionSnapshot) + } + + // Check the original snapshot tree before calling validate. + require.Equal(t, baseSnapshotTime+1, testTxn.SnapshotTime()) + + value, err = testTxn.snapshot.Get(registerId1) + require.NoError(t, err) + require.Equal(t, expectedValue1, value) + + value, err = testTxn.snapshot.Get(registerId2) + require.NoError(t, err) + require.Nil(t, value) + + value, err = testTxn.snapshot.Get(registerId3) + require.NoError(t, err) + require.Nil(t, value) + + // Validate should not detect any conflict and should rebase the snapshot. + err = testTxn.Validate() + require.NoError(t, err) + + // Ensure validate rebase to a new snapshot tree. + require.Equal(t, baseSnapshotTime+3, testTxn.SnapshotTime()) + + value, err = testTxn.snapshot.Get(registerId1) + require.NoError(t, err) + require.Equal(t, expectedValue1, value) + + value, err = testTxn.snapshot.Get(registerId2) + require.NoError(t, err) + require.Equal(t, expectedValue2, value) + + value, err = testTxn.snapshot.Get(registerId3) + require.NoError(t, err) + require.Equal(t, expectedValue3, value) + + // Note: we can't make additional Get calls on a finalized transaction. + if shouldFinalize { + _, err = testTxn.Get(registerId1) + require.ErrorContains(t, err, "cannot Get on a finalized state") + + _, err = testTxn.Get(registerId2) + require.ErrorContains(t, err, "cannot Get on a finalized state") + + _, err = testTxn.Get(registerId3) + require.ErrorContains(t, err, "cannot Get on a finalized state") + } else { + value, err = testTxn.Get(registerId1) + require.NoError(t, err) + require.Equal(t, expectedValue1, value) + + value, err = testTxn.Get(registerId2) + require.NoError(t, err) + require.Equal(t, expectedValue2, value) + + value, err = testTxn.Get(registerId3) + require.NoError(t, err) + require.Equal(t, expectedValue3, value) + } +} + +func TestBlockDataValidateInterim(t *testing.T) { + testBlockDataValidate(t, false) +} + +func TestBlockDataValidateFinalized(t *testing.T) { + testBlockDataValidate(t, true) +} + +func testBlockDataValidateRejectConflict( + t *testing.T, + shouldFinalize bool, + conflictTxn int, // [1, 2, 3] +) { + baseSnapshotTime := logical.Time(32) + block := NewBlockData(nil, baseSnapshotTime) + + // Commit a bunch of unrelated updates + + for ; baseSnapshotTime < 42; baseSnapshotTime++ { + testSetupTxn, err := block.NewTransactionData( + baseSnapshotTime, + state.DefaultParameters()) + require.NoError(t, err) + + err = testSetupTxn.Set( + flow.RegisterID{ + Owner: "", + Key: fmt.Sprintf("other key - %d", baseSnapshotTime), + }, + []byte("blah")) + require.NoError(t, err) + + err = testSetupTxn.Finalize() + require.NoError(t, err) + + _, err = testSetupTxn.Commit() + require.NoError(t, err) + } + + // Start the test transaction at an "older" snapshot to ensure valdiate + // works as expected. + + testTxnTime := baseSnapshotTime + 3 + testTxn, err := block.NewTransactionData( + testTxnTime, + state.DefaultParameters()) + require.NoError(t, err) + + // Commit one key per test setup transaction. One of these keys will + // conflicts with the test txn. + + txn1Time := baseSnapshotTime + testSetupTxn, err := block.NewTransactionData( + txn1Time, + state.DefaultParameters()) + require.NoError(t, err) + + registerId1 := flow.RegisterID{ + Owner: "", + Key: "key1", + } + + err = testSetupTxn.Set(registerId1, []byte("value1")) + require.NoError(t, err) + + err = testSetupTxn.Finalize() + require.NoError(t, err) + + _, err = testSetupTxn.Commit() + require.NoError(t, err) + + txn2Time := baseSnapshotTime + 1 + testSetupTxn, err = block.NewTransactionData( + txn2Time, + state.DefaultParameters()) + require.NoError(t, err) + + registerId2 := flow.RegisterID{ + Owner: "", + Key: "key2", + } + + err = testSetupTxn.Set(registerId2, []byte("value2")) + require.NoError(t, err) + + err = testSetupTxn.Finalize() + require.NoError(t, err) + + _, err = testSetupTxn.Commit() + require.NoError(t, err) + + txn3Time := baseSnapshotTime + 2 + testSetupTxn, err = block.NewTransactionData( + txn3Time, + state.DefaultParameters()) + require.NoError(t, err) + + registerId3 := flow.RegisterID{ + Owner: "", + Key: "key3", + } + + err = testSetupTxn.Set(registerId3, []byte("value3")) + require.NoError(t, err) + + err = testSetupTxn.Finalize() + require.NoError(t, err) + + _, err = testSetupTxn.Commit() + require.NoError(t, err) + + // Actual test + + var conflictTxnTime logical.Time + var conflictRegisterId flow.RegisterID + switch conflictTxn { + case 1: + conflictTxnTime = txn1Time + conflictRegisterId = registerId1 + case 2: + conflictTxnTime = txn2Time + conflictRegisterId = registerId2 + case 3: + conflictTxnTime = txn3Time + conflictRegisterId = registerId3 + } + + value, err := testTxn.Get(conflictRegisterId) + require.NoError(t, err) + require.Nil(t, value) + + if shouldFinalize { + err = testTxn.Finalize() + require.NoError(t, err) + + require.NotNil(t, testTxn.finalizedExecutionSnapshot) + } else { + require.Nil(t, testTxn.finalizedExecutionSnapshot) + } + + // Check the original snapshot tree before calling validate. + require.Equal(t, baseSnapshotTime, testTxn.SnapshotTime()) + + err = testTxn.Validate() + require.ErrorContains( + t, + err, + fmt.Sprintf( + conflictErrorTemplate, + conflictTxnTime, + testTxnTime, + baseSnapshotTime, + conflictRegisterId)) + require.True(t, errors.IsRetryableConflictError(err)) + + // Validate should not rebase the snapshot tree on error + require.Equal(t, baseSnapshotTime, testTxn.SnapshotTime()) +} + +func TestBlockDataValidateInterimRejectConflict(t *testing.T) { + testBlockDataValidateRejectConflict(t, false, 1) + testBlockDataValidateRejectConflict(t, false, 2) + testBlockDataValidateRejectConflict(t, false, 3) +} + +func TestBlockDataValidateFinalizedRejectConflict(t *testing.T) { + testBlockDataValidateRejectConflict(t, true, 1) + testBlockDataValidateRejectConflict(t, true, 2) + testBlockDataValidateRejectConflict(t, true, 3) +} + +func TestBlockDataCommit(t *testing.T) { + block := NewBlockData(nil, 0) + + // Start test txn at an "older" snapshot. + txn, err := block.NewTransactionData(3, state.DefaultParameters()) + require.NoError(t, err) + + // Commit a bunch of unrelated updates + + for i := logical.Time(0); i < 3; i++ { + testSetupTxn, err := block.NewTransactionData( + i, + state.DefaultParameters()) + require.NoError(t, err) + + err = testSetupTxn.Set( + flow.RegisterID{ + Owner: "", + Key: fmt.Sprintf("other key - %d", i), + }, + []byte("blah")) + require.NoError(t, err) + + err = testSetupTxn.Finalize() + require.NoError(t, err) + + _, err = testSetupTxn.Commit() + require.NoError(t, err) + } + + // "resume" test txn + + writeRegisterId := flow.RegisterID{ + Owner: "", + Key: "write", + } + expectedValue := flow.RegisterValue([]byte("value")) + + err = txn.Set(writeRegisterId, expectedValue) + require.NoError(t, err) + + readRegisterId := flow.RegisterID{ + Owner: "", + Key: "read", + } + value, err := txn.Get(readRegisterId) + require.NoError(t, err) + require.Nil(t, value) + + err = txn.Finalize() + require.NoError(t, err) + + // Actual test. Ensure the transaction is committed. + + require.Equal(t, logical.Time(0), txn.SnapshotTime()) + require.Equal(t, logical.Time(3), block.LatestSnapshot().SnapshotTime()) + + executionSnapshot, err := txn.Commit() + require.NoError(t, err) + require.NotNil(t, executionSnapshot) + require.Equal( + t, + map[flow.RegisterID]struct{}{ + readRegisterId: struct{}{}, + }, + executionSnapshot.ReadSet) + require.Equal( + t, + map[flow.RegisterID]flow.RegisterValue{ + writeRegisterId: expectedValue, + }, + executionSnapshot.WriteSet) + + require.Equal(t, logical.Time(4), block.LatestSnapshot().SnapshotTime()) + + value, err = block.LatestSnapshot().Get(writeRegisterId) + require.NoError(t, err) + require.Equal(t, expectedValue, value) +} + +func TestBlockDataCommitSnapshotReadDontAdvanceTime(t *testing.T) { + baseRegisterId := flow.RegisterID{ + Owner: "", + Key: "base", + } + baseValue := flow.RegisterValue([]byte("original")) + + baseSnapshotTime := logical.Time(16) + + block := NewBlockData( + snapshot.MapStorageSnapshot{ + baseRegisterId: baseValue, + }, + baseSnapshotTime) + + txn := block.NewSnapshotReadTransactionData(state.DefaultParameters()) + + readRegisterId := flow.RegisterID{ + Owner: "", + Key: "read", + } + value, err := txn.Get(readRegisterId) + require.NoError(t, err) + require.Nil(t, value) + + err = txn.Set(baseRegisterId, []byte("bad")) + require.NoError(t, err) + + err = txn.Finalize() + require.NoError(t, err) + + require.Equal(t, baseSnapshotTime, block.LatestSnapshot().SnapshotTime()) + + executionSnapshot, err := txn.Commit() + require.NoError(t, err) + + require.NotNil(t, executionSnapshot) + + require.Equal( + t, + map[flow.RegisterID]struct{}{ + readRegisterId: struct{}{}, + }, + executionSnapshot.ReadSet) + + // Ensure we have dropped the write set internally. + require.Nil(t, executionSnapshot.WriteSet) + + // Ensure block snapshot is not updated. + require.Equal(t, baseSnapshotTime, block.LatestSnapshot().SnapshotTime()) + + value, err = block.LatestSnapshot().Get(baseRegisterId) + require.NoError(t, err) + require.Equal(t, baseValue, value) +} + +func TestBlockDataCommitRejectNotFinalized(t *testing.T) { + block := NewBlockData(nil, 0) + + txn, err := block.NewTransactionData(0, state.DefaultParameters()) + require.NoError(t, err) + + executionSnapshot, err := txn.Commit() + require.ErrorContains(t, err, "transaction not finalized") + require.False(t, errors.IsRetryableConflictError(err)) + require.Nil(t, executionSnapshot) +} + +func TestBlockDataCommitRejectConflict(t *testing.T) { + block := NewBlockData(nil, 0) + + registerId := flow.RegisterID{ + Owner: "", + Key: "key1", + } + + // Start test txn at an "older" snapshot. + testTxn, err := block.NewTransactionData(1, state.DefaultParameters()) + require.NoError(t, err) + + // Commit a conflicting key + testSetupTxn, err := block.NewTransactionData(0, state.DefaultParameters()) + require.NoError(t, err) + + err = testSetupTxn.Set(registerId, []byte("value")) + require.NoError(t, err) + + err = testSetupTxn.Finalize() + require.NoError(t, err) + + executionSnapshot, err := testSetupTxn.Commit() + require.NoError(t, err) + require.NotNil(t, executionSnapshot) + + // Actual test + + require.Equal(t, logical.Time(1), block.LatestSnapshot().SnapshotTime()) + + value, err := testTxn.Get(registerId) + require.NoError(t, err) + require.Nil(t, value) + + err = testTxn.Finalize() + require.NoError(t, err) + + executionSnapshot, err = testTxn.Commit() + require.Error(t, err) + require.True(t, errors.IsRetryableConflictError(err)) + require.Nil(t, executionSnapshot) + + // testTxn is not committed to block. + require.Equal(t, logical.Time(1), block.LatestSnapshot().SnapshotTime()) +} + +func TestBlockDataCommitRejectCommitGap(t *testing.T) { + block := NewBlockData(nil, 1) + + for i := logical.Time(2); i < 5; i++ { + txn, err := block.NewTransactionData(i, state.DefaultParameters()) + require.NoError(t, err) + + err = txn.Finalize() + require.NoError(t, err) + + executionSnapshot, err := txn.Commit() + require.ErrorContains( + t, + err, + fmt.Sprintf("missing commit range [1, %d)", i)) + require.False(t, errors.IsRetryableConflictError(err)) + require.Nil(t, executionSnapshot) + + // testTxn is not committed to block. + require.Equal(t, logical.Time(1), block.LatestSnapshot().SnapshotTime()) + } +} + +func TestBlockDataCommitRejectNonIncreasingExecutionTime1(t *testing.T) { + block := NewBlockData(nil, 0) + + testTxn, err := block.NewTransactionData(5, state.DefaultParameters()) + require.NoError(t, err) + + err = testTxn.Finalize() + require.NoError(t, err) + + // Commit a bunch of unrelated transactions. + for i := logical.Time(0); i < 10; i++ { + txn, err := block.NewTransactionData(i, state.DefaultParameters()) + require.NoError(t, err) + + err = txn.Finalize() + require.NoError(t, err) + + _, err = txn.Commit() + require.NoError(t, err) + } + + // sanity check before testing commit. + require.Equal(t, logical.Time(10), block.LatestSnapshot().SnapshotTime()) + + // "re-commit" an already committed transaction + executionSnapshot, err := testTxn.Commit() + require.ErrorContains(t, err, "non-increasing time (9 >= 5)") + require.False(t, errors.IsRetryableConflictError(err)) + require.Nil(t, executionSnapshot) + + // testTxn is not committed to block. + require.Equal(t, logical.Time(10), block.LatestSnapshot().SnapshotTime()) +} + +func TestBlockDataCommitRejectNonIncreasingExecutionTime2(t *testing.T) { + block := NewBlockData(nil, 13) + + testTxn, err := block.NewTransactionData(13, state.DefaultParameters()) + require.NoError(t, err) + + err = testTxn.Finalize() + require.NoError(t, err) + + executionSnapshot, err := testTxn.Commit() + require.NoError(t, err) + require.NotNil(t, executionSnapshot) + + // "re-commit" an already committed transaction + executionSnapshot, err = testTxn.Commit() + require.ErrorContains(t, err, "non-increasing time (13 >= 13)") + require.False(t, errors.IsRetryableConflictError(err)) + require.Nil(t, executionSnapshot) +} diff --git a/fvm/storage/primary/intersect.go b/fvm/storage/primary/intersect.go new file mode 100644 index 00000000000..352ae6ac9cb --- /dev/null +++ b/fvm/storage/primary/intersect.go @@ -0,0 +1,42 @@ +package primary + +import ( + "github.com/onflow/flow-go/model/flow" +) + +func intersectHelper[ + T1 any, + T2 any, +]( + smallSet map[flow.RegisterID]T1, + largeSet map[flow.RegisterID]T2, +) ( + bool, + flow.RegisterID, +) { + for id := range smallSet { + _, ok := largeSet[id] + if ok { + return true, id + } + } + + return false, flow.RegisterID{} +} + +func intersect[ + T1 any, + T2 any, +]( + set1 map[flow.RegisterID]T1, + set2 map[flow.RegisterID]T2, +) ( + bool, + flow.RegisterID, +) { + if len(set1) > len(set2) { + return intersectHelper(set2, set1) + } + + return intersectHelper(set1, set2) +} diff --git a/fvm/storage/primary/intersect_test.go b/fvm/storage/primary/intersect_test.go new file mode 100644 index 00000000000..babf1423b47 --- /dev/null +++ b/fvm/storage/primary/intersect_test.go @@ -0,0 +1,110 @@ +package primary + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" +) + +func TestIntersect(t *testing.T) { + check := func( + writeSet map[flow.RegisterID]flow.RegisterValue, + readSet map[flow.RegisterID]struct{}, + expectedMatch bool, + expectedRegisterId flow.RegisterID) { + + match, registerId := intersectHelper(writeSet, readSet) + require.Equal(t, match, expectedMatch) + if match { + require.Equal(t, expectedRegisterId, registerId) + } + + match, registerId = intersectHelper(readSet, writeSet) + require.Equal(t, match, expectedMatch) + if match { + require.Equal(t, expectedRegisterId, registerId) + } + + match, registerId = intersect(writeSet, readSet) + require.Equal(t, match, expectedMatch) + if match { + require.Equal(t, expectedRegisterId, registerId) + } + + match, registerId = intersect(readSet, writeSet) + require.Equal(t, match, expectedMatch) + if match { + require.Equal(t, expectedRegisterId, registerId) + } + } + + owner := "owner" + key1 := "key1" + key2 := "key2" + + // set up readSet1 and writeSet1 such that len(readSet1) > len(writeSet1), + // and shares key1 + + readSet1 := map[flow.RegisterID]struct{}{ + flow.RegisterID{ + Owner: owner, + Key: key1, + }: struct{}{}, + flow.RegisterID{ + Owner: "1", + Key: "read 1", + }: struct{}{}, + flow.RegisterID{ + Owner: "1", + Key: "read 2", + }: struct{}{}, + } + + writeSet1 := map[flow.RegisterID]flow.RegisterValue{ + flow.RegisterID{ + Owner: owner, + Key: key1, + }: []byte("blah"), + flow.RegisterID{ + Owner: "1", + Key: "write", + }: []byte("blah"), + } + + // set up readSet2 and writeSet2 such that len(readSet2) < len(writeSet2), + // shares key2, and not share keys with readSet1 / writeSet1 + + readSet2 := map[flow.RegisterID]struct{}{ + flow.RegisterID{ + Owner: owner, + Key: key2, + }: struct{}{}, + } + + writeSet2 := map[flow.RegisterID]flow.RegisterValue{ + flow.RegisterID{ + Owner: owner, + Key: key2, + }: []byte("blah"), + flow.RegisterID{ + Owner: "2", + Key: "write 1", + }: []byte("blah"), + flow.RegisterID{ + Owner: "2", + Key: "write 2", + }: []byte("blah"), + flow.RegisterID{ + Owner: "2", + Key: "write 3", + }: []byte("blah"), + } + + check(writeSet1, readSet1, true, flow.RegisterID{Owner: owner, Key: key1}) + check(writeSet2, readSet2, true, flow.RegisterID{Owner: owner, Key: key2}) + + check(writeSet1, readSet2, false, flow.RegisterID{}) + check(writeSet2, readSet1, false, flow.RegisterID{}) +} diff --git a/fvm/storage/primary/snapshot_tree.go b/fvm/storage/primary/snapshot_tree.go new file mode 100644 index 00000000000..2d7ef325388 --- /dev/null +++ b/fvm/storage/primary/snapshot_tree.go @@ -0,0 +1,87 @@ +package primary + +import ( + "fmt" + + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" +) + +type timestampedSnapshotTree struct { + currentSnapshotTime logical.Time + baseSnapshotTime logical.Time + + snapshot.SnapshotTree + + fullLog snapshot.UpdateLog +} + +func newTimestampedSnapshotTree( + storageSnapshot snapshot.StorageSnapshot, + snapshotTime logical.Time, +) timestampedSnapshotTree { + return timestampedSnapshotTree{ + currentSnapshotTime: snapshotTime, + baseSnapshotTime: snapshotTime, + SnapshotTree: snapshot.NewSnapshotTree(storageSnapshot), + fullLog: nil, + } +} + +func (tree timestampedSnapshotTree) Append( + executionSnapshot *snapshot.ExecutionSnapshot, +) timestampedSnapshotTree { + return timestampedSnapshotTree{ + currentSnapshotTime: tree.currentSnapshotTime + 1, + baseSnapshotTime: tree.baseSnapshotTime, + SnapshotTree: tree.SnapshotTree.Append(executionSnapshot), + fullLog: append(tree.fullLog, executionSnapshot.WriteSet), + } +} + +func (tree timestampedSnapshotTree) SnapshotTime() logical.Time { + return tree.currentSnapshotTime +} + +func (tree timestampedSnapshotTree) UpdatesSince( + snapshotTime logical.Time, +) ( + snapshot.UpdateLog, + error, +) { + if snapshotTime < tree.baseSnapshotTime { + // This should never happen. + return nil, fmt.Errorf( + "missing update log range [%v, %v)", + snapshotTime, + tree.baseSnapshotTime) + } + + if snapshotTime > tree.currentSnapshotTime { + // This should never happen. + return nil, fmt.Errorf( + "missing update log range (%v, %v]", + tree.currentSnapshotTime, + snapshotTime) + } + + return tree.fullLog[int(snapshotTime-tree.baseSnapshotTime):], nil +} + +type rebaseableTimestampedSnapshotTree struct { + timestampedSnapshotTree +} + +func newRebaseableTimestampedSnapshotTree( + snapshotTree timestampedSnapshotTree, +) *rebaseableTimestampedSnapshotTree { + return &rebaseableTimestampedSnapshotTree{ + timestampedSnapshotTree: snapshotTree, + } +} + +func (tree *rebaseableTimestampedSnapshotTree) Rebase( + base timestampedSnapshotTree, +) { + tree.timestampedSnapshotTree = base +} diff --git a/fvm/storage/primary/snapshot_tree_test.go b/fvm/storage/primary/snapshot_tree_test.go new file mode 100644 index 00000000000..1c8db612632 --- /dev/null +++ b/fvm/storage/primary/snapshot_tree_test.go @@ -0,0 +1,195 @@ +package primary + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/model/flow" +) + +func TestTimestampedSnapshotTree(t *testing.T) { + // Test setup ("commit" 4 execution snapshots to the base tree) + + baseSnapshotTime := logical.Time(5) + + registerId0 := flow.RegisterID{ + Owner: "", + Key: "key0", + } + value0 := flow.RegisterValue([]byte("value0")) + + tree0 := newTimestampedSnapshotTree( + snapshot.MapStorageSnapshot{ + registerId0: value0, + }, + baseSnapshotTime) + + registerId1 := flow.RegisterID{ + Owner: "", + Key: "key1", + } + value1 := flow.RegisterValue([]byte("value1")) + writeSet1 := map[flow.RegisterID]flow.RegisterValue{ + registerId1: value1, + } + + tree1 := tree0.Append( + &snapshot.ExecutionSnapshot{ + WriteSet: writeSet1, + }) + + registerId2 := flow.RegisterID{ + Owner: "", + Key: "key2", + } + value2 := flow.RegisterValue([]byte("value2")) + writeSet2 := map[flow.RegisterID]flow.RegisterValue{ + registerId2: value2, + } + + tree2 := tree1.Append( + &snapshot.ExecutionSnapshot{ + WriteSet: writeSet2, + }) + + registerId3 := flow.RegisterID{ + Owner: "", + Key: "key3", + } + value3 := flow.RegisterValue([]byte("value3")) + writeSet3 := map[flow.RegisterID]flow.RegisterValue{ + registerId3: value3, + } + + tree3 := tree2.Append( + &snapshot.ExecutionSnapshot{ + WriteSet: writeSet3, + }) + + registerId4 := flow.RegisterID{ + Owner: "", + Key: "key4", + } + value4 := flow.RegisterValue([]byte("value4")) + writeSet4 := map[flow.RegisterID]flow.RegisterValue{ + registerId4: value4, + } + + tree4 := tree3.Append( + &snapshot.ExecutionSnapshot{ + WriteSet: writeSet4, + }) + + // Verify the trees internal values + + trees := []timestampedSnapshotTree{tree0, tree1, tree2, tree3, tree4} + logs := snapshot.UpdateLog{writeSet1, writeSet2, writeSet3, writeSet4} + + for i, tree := range trees { + require.Equal(t, baseSnapshotTime, tree.baseSnapshotTime) + require.Equal( + t, + baseSnapshotTime+logical.Time(i), + tree.SnapshotTime()) + if i == 0 { + require.Nil(t, tree.fullLog) + } else { + require.Equal(t, logs[:i], tree.fullLog) + } + + value, err := tree.Get(registerId0) + require.NoError(t, err) + require.Equal(t, value0, value) + + value, err = tree.Get(registerId1) + require.NoError(t, err) + if i >= 1 { + require.Equal(t, value1, value) + } else { + require.Nil(t, value) + } + + value, err = tree.Get(registerId2) + require.NoError(t, err) + if i >= 2 { + require.Equal(t, value2, value) + } else { + require.Nil(t, value) + } + + value, err = tree.Get(registerId3) + require.NoError(t, err) + if i >= 3 { + require.Equal(t, value3, value) + } else { + require.Nil(t, value) + } + + value, err = tree.Get(registerId4) + require.NoError(t, err) + if i == 4 { + require.Equal(t, value4, value) + } else { + require.Nil(t, value) + } + } + + // Verify UpdatesSince returns + + updates, err := tree0.UpdatesSince(baseSnapshotTime) + require.NoError(t, err) + require.Nil(t, updates) + + _, err = tree4.UpdatesSince(baseSnapshotTime - 1) + require.ErrorContains(t, err, "missing update log range [4, 5)") + + for i := 0; i < 5; i++ { + updates, err = tree4.UpdatesSince(baseSnapshotTime + logical.Time(i)) + require.NoError(t, err) + require.Equal(t, logs[i:], updates) + } + + snapshotTime := baseSnapshotTime + logical.Time(5) + require.Equal(t, tree4.SnapshotTime()+1, snapshotTime) + + _, err = tree4.UpdatesSince(snapshotTime) + require.ErrorContains(t, err, "missing update log range (9, 10]") +} + +func TestRebaseableTimestampedSnapshotTree(t *testing.T) { + registerId := flow.RegisterID{ + Owner: "owner", + Key: "key", + } + + value1 := flow.RegisterValue([]byte("value1")) + value2 := flow.RegisterValue([]byte("value2")) + + tree1 := newTimestampedSnapshotTree( + snapshot.MapStorageSnapshot{ + registerId: value1, + }, + 0) + + tree2 := newTimestampedSnapshotTree( + snapshot.MapStorageSnapshot{ + registerId: value2, + }, + 0) + + rebaseableTree := newRebaseableTimestampedSnapshotTree(tree1) + treeReference := rebaseableTree + + value, err := treeReference.Get(registerId) + require.NoError(t, err) + require.Equal(t, value, value1) + + rebaseableTree.Rebase(tree2) + + value, err = treeReference.Get(registerId) + require.NoError(t, err) + require.Equal(t, value, value2) +} diff --git a/fvm/state/execution_snapshot.go b/fvm/storage/snapshot/execution_snapshot.go similarity index 77% rename from fvm/state/execution_snapshot.go rename to fvm/storage/snapshot/execution_snapshot.go index 0ad2be63506..89cabec443a 100644 --- a/fvm/state/execution_snapshot.go +++ b/fvm/storage/snapshot/execution_snapshot.go @@ -1,4 +1,4 @@ -package state +package snapshot import ( "golang.org/x/exp/slices" @@ -7,29 +7,6 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// TOOD(patrick): rm View interface after delta view is deleted. -type View interface { - NewChild() View - - Finalize() *ExecutionSnapshot - Merge(child *ExecutionSnapshot) error - - Storage -} - -// TOOD(patrick): rm Storage interface after delta view is deleted. -// Storage is the storage interface used by the virtual machine to read and -// write register values. -type Storage interface { - // TODO(patrick): remove once fvm.VM.Run() is deprecated - Peek(id flow.RegisterID) (flow.RegisterValue, error) - - Set(id flow.RegisterID, value flow.RegisterValue) error - Get(id flow.RegisterID) (flow.RegisterValue, error) - - DropChanges() error -} - type ExecutionSnapshot struct { // Note that the ReadSet only include reads from the storage snapshot. // Reads from the WriteSet are excluded from the ReadSet. diff --git a/fvm/storage/snapshot_tree.go b/fvm/storage/snapshot/snapshot_tree.go similarity index 77% rename from fvm/storage/snapshot_tree.go rename to fvm/storage/snapshot/snapshot_tree.go index 2dd3f1b97e9..7c91b9a5c1a 100644 --- a/fvm/storage/snapshot_tree.go +++ b/fvm/storage/snapshot/snapshot_tree.go @@ -1,7 +1,6 @@ -package storage +package snapshot import ( - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/model/flow" ) @@ -9,23 +8,21 @@ const ( compactThreshold = 10 ) -type updateLog []map[flow.RegisterID]flow.RegisterValue +type UpdateLog []map[flow.RegisterID]flow.RegisterValue // SnapshotTree is a simple LSM tree representation of the key/value storage // at a given point in time. type SnapshotTree struct { - base state.StorageSnapshot + base StorageSnapshot - fullLog updateLog - compactedLog updateLog + compactedLog UpdateLog } // NewSnapshotTree returns a tree with keys/values initialized to the base // storage snapshot. -func NewSnapshotTree(base state.StorageSnapshot) SnapshotTree { +func NewSnapshotTree(base StorageSnapshot) SnapshotTree { return SnapshotTree{ base: base, - fullLog: nil, compactedLog: nil, } } @@ -33,7 +30,7 @@ func NewSnapshotTree(base state.StorageSnapshot) SnapshotTree { // Append returns a new tree with updates from the execution snapshot "applied" // to the original original tree. func (tree SnapshotTree) Append( - update *state.ExecutionSnapshot, + update *ExecutionSnapshot, ) SnapshotTree { compactedLog := tree.compactedLog if len(update.WriteSet) > 0 { @@ -51,13 +48,12 @@ func (tree SnapshotTree) Append( } } - compactedLog = updateLog{mergedSet} + compactedLog = UpdateLog{mergedSet} } } return SnapshotTree{ base: tree.base, - fullLog: append(tree.fullLog, update.WriteSet), compactedLog: compactedLog, } } diff --git a/fvm/storage/snapshot_tree_test.go b/fvm/storage/snapshot/snapshot_tree_test.go similarity index 84% rename from fvm/storage/snapshot_tree_test.go rename to fvm/storage/snapshot/snapshot_tree_test.go index 025195ccf86..5ccf83481e6 100644 --- a/fvm/storage/snapshot_tree_test.go +++ b/fvm/storage/snapshot/snapshot_tree_test.go @@ -1,4 +1,4 @@ -package storage +package snapshot import ( "fmt" @@ -6,7 +6,6 @@ import ( "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/model/flow" ) @@ -21,7 +20,7 @@ func TestSnapshotTree(t *testing.T) { // entries: // 1 -> 1v0 tree0 := NewSnapshotTree( - state.MapStorageSnapshot{ + MapStorageSnapshot{ id1: value1v0, }) @@ -35,7 +34,7 @@ func TestSnapshotTree(t *testing.T) { value2v1 := flow.RegisterValue("2v1") tree1 := tree0.Append( - &state.ExecutionSnapshot{ + &ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ id2: value2v1, }, @@ -52,7 +51,7 @@ func TestSnapshotTree(t *testing.T) { value3v1 := flow.RegisterValue("3v1") tree2 := tree1.Append( - &state.ExecutionSnapshot{ + &ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ id1: value1v1, id3: value3v1, @@ -69,7 +68,7 @@ func TestSnapshotTree(t *testing.T) { value2v2 := flow.RegisterValue("2v2") tree3 := tree2.Append( - &state.ExecutionSnapshot{ + &ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ id2: value2v2, }, @@ -95,7 +94,7 @@ func TestSnapshotTree(t *testing.T) { value := []byte(fmt.Sprintf("compacted %d", i)) expectedCompacted[id3] = value compactedTree = compactedTree.Append( - &state.ExecutionSnapshot{ + &ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ id3: value, }, @@ -105,10 +104,8 @@ func TestSnapshotTree(t *testing.T) { check := func( tree SnapshotTree, expected map[flow.RegisterID]flow.RegisterValue, - fullLogLen int, compactedLogLen int, ) { - require.Len(t, tree.fullLog, fullLogLen) require.Len(t, tree.compactedLog, compactedLogLen) for key, expectedValue := range expected { @@ -118,11 +115,11 @@ func TestSnapshotTree(t *testing.T) { } } - check(tree0, expected0, 0, 0) - check(tree1, expected1, 1, 1) - check(tree2, expected2, 2, 2) - check(tree3, expected3, 3, 3) - check(compactedTree, expectedCompacted, 3+numExtraUpdates, 4) + check(tree0, expected0, 0) + check(tree1, expected1, 1) + check(tree2, expected2, 2) + check(tree3, expected3, 3) + check(compactedTree, expectedCompacted, 4) emptyTree := NewSnapshotTree(nil) value, err := emptyTree.Get(id1) diff --git a/fvm/state/storage_snapshot.go b/fvm/storage/snapshot/storage_snapshot.go similarity index 86% rename from fvm/state/storage_snapshot.go rename to fvm/storage/snapshot/storage_snapshot.go index 840ff984ca4..7d063e0b76e 100644 --- a/fvm/state/storage_snapshot.go +++ b/fvm/storage/snapshot/storage_snapshot.go @@ -1,12 +1,14 @@ -package state +package snapshot import ( "github.com/onflow/flow-go/model/flow" ) +// Note: StorageSnapshot must be thread safe (or immutable). type StorageSnapshot interface { // Get returns the register id's value, or an empty RegisterValue if the id - // is not found. + // is not found. Get should be idempotent (i.e., the same value is returned + // for the same id). Get(id flow.RegisterID) (flow.RegisterValue, error) } diff --git a/fvm/state/execution_state.go b/fvm/storage/state/execution_state.go similarity index 88% rename from fvm/state/execution_state.go rename to fvm/storage/state/execution_state.go index f84760720cf..8f9a03f2dab 100644 --- a/fvm/state/execution_state.go +++ b/fvm/storage/state/execution_state.go @@ -7,6 +7,7 @@ import ( "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/meter" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" ) @@ -15,8 +16,6 @@ const ( DefaultMaxValueSize = 256_000_000 // ~256MB ) -// TODO(patrick): make State implement the View interface. -// // State represents the execution state // it holds draft of updates and captures // all register touches @@ -26,7 +25,7 @@ type ExecutionState struct { // bookkeeping purpose). finalized bool - view View + *spockState meter *meter.Meter // NOTE: parent and child state shares the same limits controller @@ -99,16 +98,15 @@ func (controller *limitsController) RunWithAllLimitsDisabled(f func()) { controller.enforceLimits = current } -func (state *ExecutionState) View() View { - return state.view -} - // NewExecutionState constructs a new state -func NewExecutionState(view View, params StateParameters) *ExecutionState { +func NewExecutionState( + snapshot snapshot.StorageSnapshot, + params StateParameters, +) *ExecutionState { m := meter.NewMeter(params.MeterParameters) return &ExecutionState{ finalized: false, - view: view, + spockState: newSpockState(snapshot), meter: m, limitsController: newLimitsController(params), } @@ -121,7 +119,7 @@ func (state *ExecutionState) NewChildWithMeterParams( ) *ExecutionState { return &ExecutionState{ finalized: false, - view: state.view.NewChild(), + spockState: state.spockState.NewChild(), meter: meter.NewMeter(params), limitsController: state.limitsController, } @@ -147,7 +145,7 @@ func (state *ExecutionState) DropChanges() error { return fmt.Errorf("cannot DropChanges on a finalized state") } - return state.view.DropChanges() + return state.spockState.DropChanges() } // Get returns a register value given owner and key @@ -165,7 +163,7 @@ func (state *ExecutionState) Get(id flow.RegisterID) (flow.RegisterValue, error) } } - if value, err = state.view.Get(id); err != nil { + if value, err = state.spockState.Get(id); err != nil { // wrap error into a fatal error getError := errors.NewLedgerFailure(err) // wrap with more info @@ -188,7 +186,7 @@ func (state *ExecutionState) Set(id flow.RegisterID, value flow.RegisterValue) e } } - if err := state.view.Set(id, value); err != nil { + if err := state.spockState.Set(id, value); err != nil { // wrap error into a fatal error setError := errors.NewLedgerFailure(err) // wrap with more info @@ -269,20 +267,20 @@ func (state *ExecutionState) TotalEmittedEventBytes() uint64 { return state.meter.TotalEmittedEventBytes() } -func (state *ExecutionState) Finalize() *ExecutionSnapshot { +func (state *ExecutionState) Finalize() *snapshot.ExecutionSnapshot { state.finalized = true - snapshot := state.view.Finalize() + snapshot := state.spockState.Finalize() snapshot.Meter = state.meter return snapshot } -// MergeState the changes from a the given view to this view. -func (state *ExecutionState) Merge(other *ExecutionSnapshot) error { +// MergeState the changes from a the given execution snapshot to this state. +func (state *ExecutionState) Merge(other *snapshot.ExecutionSnapshot) error { if state.finalized { return fmt.Errorf("cannot Merge on a finalized state") } - err := state.view.Merge(other) + err := state.spockState.Merge(other) if err != nil { return errors.NewStateMergeFailure(err) } @@ -311,3 +309,13 @@ func (state *ExecutionState) checkSize( } return nil } + +func (state *ExecutionState) readSetSize() int { + return state.spockState.readSetSize() +} + +func (state *ExecutionState) interimReadSet( + accumulator map[flow.RegisterID]struct{}, +) { + state.spockState.interimReadSet(accumulator) +} diff --git a/fvm/state/execution_state_test.go b/fvm/storage/state/execution_state_test.go similarity index 93% rename from fvm/state/execution_state_test.go rename to fvm/storage/state/execution_state_test.go index 5fbfd42efd5..84184f1f4f7 100644 --- a/fvm/state/execution_state_test.go +++ b/fvm/storage/state/execution_state_test.go @@ -5,9 +5,8 @@ import ( "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm/meter" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/model/flow" ) @@ -20,8 +19,7 @@ func createByteArray(size int) []byte { } func TestExecutionState_Finalize(t *testing.T) { - view := delta.NewDeltaView(nil) - parent := state.NewExecutionState(view, state.DefaultParameters()) + parent := state.NewExecutionState(nil, state.DefaultParameters()) child := parent.NewChild() @@ -65,8 +63,7 @@ func TestExecutionState_Finalize(t *testing.T) { } func TestExecutionState_ChildMergeFunctionality(t *testing.T) { - view := delta.NewDeltaView(nil) - st := state.NewExecutionState(view, state.DefaultParameters()) + st := state.NewExecutionState(nil, state.DefaultParameters()) t.Run("test read from parent state (backoff)", func(t *testing.T) { key := flow.NewRegisterID("address", "key1") @@ -137,9 +134,8 @@ func TestExecutionState_ChildMergeFunctionality(t *testing.T) { } func TestExecutionState_MaxValueSize(t *testing.T) { - view := delta.NewDeltaView(nil) st := state.NewExecutionState( - view, + nil, state.DefaultParameters().WithMaxValueSizeAllowed(6)) key := flow.NewRegisterID("address", "key") @@ -156,9 +152,8 @@ func TestExecutionState_MaxValueSize(t *testing.T) { } func TestExecutionState_MaxKeySize(t *testing.T) { - view := delta.NewDeltaView(nil) st := state.NewExecutionState( - view, + nil, // Note: owners are always 8 bytes state.DefaultParameters().WithMaxKeySizeAllowed(8+2)) @@ -184,8 +179,6 @@ func TestExecutionState_MaxKeySize(t *testing.T) { } func TestExecutionState_MaxInteraction(t *testing.T) { - view := delta.NewDeltaView(nil) - key1 := flow.NewRegisterID("1", "2") key1Size := uint64(8 + 1) @@ -202,7 +195,7 @@ func TestExecutionState_MaxInteraction(t *testing.T) { key4Size := uint64(8 + 4) st := state.NewExecutionState( - view, + nil, state.DefaultParameters(). WithMeterParameters( meter.DefaultParameters().WithStorageInteractionLimit( @@ -224,7 +217,7 @@ func TestExecutionState_MaxInteraction(t *testing.T) { require.Equal(t, st.InteractionUsed(), key1Size+key2Size+key3Size) st = state.NewExecutionState( - view, + nil, state.DefaultParameters(). WithMeterParameters( meter.DefaultParameters().WithStorageInteractionLimit( diff --git a/fvm/state/spock_state.go b/fvm/storage/state/spock_state.go similarity index 87% rename from fvm/state/spock_state.go rename to fvm/storage/state/spock_state.go index c1f5cd3ace0..9a47ac08710 100644 --- a/fvm/state/spock_state.go +++ b/fvm/storage/state/spock_state.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/onflow/flow-go/crypto/hash" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" ) @@ -29,27 +30,21 @@ type spockState struct { finalizedSpockSecret []byte } -// TODO(patrick): rm after delta view is deleted. -func NewSpockState(base StorageSnapshot) *spockState { - return newSpockState(base) -} - -func newSpockState(base StorageSnapshot) *spockState { +func newSpockState(base snapshot.StorageSnapshot) *spockState { return &spockState{ storageState: newStorageState(base), spockSecretHasher: hash.NewSHA3_256(), } } -// TODO(patrick): change return type to *spockState -func (state *spockState) NewChild() View { +func (state *spockState) NewChild() *spockState { return &spockState{ storageState: state.storageState.NewChild(), spockSecretHasher: hash.NewSHA3_256(), } } -func (state *spockState) Finalize() *ExecutionSnapshot { +func (state *spockState) Finalize() *snapshot.ExecutionSnapshot { if state.finalizedSpockSecret == nil { state.finalizedSpockSecret = state.spockSecretHasher.SumHash() } @@ -59,7 +54,7 @@ func (state *spockState) Finalize() *ExecutionSnapshot { return snapshot } -func (state *spockState) Merge(snapshot *ExecutionSnapshot) error { +func (state *spockState) Merge(snapshot *snapshot.ExecutionSnapshot) error { if state.finalizedSpockSecret != nil { return fmt.Errorf("cannot Merge on a finalized state") } @@ -170,3 +165,13 @@ func (state *spockState) DropChanges() error { return state.storageState.DropChanges() } + +func (state *spockState) readSetSize() int { + return state.storageState.readSetSize() +} + +func (state *spockState) interimReadSet( + accumulator map[flow.RegisterID]struct{}, +) { + state.storageState.interimReadSet(accumulator) +} diff --git a/fvm/state/spock_state_test.go b/fvm/storage/state/spock_state_test.go similarity index 95% rename from fvm/state/spock_state_test.go rename to fvm/storage/state/spock_state_test.go index 6957e9fd2d6..eafd30c1305 100644 --- a/fvm/state/spock_state_test.go +++ b/fvm/storage/state/spock_state_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/rand" ) @@ -27,8 +28,8 @@ func testSpock( ) []*spockState { resultStates := []*spockState{} for _, experiment := range counterfactualExperiments { - run1 := newSpockState(MapStorageSnapshot{}) - run2 := newSpockState(MapStorageSnapshot{}) + run1 := newSpockState(snapshot.MapStorageSnapshot{}) + run2 := newSpockState(snapshot.MapStorageSnapshot{}) if experiment != nil { experiment(t, run1) @@ -99,12 +100,12 @@ func TestSpockStateGetDifferentUnderlyingStorage(t *testing.T) { value2 := flow.RegisterValue([]byte("blah")) state1 := newSpockState( - MapStorageSnapshot{ + snapshot.MapStorageSnapshot{ badRegisterId: value1, }) state2 := newSpockState( - MapStorageSnapshot{ + snapshot.MapStorageSnapshot{ badRegisterId: value2, }) @@ -223,7 +224,7 @@ func TestSpockStateMerge(t *testing.T) { // primary experiment func(t *testing.T, state *spockState) { err := state.Merge( - &ExecutionSnapshot{ + &snapshot.ExecutionSnapshot{ ReadSet: readSet, SpockSecret: []byte("secret"), }) @@ -232,13 +233,13 @@ func TestSpockStateMerge(t *testing.T) { // duplicate calls result in different spock func(t *testing.T, state *spockState) { err := state.Merge( - &ExecutionSnapshot{ + &snapshot.ExecutionSnapshot{ ReadSet: readSet, SpockSecret: []byte("secret"), }) require.NoError(t, err) err = state.Merge( - &ExecutionSnapshot{ + &snapshot.ExecutionSnapshot{ ReadSet: readSet, SpockSecret: []byte("secret"), }) @@ -248,7 +249,7 @@ func TestSpockStateMerge(t *testing.T) { // different spock func(t *testing.T, state *spockState) { err := state.Merge( - &ExecutionSnapshot{ + &snapshot.ExecutionSnapshot{ ReadSet: readSet, SpockSecret: []byte("secreT"), }) @@ -260,7 +261,7 @@ func TestSpockStateMerge(t *testing.T) { require.Equal(t, readSet, states[1].Finalize().ReadSet) // Sanity check finalized state is no longer accessible. - err := states[1].Merge(&ExecutionSnapshot{}) + err := states[1].Merge(&snapshot.ExecutionSnapshot{}) require.ErrorContains(t, err, "cannot Merge on a finalized state") } func TestSpockStateDropChanges(t *testing.T) { @@ -360,7 +361,7 @@ func TestSpockStateRandomOps(t *testing.T) { chain[len(chain)-1], func(t *testing.T, state *spockState) { err := state.Merge( - &ExecutionSnapshot{ + &snapshot.ExecutionSnapshot{ SpockSecret: []byte(fmt.Sprintf("%d", spock)), }) require.NoError(t, err) @@ -381,7 +382,6 @@ func TestSpockStateRandomOps(t *testing.T) { _ = testSpock(t, chain) } - func TestSpockStateNewChild(t *testing.T) { baseRegisterId := flow.NewRegisterID("", "base") baseValue := flow.RegisterValue([]byte("base")) @@ -397,7 +397,7 @@ func TestSpockStateNewChild(t *testing.T) { childRegisterId2 := flow.NewRegisterID("child", "2") parent := newSpockState( - MapStorageSnapshot{ + snapshot.MapStorageSnapshot{ baseRegisterId: baseValue, }) diff --git a/fvm/state/storage_state.go b/fvm/storage/state/storage_state.go similarity index 71% rename from fvm/state/storage_state.go rename to fvm/storage/state/storage_state.go index 1b2ad0f6cbf..e4b92e16969 100644 --- a/fvm/state/storage_state.go +++ b/fvm/storage/state/storage_state.go @@ -3,11 +3,12 @@ package state import ( "fmt" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" ) type storageState struct { - baseStorage StorageSnapshot + baseStorage snapshot.StorageSnapshot // The read set only include reads from the baseStorage readSet map[flow.RegisterID]struct{} @@ -15,7 +16,7 @@ type storageState struct { writeSet map[flow.RegisterID]flow.RegisterValue } -func newStorageState(base StorageSnapshot) *storageState { +func newStorageState(base snapshot.StorageSnapshot) *storageState { return &storageState{ baseStorage: base, readSet: map[flow.RegisterID]struct{}{}, @@ -24,17 +25,17 @@ func newStorageState(base StorageSnapshot) *storageState { } func (state *storageState) NewChild() *storageState { - return newStorageState(NewPeekerStorageSnapshot(state)) + return newStorageState(snapshot.NewPeekerStorageSnapshot(state)) } -func (state *storageState) Finalize() *ExecutionSnapshot { - return &ExecutionSnapshot{ +func (state *storageState) Finalize() *snapshot.ExecutionSnapshot { + return &snapshot.ExecutionSnapshot{ ReadSet: state.readSet, WriteSet: state.writeSet, } } -func (state *storageState) Merge(snapshot *ExecutionSnapshot) error { +func (state *storageState) Merge(snapshot *snapshot.ExecutionSnapshot) error { for id := range snapshot.ReadSet { _, ok := state.writeSet[id] if ok { @@ -114,3 +115,19 @@ func (state *storageState) DropChanges() error { state.writeSet = map[flow.RegisterID]flow.RegisterValue{} return nil } + +func (state *storageState) readSetSize() int { + return len(state.readSet) +} + +func (state *storageState) interimReadSet( + accumulator map[flow.RegisterID]struct{}, +) { + for id := range state.writeSet { + delete(accumulator, id) + } + + for id := range state.readSet { + accumulator[id] = struct{}{} + } +} diff --git a/fvm/state/storage_state_test.go b/fvm/storage/state/storage_state_test.go similarity index 98% rename from fvm/state/storage_state_test.go rename to fvm/storage/state/storage_state_test.go index e682c65a29f..87ff6a195ac 100644 --- a/fvm/state/storage_state_test.go +++ b/fvm/storage/state/storage_state_test.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" ) @@ -49,7 +50,7 @@ func TestStorageStateGetFromBase(t *testing.T) { baseValue := flow.RegisterValue([]byte("base")) state := newStorageState( - MapStorageSnapshot{ + snapshot.MapStorageSnapshot{ registerId: baseValue, }) @@ -129,7 +130,7 @@ func TestStorageStateMerge(t *testing.T) { childRegisterId2 := flow.NewRegisterID("child", "2") parent := newStorageState( - MapStorageSnapshot{ + snapshot.MapStorageSnapshot{ baseRegisterId: baseValue, }) diff --git a/fvm/state/transaction_state.go b/fvm/storage/state/transaction_state.go similarity index 85% rename from fvm/state/transaction_state.go rename to fvm/storage/state/transaction_state.go index 677c3b8896d..602fa282585 100644 --- a/fvm/state/transaction_state.go +++ b/fvm/storage/state/transaction_state.go @@ -6,6 +6,7 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-go/fvm/meter" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" ) @@ -37,9 +38,9 @@ type Meter interface { RunWithAllLimitsDisabled(f func()) } -// NestedTransaction provides active transaction states and facilitates common -// state management operations. -type NestedTransaction interface { +// NestedTransactionPreparer provides active transaction states and facilitates +// common state management operations. +type NestedTransactionPreparer interface { Meter // NumNestedTransactions returns the number of uncommitted nested @@ -57,11 +58,15 @@ type NestedTransaction interface { // transaction. IsCurrent(id NestedTransactionId) bool + // InterimReadSet returns the current read set aggregated from all + // outstanding nested transactions. + InterimReadSet() map[flow.RegisterID]struct{} + // FinalizeMainTransaction finalizes the main transaction and returns // its execution snapshot. The finalized main transaction will not accept // any new commits after this point. This returns an error if there are // outstanding nested transactions. - FinalizeMainTransaction() (*ExecutionSnapshot, error) + FinalizeMainTransaction() (*snapshot.ExecutionSnapshot, error) // BeginNestedTransaction creates a unrestricted nested transaction within // the current unrestricted (nested) transaction. The meter parameters are @@ -106,7 +111,7 @@ type NestedTransaction interface { CommitNestedTransaction( expectedId NestedTransactionId, ) ( - *ExecutionSnapshot, + *snapshot.ExecutionSnapshot, error, ) @@ -124,35 +129,16 @@ type NestedTransaction interface { CommitParseRestrictedNestedTransaction( location common.AddressLocation, ) ( - *ExecutionSnapshot, - error, - ) - - // PauseNestedTransaction detaches the current nested transaction from the - // parent transaction, and returns the paused nested transaction state. - // The paused nested transaction may be resume via Resume. - // - // WARNING: Pause and Resume are intended for implementing continuation - // passing style behavior for the transaction executor, with the assumption - // that the states accessed prior to pausing remain valid after resumption. - // The paused nested transaction should not be reused across transactions. - // IT IS NOT SAFE TO PAUSE A NESTED TRANSACTION IN GENERAL SINCE THAT - // COULD LEAD TO PHANTOM READS. - PauseNestedTransaction( - expectedId NestedTransactionId, - ) ( - *ExecutionState, + *snapshot.ExecutionSnapshot, error, ) - // ResumeNestedTransaction attaches the paused nested transaction (state) - // to the current transaction. - ResumeNestedTransaction(pausedState *ExecutionState) - // AttachAndCommitNestedTransaction commits the changes from the cached // nested transaction execution snapshot to the current (nested) // transaction. - AttachAndCommitNestedTransaction(cachedSnapshot *ExecutionSnapshot) error + AttachAndCommitNestedTransaction( + cachedSnapshot *snapshot.ExecutionSnapshot, + ) error // RestartNestedTransaction merges all changes that belongs to the nested // transaction about to be restart (for spock/meter bookkeeping), then @@ -164,8 +150,6 @@ type NestedTransaction interface { Get(id flow.RegisterID) (flow.RegisterValue, error) Set(id flow.RegisterID, value flow.RegisterValue) error - - ViewForTestingOnly() View } type nestedTransactionStackFrame struct { @@ -188,10 +172,10 @@ type transactionState struct { // NewTransactionState constructs a new state transaction which manages nested // transactions. func NewTransactionState( - startView View, + snapshot snapshot.StorageSnapshot, params StateParameters, -) NestedTransaction { - startState := NewExecutionState(startView, params) +) NestedTransactionPreparer { + startState := NewExecutionState(snapshot, params) return &transactionState{ nestedTransactions: []nestedTransactionStackFrame{ nestedTransactionStackFrame{ @@ -224,8 +208,25 @@ func (txnState *transactionState) IsCurrent(id NestedTransactionId) bool { return txnState.current().ExecutionState == id.state } +func (txnState *transactionState) InterimReadSet() map[flow.RegisterID]struct{} { + sizeEstimate := 0 + for _, frame := range txnState.nestedTransactions { + sizeEstimate += frame.readSetSize() + } + + result := make(map[flow.RegisterID]struct{}, sizeEstimate) + + // Note: the interim read set must be accumulated in reverse order since + // the parent frame's write set will override the child frame's read set. + for i := len(txnState.nestedTransactions) - 1; i >= 0; i-- { + txnState.nestedTransactions[i].interimReadSet(result) + } + + return result +} + func (txnState *transactionState) FinalizeMainTransaction() ( - *ExecutionSnapshot, + *snapshot.ExecutionSnapshot, error, ) { if len(txnState.nestedTransactions) > 1 { @@ -314,7 +315,10 @@ func (txnState *transactionState) pop(op string) (*ExecutionState, error) { return child.ExecutionState, nil } -func (txnState *transactionState) mergeIntoParent() (*ExecutionSnapshot, error) { +func (txnState *transactionState) mergeIntoParent() ( + *snapshot.ExecutionSnapshot, + error, +) { childState, err := txnState.pop("commit") if err != nil { return nil, err @@ -333,7 +337,7 @@ func (txnState *transactionState) mergeIntoParent() (*ExecutionSnapshot, error) func (txnState *transactionState) CommitNestedTransaction( expectedId NestedTransactionId, ) ( - *ExecutionSnapshot, + *snapshot.ExecutionSnapshot, error, ) { if !txnState.IsCurrent(expectedId) { @@ -355,7 +359,7 @@ func (txnState *transactionState) CommitNestedTransaction( func (txnState *transactionState) CommitParseRestrictedNestedTransaction( location common.AddressLocation, ) ( - *ExecutionSnapshot, + *snapshot.ExecutionSnapshot, error, ) { currentFrame := txnState.current() @@ -373,32 +377,8 @@ func (txnState *transactionState) CommitParseRestrictedNestedTransaction( return txnState.mergeIntoParent() } -func (txnState *transactionState) PauseNestedTransaction( - expectedId NestedTransactionId, -) ( - *ExecutionState, - error, -) { - if !txnState.IsCurrent(expectedId) { - return nil, fmt.Errorf( - "cannot pause unexpected nested transaction: id mismatch", - ) - } - - if txnState.IsParseRestricted() { - return nil, fmt.Errorf( - "cannot Pause parse restricted nested transaction") - } - - return txnState.pop("pause") -} - -func (txnState *transactionState) ResumeNestedTransaction(pausedState *ExecutionState) { - txnState.push(pausedState, nil) -} - func (txnState *transactionState) AttachAndCommitNestedTransaction( - cachedSnapshot *ExecutionSnapshot, + cachedSnapshot *snapshot.ExecutionSnapshot, ) error { return txnState.current().Merge(cachedSnapshot) } @@ -494,10 +474,6 @@ func (txnState *transactionState) TotalEmittedEventBytes() uint64 { return txnState.current().TotalEmittedEventBytes() } -func (txnState *transactionState) ViewForTestingOnly() View { - return txnState.current().View() -} - func (txnState *transactionState) RunWithAllLimitsDisabled(f func()) { txnState.current().RunWithAllLimitsDisabled(f) } diff --git a/fvm/state/transaction_state_test.go b/fvm/storage/state/transaction_state_test.go similarity index 86% rename from fvm/state/transaction_state_test.go rename to fvm/storage/state/transaction_state_test.go index 0b0b67c48b0..5f91fe8b4b5 100644 --- a/fvm/state/transaction_state_test.go +++ b/fvm/storage/state/transaction_state_test.go @@ -7,15 +7,14 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm/meter" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/model/flow" ) -func newTestTransactionState() state.NestedTransaction { +func newTestTransactionState() state.NestedTransactionPreparer { return state.NewTransactionState( - delta.NewDeltaView(nil), + nil, state.DefaultParameters(), ) } @@ -197,7 +196,7 @@ func TestParseRestrictedNestedTransactionBasic(t *testing.T) { val := createByteArray(2) cachedState := state.NewExecutionState( - delta.NewDeltaView(nil), + nil, state.DefaultParameters(), ) @@ -310,7 +309,7 @@ func TestRestartNestedTransaction(t *testing.T) { state := id.StateForTestingOnly() require.Equal(t, uint64(0), state.InteractionUsed()) - // Restart will merge the meter stat, but not the view delta + // Restart will merge the meter stat, but not the register updates err = txn.RestartNestedTransaction(id) require.NoError(t, err) @@ -480,50 +479,6 @@ func TestParseRestrictedCannotCommitLocationMismatch(t *testing.T) { require.True(t, txn.IsCurrent(id)) } -func TestPauseAndResume(t *testing.T) { - txn := newTestTransactionState() - - key1 := flow.NewRegisterID("addr", "key") - key2 := flow.NewRegisterID("addr2", "key2") - - val, err := txn.Get(key1) - require.NoError(t, err) - require.Nil(t, val) - - id1, err := txn.BeginNestedTransaction() - require.NoError(t, err) - - err = txn.Set(key1, createByteArray(2)) - require.NoError(t, err) - - val, err = txn.Get(key1) - require.NoError(t, err) - require.NotNil(t, val) - - pausedState, err := txn.PauseNestedTransaction(id1) - require.NoError(t, err) - - val, err = txn.Get(key1) - require.NoError(t, err) - require.Nil(t, val) - - txn.ResumeNestedTransaction(pausedState) - - val, err = txn.Get(key1) - require.NoError(t, err) - require.NotNil(t, val) - - err = txn.Set(key2, createByteArray(2)) - require.NoError(t, err) - - _, err = txn.CommitNestedTransaction(id1) - require.NoError(t, err) - - val, err = txn.Get(key2) - require.NoError(t, err) - require.NotNil(t, val) -} - func TestFinalizeMainTransactionFailWithUnexpectedNestedTransactions( t *testing.T, ) { @@ -570,3 +525,85 @@ func TestFinalizeMainTransaction(t *testing.T) { _, err = txn.Get(registerId) require.ErrorContains(t, err, "cannot Get on a finalized state") } + +func TestInterimReadSet(t *testing.T) { + txn := newTestTransactionState() + + // Setup test with a bunch of outstanding nested transaction. + + readRegisterId1 := flow.NewRegisterID("read", "1") + readRegisterId2 := flow.NewRegisterID("read", "2") + readRegisterId3 := flow.NewRegisterID("read", "3") + readRegisterId4 := flow.NewRegisterID("read", "4") + + writeRegisterId1 := flow.NewRegisterID("write", "1") + writeValue1 := flow.RegisterValue([]byte("value1")) + + writeRegisterId2 := flow.NewRegisterID("write", "2") + writeValue2 := flow.RegisterValue([]byte("value2")) + + writeRegisterId3 := flow.NewRegisterID("write", "3") + writeValue3 := flow.RegisterValue([]byte("value3")) + + err := txn.Set(writeRegisterId1, writeValue1) + require.NoError(t, err) + + _, err = txn.Get(readRegisterId1) + require.NoError(t, err) + + _, err = txn.Get(readRegisterId2) + require.NoError(t, err) + + value, err := txn.Get(writeRegisterId1) + require.NoError(t, err) + require.Equal(t, writeValue1, value) + + _, err = txn.BeginNestedTransaction() + require.NoError(t, err) + + err = txn.Set(readRegisterId2, []byte("blah")) + require.NoError(t, err) + + _, err = txn.Get(readRegisterId3) + require.NoError(t, err) + + value, err = txn.Get(writeRegisterId1) + require.NoError(t, err) + require.Equal(t, writeValue1, value) + + err = txn.Set(writeRegisterId2, writeValue2) + require.NoError(t, err) + + _, err = txn.BeginNestedTransaction() + require.NoError(t, err) + + err = txn.Set(writeRegisterId3, writeValue3) + require.NoError(t, err) + + value, err = txn.Get(writeRegisterId1) + require.NoError(t, err) + require.Equal(t, writeValue1, value) + + value, err = txn.Get(writeRegisterId2) + require.NoError(t, err) + require.Equal(t, writeValue2, value) + + value, err = txn.Get(writeRegisterId3) + require.NoError(t, err) + require.Equal(t, writeValue3, value) + + _, err = txn.Get(readRegisterId4) + require.NoError(t, err) + + // Actual test + + require.Equal( + t, + map[flow.RegisterID]struct{}{ + readRegisterId1: struct{}{}, + readRegisterId2: struct{}{}, + readRegisterId3: struct{}{}, + readRegisterId4: struct{}{}, + }, + txn.InterimReadSet()) +} diff --git a/fvm/storage/testutils/utils.go b/fvm/storage/testutils/utils.go index e2727a9a247..2c5fb00311f 100644 --- a/fvm/storage/testutils/utils.go +++ b/fvm/storage/testutils/utils.go @@ -1,26 +1,22 @@ package testutils import ( - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage" - "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" ) // NewSimpleTransaction returns a transaction which can be used to test // fvm evaluation. The returned transaction should not be committed. func NewSimpleTransaction( - snapshot state.StorageSnapshot, -) *storage.SerialTransaction { - derivedBlockData := derived.NewEmptyDerivedBlockData() - derivedTxnData, err := derivedBlockData.NewDerivedTransactionData(0, 0) + snapshot snapshot.StorageSnapshot, +) storage.Transaction { + blockDatabase := storage.NewBlockDatabase(snapshot, 0, nil) + + txn, err := blockDatabase.NewTransaction(0, state.DefaultParameters()) if err != nil { panic(err) } - return &storage.SerialTransaction{ - NestedTransaction: state.NewTransactionState( - state.NewSpockState(snapshot), - state.DefaultParameters()), - DerivedTransactionCommitter: derivedTxnData, - } + return txn } diff --git a/fvm/storage/transaction.go b/fvm/storage/transaction.go index fe1520bc52b..cd89eb302da 100644 --- a/fvm/storage/transaction.go +++ b/fvm/storage/transaction.go @@ -1,17 +1,26 @@ package storage import ( - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" ) -type Transaction interface { - state.NestedTransaction - derived.DerivedTransaction +type TransactionPreparer interface { + state.NestedTransactionPreparer + derived.DerivedTransactionPreparer } -type TransactionComitter interface { - Transaction +type Transaction interface { + TransactionPreparer + + // SnapshotTime returns the transaction's current snapshot time. + SnapshotTime() logical.Time + + // Finalize convert transaction preparer's intermediate state into + // committable state. + Finalize() error // Validate returns nil if the transaction does not conflict with // previously committed transactions. It returns an error otherwise. @@ -20,11 +29,5 @@ type TransactionComitter interface { // Commit commits the transaction. If the transaction conflict with // previously committed transactions, an error is returned and the // transaction is not committed. - Commit() error -} - -// TODO(patrick): implement proper transaction. -type SerialTransaction struct { - state.NestedTransaction - derived.DerivedTransactionCommitter + Commit() (*snapshot.ExecutionSnapshot, error) } diff --git a/fvm/systemcontracts/system_contracts.go b/fvm/systemcontracts/system_contracts.go index 99555c640a0..42da24154d0 100644 --- a/fvm/systemcontracts/system_contracts.go +++ b/fvm/systemcontracts/system_contracts.go @@ -23,17 +23,19 @@ const ( // Unqualified names of system smart contracts (not including address prefix) - ContractNameEpoch = "FlowEpoch" - ContractNameClusterQC = "FlowClusterQC" - ContractNameDKG = "FlowDKG" - ContractNameServiceAccount = "FlowServiceAccount" - ContractNameFlowFees = "FlowFees" - ContractNameStorageFees = "FlowStorageFees" + ContractNameEpoch = "FlowEpoch" + ContractNameClusterQC = "FlowClusterQC" + ContractNameDKG = "FlowDKG" + ContractNameServiceAccount = "FlowServiceAccount" + ContractNameFlowFees = "FlowFees" + ContractNameStorageFees = "FlowStorageFees" + ContractNameNodeVersionBeacon = "NodeVersionBeacon" // Unqualified names of service events (not including address prefix or contract name) - EventNameEpochSetup = "EpochSetup" - EventNameEpochCommit = "EpochCommit" + EventNameEpochSetup = "EpochSetup" + EventNameEpochCommit = "EpochCommit" + EventNameVersionBeacon = "VersionBeacon" // Unqualified names of service event contract functions (not including address prefix or contract name) @@ -73,15 +75,17 @@ func (se ServiceEvent) EventType() flow.EventType { // SystemContracts is a container for all system contracts on a particular chain. type SystemContracts struct { - Epoch SystemContract - ClusterQC SystemContract - DKG SystemContract + Epoch SystemContract + ClusterQC SystemContract + DKG SystemContract + NodeVersionBeacon SystemContract } // ServiceEvents is a container for all service events on a particular chain. type ServiceEvents struct { - EpochSetup ServiceEvent - EpochCommit ServiceEvent + EpochSetup ServiceEvent + EpochCommit ServiceEvent + VersionBeacon ServiceEvent } // All returns all service events as a slice. @@ -89,6 +93,7 @@ func (se ServiceEvents) All() []ServiceEvent { return []ServiceEvent{ se.EpochSetup, se.EpochCommit, + se.VersionBeacon, } } @@ -112,6 +117,10 @@ func SystemContractsForChain(chainID flow.ChainID) (*SystemContracts, error) { Address: addresses[ContractNameDKG], Name: ContractNameDKG, }, + NodeVersionBeacon: SystemContract{ + Address: addresses[ContractNameNodeVersionBeacon], + Name: ContractNameNodeVersionBeacon, + }, } return contracts, nil @@ -135,6 +144,11 @@ func ServiceEventsForChain(chainID flow.ChainID) (*ServiceEvents, error) { ContractName: ContractNameEpoch, Name: EventNameEpochCommit, }, + VersionBeacon: ServiceEvent{ + Address: addresses[ContractNameNodeVersionBeacon], + ContractName: ContractNameNodeVersionBeacon, + Name: EventNameVersionBeacon, + }, } return events, nil @@ -162,40 +176,43 @@ func init() { // Main Flow network // All system contracts are deployed to the account of the staking contract mainnet := map[string]flow.Address{ - ContractNameEpoch: stakingContractAddressMainnet, - ContractNameClusterQC: stakingContractAddressMainnet, - ContractNameDKG: stakingContractAddressMainnet, + ContractNameEpoch: stakingContractAddressMainnet, + ContractNameClusterQC: stakingContractAddressMainnet, + ContractNameDKG: stakingContractAddressMainnet, + ContractNameNodeVersionBeacon: flow.Mainnet.Chain().ServiceAddress(), } contractAddressesByChainID[flow.Mainnet] = mainnet // Long-lived test networks // All system contracts are deployed to the account of the staking contract testnet := map[string]flow.Address{ - ContractNameEpoch: stakingContractAddressTestnet, - ContractNameClusterQC: stakingContractAddressTestnet, - ContractNameDKG: stakingContractAddressTestnet, + ContractNameEpoch: stakingContractAddressTestnet, + ContractNameClusterQC: stakingContractAddressTestnet, + ContractNameDKG: stakingContractAddressTestnet, + ContractNameNodeVersionBeacon: flow.Testnet.Chain().ServiceAddress(), } contractAddressesByChainID[flow.Testnet] = testnet // Sandboxnet test network // All system contracts are deployed to the service account sandboxnet := map[string]flow.Address{ - ContractNameEpoch: flow.Sandboxnet.Chain().ServiceAddress(), - ContractNameClusterQC: flow.Sandboxnet.Chain().ServiceAddress(), - ContractNameDKG: flow.Sandboxnet.Chain().ServiceAddress(), + ContractNameEpoch: flow.Sandboxnet.Chain().ServiceAddress(), + ContractNameClusterQC: flow.Sandboxnet.Chain().ServiceAddress(), + ContractNameDKG: flow.Sandboxnet.Chain().ServiceAddress(), + ContractNameNodeVersionBeacon: flow.Sandboxnet.Chain().ServiceAddress(), } contractAddressesByChainID[flow.Sandboxnet] = sandboxnet // Transient test networks // All system contracts are deployed to the service account transient := map[string]flow.Address{ - ContractNameEpoch: flow.Emulator.Chain().ServiceAddress(), - ContractNameClusterQC: flow.Emulator.Chain().ServiceAddress(), - ContractNameDKG: flow.Emulator.Chain().ServiceAddress(), + ContractNameEpoch: flow.Emulator.Chain().ServiceAddress(), + ContractNameClusterQC: flow.Emulator.Chain().ServiceAddress(), + ContractNameDKG: flow.Emulator.Chain().ServiceAddress(), + ContractNameNodeVersionBeacon: flow.Emulator.Chain().ServiceAddress(), } contractAddressesByChainID[flow.Emulator] = transient contractAddressesByChainID[flow.Localnet] = transient contractAddressesByChainID[flow.BftTestnet] = transient contractAddressesByChainID[flow.Benchnet] = transient - } diff --git a/fvm/systemcontracts/system_contracts_test.go b/fvm/systemcontracts/system_contracts_test.go index 0444e737286..bae3308aac0 100644 --- a/fvm/systemcontracts/system_contracts_test.go +++ b/fvm/systemcontracts/system_contracts_test.go @@ -13,7 +13,14 @@ import ( // TestSystemContract_Address tests that we can retrieve a canonical address // for all accepted chains and contracts. func TestSystemContracts(t *testing.T) { - chains := []flow.ChainID{flow.Mainnet, flow.Testnet, flow.Sandboxnet, flow.Benchnet, flow.Localnet, flow.Emulator} + chains := []flow.ChainID{ + flow.Mainnet, + flow.Testnet, + flow.Sandboxnet, + flow.Benchnet, + flow.Localnet, + flow.Emulator, + } for _, chain := range chains { _, err := SystemContractsForChain(chain) @@ -34,7 +41,14 @@ func TestSystemContract_InvalidChainID(t *testing.T) { // TestServiceEvents tests that we can retrieve service events for all accepted // chains and contracts. func TestServiceEvents(t *testing.T) { - chains := []flow.ChainID{flow.Mainnet, flow.Testnet, flow.Sandboxnet, flow.Benchnet, flow.Localnet, flow.Emulator} + chains := []flow.ChainID{ + flow.Mainnet, + flow.Testnet, + flow.Sandboxnet, + flow.Benchnet, + flow.Localnet, + flow.Emulator, + } for _, chain := range chains { _, err := ServiceEventsForChain(chain) @@ -46,7 +60,14 @@ func TestServiceEvents(t *testing.T) { // TestServiceEventLookup_Consistency sanity checks consistency of the lookup // method, in case an update to ServiceEvents forgets to update the lookup. func TestServiceEventAll_Consistency(t *testing.T) { - chains := []flow.ChainID{flow.Mainnet, flow.Testnet, flow.Sandboxnet, flow.Benchnet, flow.Localnet, flow.Emulator} + chains := []flow.ChainID{ + flow.Mainnet, + flow.Testnet, + flow.Sandboxnet, + flow.Benchnet, + flow.Localnet, + flow.Emulator, + } fields := reflect.TypeOf(ServiceEvents{}).NumField() for _, chain := range chains { @@ -79,11 +100,13 @@ func checkSystemContracts(t *testing.T, chainID flow.ChainID) { assert.NotEqual(t, flow.EmptyAddress, addresses[ContractNameEpoch]) assert.NotEqual(t, flow.EmptyAddress, addresses[ContractNameClusterQC]) assert.NotEqual(t, flow.EmptyAddress, addresses[ContractNameDKG]) + assert.NotEqual(t, flow.EmptyAddress, addresses[ContractNameNodeVersionBeacon]) // entries must match internal mapping assert.Equal(t, addresses[ContractNameEpoch], contracts.Epoch.Address) assert.Equal(t, addresses[ContractNameClusterQC], contracts.ClusterQC.Address) assert.Equal(t, addresses[ContractNameDKG], contracts.DKG.Address) + assert.Equal(t, addresses[ContractNameNodeVersionBeacon], contracts.NodeVersionBeacon.Address) } func checkServiceEvents(t *testing.T, chainID flow.ChainID) { @@ -94,10 +117,13 @@ func checkServiceEvents(t *testing.T, chainID flow.ChainID) { require.True(t, ok, "missing chain %w", chainID.String()) epochContractAddr := addresses[ContractNameEpoch] + versionContractAddr := addresses[ContractNameNodeVersionBeacon] // entries may not be empty assert.NotEqual(t, flow.EmptyAddress, epochContractAddr) + assert.NotEqual(t, flow.EmptyAddress, versionContractAddr) // entries must match internal mapping assert.Equal(t, epochContractAddr, events.EpochSetup.Address) assert.Equal(t, epochContractAddr, events.EpochCommit.Address) + assert.Equal(t, versionContractAddr, events.VersionBeacon.Address) } diff --git a/fvm/transaction.go b/fvm/transaction.go index 5a00ac5223c..c68f3f49528 100644 --- a/fvm/transaction.go +++ b/fvm/transaction.go @@ -13,8 +13,6 @@ func Transaction( return NewTransaction(txn.ID(), txnIndex, txn) } -// TODO(patrick): pass in initial snapshot time when we start supporting -// speculative pre-processing / execution. func NewTransaction( txnId flow.Identifier, txnIndex uint32, @@ -31,22 +29,15 @@ type TransactionProcedure struct { ID flow.Identifier Transaction *flow.TransactionBody TxIndex uint32 - - // TODO(patrick): remove - ProcedureOutput } func (proc *TransactionProcedure) NewExecutor( ctx Context, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) ProcedureExecutor { return newTransactionExecutor(ctx, proc, txnState) } -func (proc *TransactionProcedure) SetOutput(output ProcedureOutput) { - proc.ProcedureOutput = output -} - func (proc *TransactionProcedure) ComputationLimit(ctx Context) uint64 { // TODO for BFT (enforce max computation limit, already checked by collection nodes) // TODO replace tx.Gas with individual limits for computation and memory diff --git a/fvm/transactionInvoker.go b/fvm/transactionInvoker.go index 1a6785cb3c3..2e46664f13f 100644 --- a/fvm/transactionInvoker.go +++ b/fvm/transactionInvoker.go @@ -13,9 +13,10 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/errors" reusableRuntime "github.com/onflow/flow-go/fvm/runtime" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/module/trace" ) @@ -53,15 +54,15 @@ type transactionExecutor struct { ctx Context proc *TransactionProcedure - txnState storage.Transaction + txnState storage.TransactionPreparer span otelTrace.Span env environment.Environment errs *errors.ErrorsCollector - nestedTxnId state.NestedTransactionId - pausedState *state.ExecutionState + startedTransactionBodyExecution bool + nestedTxnId state.NestedTransactionId cadenceRuntime *reusableRuntime.ReusableCadenceRuntime txnBodyExecutor runtime.Executor @@ -72,7 +73,7 @@ type transactionExecutor struct { func newTransactionExecutor( ctx Context, proc *TransactionProcedure, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) *transactionExecutor { span := ctx.StartChildSpan(trace.FVMExecuteTransaction) span.SetAttributes(attribute.String("transaction_id", proc.ID.String())) @@ -91,13 +92,14 @@ func newTransactionExecutor( TransactionVerifier: TransactionVerifier{ VerificationConcurrency: 4, }, - ctx: ctx, - proc: proc, - txnState: txnState, - span: span, - env: env, - errs: errors.NewErrorsCollector(), - cadenceRuntime: env.BorrowCadenceRuntime(), + ctx: ctx, + proc: proc, + txnState: txnState, + span: span, + env: env, + errs: errors.NewErrorsCollector(), + startedTransactionBodyExecution: false, + cadenceRuntime: env.BorrowCadenceRuntime(), } } @@ -131,22 +133,53 @@ func (executor *transactionExecutor) handleError( } func (executor *transactionExecutor) Preprocess() error { + return executor.handleError(executor.preprocess(), "preprocess") +} + +func (executor *transactionExecutor) Execute() error { + return executor.handleError(executor.execute(), "executing") +} + +func (executor *transactionExecutor) preprocess() error { + if executor.AuthorizationChecksEnabled { + err := executor.CheckAuthorization( + executor.ctx.TracerSpan, + executor.proc, + executor.txnState, + executor.AccountKeyWeightThreshold) + if err != nil { + executor.errs.Collect(err) + return executor.errs.ErrorOrNil() + } + } + + if executor.SequenceNumberCheckAndIncrementEnabled { + err := executor.CheckAndIncrementSequenceNumber( + executor.ctx.TracerSpan, + executor.proc, + executor.txnState) + if err != nil { + executor.errs.Collect(err) + return executor.errs.ErrorOrNil() + } + } + if !executor.TransactionBodyExecutionEnabled { return nil } - err := executor.PreprocessTransactionBody() - return executor.handleError(err, "preprocessing") -} + executor.errs.Collect(executor.preprocessTransactionBody()) + if executor.errs.CollectedFailure() { + return executor.errs.ErrorOrNil() + } -func (executor *transactionExecutor) Execute() error { - return executor.handleError(executor.execute(), "executing") + return nil } -// PreprocessTransactionBody preprocess parts of a transaction body that are +// preprocessTransactionBody preprocess parts of a transaction body that are // infrequently modified and are expensive to compute. For now this includes // reading meter parameter overrides and parsing programs. -func (executor *transactionExecutor) PreprocessTransactionBody() error { +func (executor *transactionExecutor) preprocessTransactionBody() error { meterParams, err := getBodyMeterParameters( executor.ctx, executor.proc, @@ -160,6 +193,7 @@ func (executor *transactionExecutor) PreprocessTransactionBody() error { if err != nil { return err } + executor.startedTransactionBodyExecution = true executor.nestedTxnId = txnId executor.txnBodyExecutor = executor.cadenceRuntime.NewTransactionExecutor( @@ -173,93 +207,23 @@ func (executor *transactionExecutor) PreprocessTransactionBody() error { // by the transaction body. err = executor.txnBodyExecutor.Preprocess() if err != nil { - executor.errs.Collect( - fmt.Errorf( - "transaction preprocess failed: %w", - err)) - - // We shouldn't early exit on non-failure since we need to deduct fees. - if executor.errs.CollectedFailure() { - return executor.errs.ErrorOrNil() - } - - // NOTE: We need to restart the nested transaction in order to pause - // for fees deduction. - err = executor.txnState.RestartNestedTransaction(txnId) - if err != nil { - return err - } - } - - // Pause the transaction body's nested transaction in order to interleave - // auth and seq num checks. - pausedState, err := executor.txnState.PauseNestedTransaction(txnId) - if err != nil { - return err + return fmt.Errorf( + "transaction preprocess failed: %w", + err) } - executor.pausedState = pausedState return nil } func (executor *transactionExecutor) execute() error { - if executor.AuthorizationChecksEnabled { - err := executor.CheckAuthorization( - executor.ctx.TracerSpan, - executor.proc, - executor.txnState, - executor.AccountKeyWeightThreshold) - if err != nil { - executor.errs.Collect(err) - executor.errs.Collect(executor.abortPreprocessed()) - return executor.errs.ErrorOrNil() - } + if !executor.startedTransactionBodyExecution { + return executor.errs.ErrorOrNil() } - if executor.SequenceNumberCheckAndIncrementEnabled { - err := executor.CheckAndIncrementSequenceNumber( - executor.ctx.TracerSpan, - executor.proc, - executor.txnState) - if err != nil { - executor.errs.Collect(err) - executor.errs.Collect(executor.abortPreprocessed()) - return executor.errs.ErrorOrNil() - } - } - - if executor.TransactionBodyExecutionEnabled { - err := executor.ExecuteTransactionBody() - if err != nil { - return err - } - } - - return nil -} - -func (executor *transactionExecutor) abortPreprocessed() error { - if !executor.TransactionBodyExecutionEnabled { - return nil - } - - executor.txnState.ResumeNestedTransaction(executor.pausedState) - - // There shouldn't be any update, but drop all updates just in case. - err := executor.txnState.RestartNestedTransaction(executor.nestedTxnId) - if err != nil { - return err - } - - // We need to commit the aborted state unconditionally to include - // the touched registers in the execution receipt. - _, err = executor.txnState.CommitNestedTransaction(executor.nestedTxnId) - return err + return executor.ExecuteTransactionBody() } func (executor *transactionExecutor) ExecuteTransactionBody() error { - executor.txnState.ResumeNestedTransaction(executor.pausedState) - var invalidator derived.TransactionInvalidator if !executor.errs.CollectedError() { @@ -385,7 +349,7 @@ func (executor *transactionExecutor) normalExecution() ( return } - var bodySnapshot *state.ExecutionSnapshot + var bodySnapshot *snapshot.ExecutionSnapshot bodySnapshot, err = executor.txnState.CommitNestedTransaction(bodyTxnId) if err != nil { return diff --git a/fvm/transactionPayerBalanceChecker.go b/fvm/transactionPayerBalanceChecker.go index 038953dc150..96618582863 100644 --- a/fvm/transactionPayerBalanceChecker.go +++ b/fvm/transactionPayerBalanceChecker.go @@ -14,7 +14,7 @@ type TransactionPayerBalanceChecker struct{} func (_ TransactionPayerBalanceChecker) CheckPayerBalanceAndReturnMaxFees( proc *TransactionProcedure, - txnState storage.Transaction, + txnState storage.TransactionPreparer, env environment.Environment, ) (uint64, error) { if !env.TransactionFeesEnabled() { diff --git a/fvm/transactionSequenceNum.go b/fvm/transactionSequenceNum.go index 2f9f8916d22..81b77e4868f 100644 --- a/fvm/transactionSequenceNum.go +++ b/fvm/transactionSequenceNum.go @@ -16,7 +16,7 @@ type TransactionSequenceNumberChecker struct{} func (c TransactionSequenceNumberChecker) CheckAndIncrementSequenceNumber( tracer tracing.TracerSpan, proc *TransactionProcedure, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) error { // TODO(Janez): verification is part of inclusion fees, not execution fees. var err error @@ -34,7 +34,7 @@ func (c TransactionSequenceNumberChecker) CheckAndIncrementSequenceNumber( func (c TransactionSequenceNumberChecker) checkAndIncrementSequenceNumber( tracer tracing.TracerSpan, proc *TransactionProcedure, - txnState storage.Transaction, + txnState storage.TransactionPreparer, ) error { defer tracer.StartChildSpan(trace.FVMSeqNumCheckTransaction).End() diff --git a/fvm/transactionStorageLimiter.go b/fvm/transactionStorageLimiter.go index 9ce382978a4..9d504adf7bf 100644 --- a/fvm/transactionStorageLimiter.go +++ b/fvm/transactionStorageLimiter.go @@ -10,7 +10,7 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" ) @@ -34,7 +34,7 @@ type TransactionStorageLimiter struct{} // the fee deduction step happens after the storage limit check. func (limiter TransactionStorageLimiter) CheckStorageLimits( env environment.Environment, - snapshot *state.ExecutionSnapshot, + snapshot *snapshot.ExecutionSnapshot, payer flow.Address, maxTxFees uint64, ) error { @@ -55,7 +55,7 @@ func (limiter TransactionStorageLimiter) CheckStorageLimits( // storage limit is exceeded. The returned list include addresses of updated // registers (and the payer's address). func (limiter TransactionStorageLimiter) getStorageCheckAddresses( - snapshot *state.ExecutionSnapshot, + snapshot *snapshot.ExecutionSnapshot, payer flow.Address, maxTxFees uint64, ) []flow.Address { @@ -100,7 +100,7 @@ func (limiter TransactionStorageLimiter) getStorageCheckAddresses( // address and exceeded the storage limit. func (limiter TransactionStorageLimiter) checkStorageLimits( env environment.Environment, - snapshot *state.ExecutionSnapshot, + snapshot *snapshot.ExecutionSnapshot, payer flow.Address, maxTxFees uint64, ) error { diff --git a/fvm/transactionStorageLimiter_test.go b/fvm/transactionStorageLimiter_test.go index 1a9fcc153ff..b9b2a87ec3a 100644 --- a/fvm/transactionStorageLimiter_test.go +++ b/fvm/transactionStorageLimiter_test.go @@ -10,14 +10,14 @@ import ( "github.com/onflow/flow-go/fvm" fvmmock "github.com/onflow/flow-go/fvm/environment/mock" "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" ) func TestTransactionStorageLimiter(t *testing.T) { owner := flow.HexToAddress("1") - snapshot := &state.ExecutionSnapshot{ + executionSnapshot := &snapshot.ExecutionSnapshot{ WriteSet: map[flow.RegisterID]flow.RegisterValue{ flow.NewRegisterID(string(owner[:]), "a"): flow.RegisterValue("foo"), flow.NewRegisterID(string(owner[:]), "b"): flow.RegisterValue("bar"), @@ -40,7 +40,7 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, snapshot, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(env, executionSnapshot, flow.EmptyAddress, 0) require.NoError(t, err, "Transaction with higher capacity than storage used should work") }) t.Run("capacity = storage -> OK", func(t *testing.T) { @@ -59,7 +59,7 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, snapshot, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(env, executionSnapshot, flow.EmptyAddress, 0) require.NoError(t, err, "Transaction with equal capacity than storage used should work") }) t.Run("capacity = storage -> OK (dedup payer)", func(t *testing.T) { @@ -78,7 +78,7 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, snapshot, owner, 0) + err := d.CheckStorageLimits(env, executionSnapshot, owner, 0) require.NoError(t, err, "Transaction with equal capacity than storage used should work") }) t.Run("capacity < storage -> Not OK", func(t *testing.T) { @@ -97,7 +97,7 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, snapshot, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(env, executionSnapshot, flow.EmptyAddress, 0) require.Error(t, err, "Transaction with lower capacity than storage used should fail") }) t.Run("capacity > storage -> OK (payer not updated)", func(t *testing.T) { @@ -115,10 +115,10 @@ func TestTransactionStorageLimiter(t *testing.T) { nil, ) - snapshot = &state.ExecutionSnapshot{} + executionSnapshot = &snapshot.ExecutionSnapshot{} d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, snapshot, owner, 1) + err := d.CheckStorageLimits(env, executionSnapshot, owner, 1) require.NoError(t, err, "Transaction with higher capacity than storage used should work") }) t.Run("capacity < storage -> Not OK (payer not updated)", func(t *testing.T) { @@ -136,10 +136,10 @@ func TestTransactionStorageLimiter(t *testing.T) { nil, ) - snapshot = &state.ExecutionSnapshot{} + executionSnapshot = &snapshot.ExecutionSnapshot{} d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, snapshot, owner, 1000) + err := d.CheckStorageLimits(env, executionSnapshot, owner, 1000) require.Error(t, err, "Transaction with lower capacity than storage used should fail") }) t.Run("if ctx LimitAccountStorage false-> OK", func(t *testing.T) { @@ -159,7 +159,7 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, snapshot, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(env, executionSnapshot, flow.EmptyAddress, 0) require.NoError(t, err, "Transaction with higher capacity than storage used should work") }) t.Run("non existing accounts or any other errors on fetching storage used -> Not OK", func(t *testing.T) { @@ -178,7 +178,7 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, snapshot, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(env, executionSnapshot, flow.EmptyAddress, 0) require.Error(t, err, "check storage used on non existing account (not general registers) should fail") }) } diff --git a/fvm/transactionVerifier.go b/fvm/transactionVerifier.go index a0c20f33c70..67c3b76db5f 100644 --- a/fvm/transactionVerifier.go +++ b/fvm/transactionVerifier.go @@ -168,7 +168,7 @@ type TransactionVerifier struct { func (v *TransactionVerifier) CheckAuthorization( tracer tracing.TracerSpan, proc *TransactionProcedure, - txnState storage.Transaction, + txnState storage.TransactionPreparer, keyWeightThreshold int, ) error { // TODO(Janez): verification is part of inclusion fees, not execution fees. @@ -188,7 +188,7 @@ func (v *TransactionVerifier) CheckAuthorization( func (v *TransactionVerifier) verifyTransaction( tracer tracing.TracerSpan, proc *TransactionProcedure, - txnState storage.Transaction, + txnState storage.TransactionPreparer, keyWeightThreshold int, ) error { span := tracer.StartChildSpan(trace.FVMVerifyTransaction) @@ -259,7 +259,7 @@ func (v *TransactionVerifier) verifyTransaction( // getAccountKeys gets the signatures' account keys and populate the account // keys into the signature continuation structs. func (v *TransactionVerifier) getAccountKeys( - txnState storage.Transaction, + txnState storage.TransactionPreparer, accounts environment.Accounts, signatures []*signatureContinuation, proposalKey flow.ProposalKey, diff --git a/fvm/transactionVerifier_test.go b/fvm/transactionVerifier_test.go index c69af4f32db..3fb0e5d9aa8 100644 --- a/fvm/transactionVerifier_test.go +++ b/fvm/transactionVerifier_test.go @@ -39,7 +39,7 @@ func TestTransactionVerification(t *testing.T) { run := func( body *flow.TransactionBody, ctx fvm.Context, - txn storage.Transaction, + txn storage.TransactionPreparer, ) error { executor := fvm.Transaction(body, 0).NewExecutor(ctx, txn) err := fvm.Run(executor) diff --git a/go.mod b/go.mod index 3ae4e603234..d5890cfa995 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go -go 1.19 +go 1.20 require ( cloud.google.com/go/compute/metadata v0.2.3 @@ -15,13 +15,13 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 github.com/ef-ds/deque v1.0.4 github.com/ethereum/go-ethereum v1.9.13 - github.com/fxamacker/cbor/v2 v2.4.1-0.20220515183430-ad2eae63303f + github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c github.com/gammazero/workerpool v1.1.2 github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.3 github.com/google/go-cmp v0.5.9 - github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 + github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2 @@ -31,35 +31,34 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru v0.5.4 github.com/improbable-eng/grpc-web v0.15.0 - github.com/ipfs/go-block-format v0.0.3 + github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-blockservice v0.4.0 - github.com/ipfs/go-cid v0.3.2 + github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger2 v0.1.3 - github.com/ipfs/go-ipfs-blockstore v1.2.0 + github.com/ipfs/go-ipfs-blockstore v1.3.0 github.com/ipfs/go-ipfs-provider v0.7.0 - github.com/ipfs/go-ipld-format v0.3.0 + github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 github.com/libp2p/go-addr-util v0.1.0 - github.com/libp2p/go-libp2p v0.24.2 - github.com/libp2p/go-libp2p-kad-dht v0.19.0 - github.com/libp2p/go-libp2p-kbucket v0.5.0 - github.com/libp2p/go-libp2p-pubsub v0.8.2-0.20221201175637-3d2eab35722e - github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c + github.com/libp2p/go-libp2p v0.28.1 + github.com/libp2p/go-libp2p-kad-dht v0.24.2 + github.com/libp2p/go-libp2p-kbucket v0.6.3 + github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/montanaflynn/stats v0.6.6 - github.com/multiformats/go-multiaddr v0.8.0 + github.com/multiformats/go-multiaddr v0.9.0 github.com/multiformats/go-multiaddr-dns v0.3.1 - github.com/multiformats/go-multihash v0.2.1 - github.com/onflow/atree v0.5.0 - github.com/onflow/cadence v0.38.1 + github.com/multiformats/go-multihash v0.2.3 + github.com/onflow/atree v0.6.0 + github.com/onflow/cadence v0.39.14 github.com/onflow/flow v0.3.4 - github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 - github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 - github.com/onflow/flow-go-sdk v0.40.0 - github.com/onflow/flow-go/crypto v0.24.7 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230330183547-d0dd18f6f20d - github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 + github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d + github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 + github.com/onflow/flow-go-sdk v0.41.9 + github.com/onflow/flow-go/crypto v0.24.9 + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce + github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible github.com/pkg/errors v0.9.1 @@ -67,31 +66,31 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/rs/cors v1.8.0 github.com/rs/zerolog v1.29.0 - github.com/schollz/progressbar/v3 v3.8.3 + github.com/schollz/progressbar/v3 v3.13.1 github.com/sethvargo/go-retry v0.2.3 github.com/shirou/gopsutil/v3 v3.22.2 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.12.0 - github.com/stretchr/testify v1.8.2 + github.com/spf13/viper v1.15.0 + github.com/stretchr/testify v1.8.4 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.11 - go.opentelemetry.io/otel v1.8.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.8.0 - go.opentelemetry.io/otel/sdk v1.8.0 - go.opentelemetry.io/otel/trace v1.8.0 - go.uber.org/atomic v1.10.0 - go.uber.org/multierr v1.9.0 - golang.org/x/crypto v0.4.0 - golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 - golang.org/x/sync v0.1.0 - golang.org/x/sys v0.6.0 - golang.org/x/text v0.8.0 - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac - golang.org/x/tools v0.6.0 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 + go.opentelemetry.io/otel/sdk v1.16.0 + go.opentelemetry.io/otel/trace v1.16.0 + go.uber.org/atomic v1.11.0 + go.uber.org/multierr v1.11.0 + golang.org/x/crypto v0.10.0 + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 + golang.org/x/sync v0.2.0 + golang.org/x/sys v0.9.0 + golang.org/x/text v0.10.0 + golang.org/x/time v0.1.0 + golang.org/x/tools v0.9.1 google.golang.org/api v0.114.0 google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 - google.golang.org/grpc v1.53.0 + google.golang.org/grpc v1.55.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 google.golang.org/protobuf v1.30.0 gotest.tools v2.2.0+incompatible @@ -99,8 +98,13 @@ require ( ) require ( + github.com/coreos/go-semver v0.3.0 + github.com/go-playground/validator/v10 v10.14.1 + github.com/hashicorp/golang-lru/v2 v2.0.2 + github.com/mitchellh/mapstructure v1.5.0 + github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d github.com/slok/go-http-metrics v0.10.0 - gonum.org/v1/gonum v0.8.2 + github.com/sony/gobreaker v0.5.0 ) require ( @@ -120,18 +124,18 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect github.com/aws/smithy-go v1.13.5 // indirect - github.com/benbjohnson/clock v1.3.0 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.3.0 // indirect + github.com/bits-and-blooms/bitset v1.5.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/containerd/cgroups v1.0.4 // indirect + github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect @@ -141,20 +145,23 @@ require ( github.com/felixge/fgprof v0.9.3 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/fxamacker/circlehash v0.3.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gammazero/deque v0.1.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect - github.com/go-test/deep v1.0.8 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/go-test/deep v1.1.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/golang/glog v1.0.0 // indirect + github.com/golang/glog v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/gopacket v1.1.19 // indirect @@ -163,117 +170,117 @@ require ( github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huin/goupnp v1.0.3 // indirect + github.com/huin/goupnp v1.2.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/boxo v0.10.0 // indirect github.com/ipfs/go-bitswap v0.9.0 // indirect github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-fetcher v1.5.0 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect - github.com/ipfs/go-ipfs-pq v0.0.2 // indirect + github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect - github.com/ipfs/go-ipns v0.2.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect - github.com/ipfs/go-peertaskqueue v0.7.0 // indirect + github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-verifcid v0.0.1 // indirect - github.com/ipld/go-ipld-prime v0.14.1 // indirect + github.com/ipld/go-ipld-prime v0.20.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/go-bindata v3.23.0+incompatible // indirect - github.com/klauspost/compress v1.15.13 // indirect - github.com/klauspost/cpuid/v2 v2.2.2 // indirect - github.com/koron/go-ssdp v0.0.3 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/koron/go-ssdp v0.0.4 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect - github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect + github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect github.com/libp2p/go-libp2p-core v0.20.1 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-msgio v0.2.0 // indirect - github.com/libp2p/go-nat v0.1.0 // indirect + github.com/libp2p/go-msgio v0.3.0 // indirect + github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect - github.com/libp2p/go-openssl v0.1.0 // indirect - github.com/libp2p/go-reuseport v0.2.0 // indirect + github.com/libp2p/go-reuseport v0.3.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.0 // indirect - github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/lucas-clemente/quic-go v0.31.1 // indirect + github.com/logrusorgru/aurora/v4 v4.0.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/magiconair/properties v1.8.6 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect - github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect - github.com/mattn/go-pointer v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/miekg/dns v1.1.50 // indirect + github.com/miekg/dns v1.1.54 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect - github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multicodec v0.7.0 // indirect - github.com/multiformats/go-multistream v0.3.3 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multicodec v0.9.0 // indirect + github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onflow/flow-ft/lib/go/contracts v0.5.0 // indirect + github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect + github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect github.com/onflow/sdks v0.5.0 // indirect - github.com/onsi/ginkgo/v2 v2.6.1 // indirect + github.com/onsi/ginkgo/v2 v2.9.7 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect + github.com/polydawn/refmt v0.89.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/psiemens/sconfig v0.1.0 // indirect + github.com/quic-go/qpack v0.4.0 // indirect + github.com/quic-go/qtls-go1-19 v0.3.2 // indirect + github.com/quic-go/qtls-go1-20 v0.2.2 // indirect + github.com/quic-go/quic-go v0.33.0 // indirect + github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect - github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a // indirect - github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.9.0 // indirect + github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/subosito/gotenv v1.4.0 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d // indirect github.com/vmihailenco/tagparser v0.1.1 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect - github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0 // indirect - go.opentelemetry.io/proto/otlp v0.18.0 // indirect - go.uber.org/dig v1.15.0 // indirect - go.uber.org/fx v1.18.2 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.uber.org/dig v1.17.0 // indirect + go.uber.org/fx v1.19.2 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.6.0 // indirect - golang.org/x/term v0.6.0 // indirect + golang.org/x/term v0.9.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect - gopkg.in/ini.v1 v1.66.6 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/blake3 v1.1.7 // indirect - nhooyr.io/websocket v1.8.6 // indirect + lukechampine.com/blake3 v1.2.1 // indirect + nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index 9b4664eed9c..73eadfddbf8 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,7 @@ cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1 cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/kms v1.0.0/go.mod h1:nhUehi+w7zht2XrUfvTRNpxrfayBHqP4lu2NSywui/0= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/profiler v0.3.0 h1:R6y/xAeifaUXxd2x6w+jIwKxoKl8Cv5HJvcvASTPWJo= cloud.google.com/go/profiler v0.3.0/go.mod h1:9wYk9eY4iZHsev8TQb61kh3wiOiSyz/xOYixWPzweCU= @@ -169,15 +170,16 @@ github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAm github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.3.0 h1:h7mv5q31cthBTd7V4kLAZaIThj1e8vPGcSqpPue9KVI= -github.com/bits-and-blooms/bitset v1.3.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= +github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= @@ -200,12 +202,14 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bytecodealliance/wasmtime-go v0.22.0/go.mod h1:q320gUxqyI8yB+ZqRuaJOEnGkAnHh6WtJjMaT2CW4wI= +github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -234,12 +238,13 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -255,7 +260,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -266,9 +270,9 @@ github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQY github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= @@ -319,6 +323,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= github.com/ethereum/go-ethereum v1.9.13 h1:rOPqjSngvs1VSYH2H+PMPiWt4VEulvNRbFgqiGqJM3E= github.com/ethereum/go-ethereum v1.9.13/go.mod h1:qwN9d1GLyDh0N7Ab8bMGd0H9knaji2jOBm2RrMGjXls= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -336,16 +341,19 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/fxamacker/cbor/v2 v2.4.1-0.20220515183430-ad2eae63303f h1:dxTR4AaxCwuQv9LAVTAC2r1szlS+epeuPT5ClLKT6ZY= -github.com/fxamacker/cbor/v2 v2.4.1-0.20220515183430-ad2eae63303f/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fxamacker/cbor/v2 v2.2.1-0.20210927235116-3d6d5d1de29b/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c h1:5tm/Wbs9d9r+qZaUFXk59CWDD0+77PBqDREffYkyi5c= +github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/circlehash v0.1.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM= github.com/fxamacker/circlehash v0.3.0 h1:XKdvTtIJV9t7DDUtsf0RIpC1OcxZtPbmgIH7ekx28WA= github.com/fxamacker/circlehash v0.3.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gammazero/deque v0.1.0 h1:f9LnNmq66VDeuAlSAapemq/U7hJ2jpIWa4c09q8Dlik= github.com/gammazero/deque v0.1.0/go.mod h1:KQw7vFau1hHuM8xmI9RbgKFbAsQFWmBpqQ2KenFLk6M= github.com/gammazero/workerpool v1.1.2 h1:vuioDQbgrz4HoaCi2q1HLlOXdpbap5AET7xu5/qj87g= @@ -377,29 +385,35 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= +github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= @@ -420,8 +434,9 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -459,8 +474,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -513,8 +529,8 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20220412212628-83db2b799d1f/go.mod h1:Pt31oes+eGImORns3McJn8zHefuQl2rG8l6xQjGYB4U= -github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 h1:wORs2YN3R3ona/CXYuTvLM31QlgoNKHvlCNuArCDDCU= -github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -537,8 +553,8 @@ github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6c github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -593,6 +609,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= +github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -603,8 +621,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= -github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= +github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= +github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -619,6 +637,8 @@ github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= +github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUiJ2teMvI= github.com/ipfs/go-bitswap v0.5.0/go.mod h1:WwyyYD33RHCpczgHjpx+xjWYIy8l41K+l5EMy4/ctSM= @@ -626,8 +646,9 @@ github.com/ipfs/go-bitswap v0.9.0 h1:/dZi/XhUN/aIk78pI4kaZrilUglJ+7/SCmOHWIpiy8E github.com/ipfs/go-bitswap v0.9.0/go.mod h1:zkfBcGWp4dQTQd0D0akpudhpOVUAJT9GbH9tDmR8/s4= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= +github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= +github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-blockservice v0.2.0/go.mod h1:Vzvj2fAnbbyly4+T7D5+p9n3+ZKVHA2bRMMo1QoILtQ= github.com/ipfs/go-blockservice v0.4.0 h1:7MUijAW5SqdsqEW/EhnNFRJXVF8mGU5aGhZ3CQaCWbY= @@ -640,8 +661,8 @@ github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67Fexh github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= -github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= @@ -672,8 +693,8 @@ github.com/ipfs/go-fetcher v1.5.0/go.mod h1:5pDZ0393oRF/fHiLmtFZtpMNBQfHOYNPtryW github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blockstore v0.2.0/go.mod h1:SNeEpz/ICnMYZQYr7KNZTjdn7tEPB/99xpe8xI1RW7o= -github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= -github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= +github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12XwJyvYvMM= +github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -691,8 +712,9 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAz github.com/ipfs/go-ipfs-exchange-offline v0.1.0/go.mod h1:YdJXa+yPF1na+gfYHYejtLwHFpuKv22eatApNiSfanM= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= -github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= +github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= +github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= github.com/ipfs/go-ipfs-provider v0.7.0 h1:5GpHv46eIS8h2mbbKg1ckU5paajDYJtE4GA/SBepOQg= github.com/ipfs/go-ipfs-provider v0.7.0/go.mod h1:mgjsWgDt9j19N1REPxRa31p+eRIQmjNt5McNdQQ5CsA= github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= @@ -701,10 +723,8 @@ github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FF github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-format v0.3.0 h1:Mwm2oRLzIuUwEPewWAWyMuuBQUsn3awfFEYVb8akMOQ= -github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipns v0.2.0 h1:BgmNtQhqOw5XEZ8RAfWEpK4DhqaYiuP6h71MhIp7xXU= -github.com/ipfs/go-ipns v0.2.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= +github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds= +github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= @@ -724,13 +744,14 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= -github.com/ipfs/go-peertaskqueue v0.7.0 h1:VyO6G4sbzX80K58N60cCaHsSsypbUNs1GjO5seGNsQ0= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= +github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= +github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.14.1 h1:n9obcUnuqPK34HlfbiB+o9GhXE/x59uue4z9YTsaoj4= -github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= +github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -773,6 +794,7 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/go-bindata v3.23.0+incompatible h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8= github.com/kevinburke/go-bindata v3.23.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -783,37 +805,36 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= -github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= -github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= -github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= -github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= +github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= +github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= github.com/libp2p/go-addr-util v0.1.0 h1:acKsntI33w2bTU7tC9a0SaPimJGfSI0bFKC18ChxeVI= @@ -840,10 +861,10 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= -github.com/libp2p/go-libp2p v0.24.2 h1:iMViPIcLY0D6zr/f+1Yq9EavCZu2i7eDstsr1nEwSAk= -github.com/libp2p/go-libp2p v0.24.2/go.mod h1:WuxtL2V8yGjam03D93ZBC19tvOUiPpewYv1xdFGWu1k= -github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= -github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= +github.com/libp2p/go-libp2p v0.28.1 h1:YurK+ZAI6cKfASLJBVFkpVBdl3wGhFi6fusOt725ii8= +github.com/libp2p/go-libp2p v0.28.1/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= +github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= +github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI= @@ -887,10 +908,10 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-kad-dht v0.19.0 h1:2HuiInHZTm9ZvQajaqdaPLHr0PCKKigWiflakimttE0= -github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= -github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= -github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= +github.com/libp2p/go-libp2p-kad-dht v0.24.2 h1:zd7myKBKCmtZBhI3I0zm8xBkb28v3gmSEtQfBdAdFwc= +github.com/libp2p/go-libp2p-kad-dht v0.24.2/go.mod h1:BShPzRbK6+fN3hk8a0WGAYKpb8m4k+DtchkqouGTrSg= +github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= +github.com/libp2p/go-libp2p-kbucket v0.6.3/go.mod h1:RCseT7AH6eJWxxk2ol03xtP9pEHetYSPXOaJnOiD8i0= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= @@ -915,8 +936,8 @@ github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRj github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= -github.com/libp2p/go-libp2p-pubsub v0.8.2-0.20221201175637-3d2eab35722e h1:phmi6mEoO5y2AQP68+4vZhNpHtZ4dum2ieFtWdmjXak= -github.com/libp2p/go-libp2p-pubsub v0.8.2-0.20221201175637-3d2eab35722e/go.mod h1:e4kT+DYjzPUYGZeWk4I+oxCSYTXizzXii5LDRRhjKSw= +github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= +github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc= github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= @@ -969,13 +990,13 @@ github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= -github.com/libp2p/go-msgio v0.2.0 h1:W6shmB+FeynDrUVl2dgFQvzfBZcXiyqY4VmpQLu9FqU= -github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= +github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= +github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= -github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= -github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= +github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= +github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= @@ -987,12 +1008,10 @@ github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= -github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= -github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= -github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= +github.com/libp2p/go-reuseport v0.3.0 h1:iiZslO5byUYZEg9iCwJGf5h+sf1Agmqx2V2FDjPyvUw= +github.com/libp2p/go-reuseport v0.3.0/go.mod h1:laea40AimhtfEqysZ71UpYj4S+R9VpH8PgqLo7L+SwI= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= @@ -1025,37 +1044,30 @@ github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rB github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= -github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= +github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= -github.com/lucas-clemente/quic-go v0.31.1 h1:O8Od7hfioqq0PMYHDyBkxU2aA7iZ2W9pjbrWuja2YR4= -github.com/lucas-clemente/quic-go v0.31.1/go.mod h1:0wFbizLgYzqHqtlyxyCaJKlE7bYgE6JQ+54TLd/Dq2g= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c h1:OqVcb1Dkheracn4fgCjxlfhuSnM8jmPbrWkJbRIC4fo= -github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c/go.mod h1:5/Yq7mnb+VdE44ff+FL8LSOPEquOVqm/7Hz40U4VUZo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= -github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= -github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= -github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/marten-seemann/webtransport-go v0.4.3 h1:vkt5o/Ci+luknRteWdYGYH1KcB7ziup+J+1PzZJIvmg= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -1067,19 +1079,23 @@ github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXT github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= -github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1090,8 +1106,8 @@ github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= +github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -1104,8 +1120,9 @@ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= @@ -1150,8 +1167,8 @@ github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= -github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= -github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= +github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -1170,11 +1187,10 @@ github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysj github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= -github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= -github.com/multiformats/go-multicodec v0.7.0 h1:rTUjGOwjlhGHbEMbPoSUJowG1spZTVsITRANCjKTUAQ= -github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= +github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1183,16 +1199,15 @@ github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.0.16/go.mod h1:zhfEIgVnB/rPMfxgFw15ZmGoNaKyNUIE4IWHG/kC+Ag= -github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= github.com/multiformats/go-multistream v0.2.0/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= github.com/multiformats/go-multistream v0.2.2/go.mod h1:UIcnm7Zuo8HKG+HkWgfQsGL+/MIEhyTqbODbIUwSXKs= -github.com/multiformats/go-multistream v0.3.3 h1:d5PZpjwRgVlbwfdTDjife7XszfZd8KYWfROYFlGcR8o= -github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= +github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= +github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -1221,43 +1236,52 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onflow/atree v0.5.0 h1:y3lh8hY2fUo8KVE2ALVcz0EiNTq0tXJ6YTXKYVDA+3E= -github.com/onflow/atree v0.5.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= -github.com/onflow/cadence v0.38.1 h1:8YpnE1ixAGB8hF3t+slkHGhjfIBJ95dqUS+sEHrM2kY= -github.com/onflow/cadence v0.38.1/go.mod h1:SpfjNhPsJxGIHbOthE9JD/e8JFaFY73joYLPsov+PY4= +github.com/onflow/atree v0.1.0-beta1.0.20211027184039-559ee654ece9/go.mod h1:+6x071HgCF/0v5hQcaE5qqjc2UqN5gCU8h5Mk6uqpOg= +github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= +github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= +github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= +github.com/onflow/cadence v0.39.14 h1:YoR3YFUga49rqzVY1xwI6I2ZDBmvwGh13jENncsleC8= +github.com/onflow/cadence v0.39.14/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 h1:9QEI+C9k/Cx/TRC3SCAHmNQqV7UlLG0DHQewTl8Lg6w= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1/go.mod h1:xiSs5IkirazpG89H5jH8xVUKNPlCZqUhLH4+vikQVS4= -github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 h1:dhXSFiBkS6Q3XmBctJAfwR4XPkgBT7VNx08F/zTBgkM= -github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1/go.mod h1:cBimYbTvHK77lclJ1JyhvmKAB9KDzCeWm7OW1EeQSr0= -github.com/onflow/flow-ft/lib/go/contracts v0.5.0 h1:Cg4gHGVblxcejfNNG5Mfj98Wf4zbY76O0Y28QB0766A= -github.com/onflow/flow-ft/lib/go/contracts v0.5.0/go.mod h1:1zoTjp1KzNnOPkyqKmWKerUyf0gciw+e6tAEt0Ks3JE= -github.com/onflow/flow-go-sdk v0.40.0 h1:s8uwoyTquN8tjdXpqGmNkXTjf79yUII8JExc5QEl4Xw= -github.com/onflow/flow-go-sdk v0.40.0/go.mod h1:34dxXk9Hp/bQw6Zy6+H44Xo0kQU+aJyQoqdDxq00rJM= -github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= -github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230330183547-d0dd18f6f20d h1:Wl8bE1YeZEcRNnCpxw2rikOEaivuYKDrnJd2vsfIWoA= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230330183547-d0dd18f6f20d/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= -github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8/go.mod h1:73C8FlT4L/Qe4Cf5iXUNL8b2pvu4zs5dJMMJ5V2TjUI= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d h1:B7PdhdUNkve5MVrekWDuQf84XsGBxNZ/D3x+QQ8XeVs= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d/go.mod h1:xAiV/7TKhw863r6iO3CS5RnQ4F+pBY1TxD272BsILlo= +github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= +github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= +github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= +github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= +github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= +github.com/onflow/flow-go-sdk v0.41.9 h1:cyplhhhc0RnfOAan2t7I/7C9g1hVGDDLUhWj6ZHAkk4= +github.com/onflow/flow-go-sdk v0.41.9/go.mod h1:e9Q5TITCy7g08lkdQJxP8fAKBnBoC5FjALvUKr36j4I= +github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= +github.com/onflow/flow-go/crypto v0.24.9 h1:0EQp+kSZYJepMIiSypfJVe7tzsPcb6UXOdOtsTCDhBs= +github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= +github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= +github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= github.com/onflow/sdks v0.5.0/go.mod h1:F0dj0EyHC55kknLkeD10js4mo14yTdMotnWMslPirrU= +github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d h1:gAEqYPn3DS83rHIKEpsajnppVD1+zwuYPFyeDVFaQvg= +github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d/go.mod h1:iMC8gkLqu4nkbkAla5HkSBb+FGyQOZiWz3DYm2wSXCk= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.6.1 h1:1xQPCjcqYw/J5LchOcp4/2q/jzJFjiAOc25chhnDw+Q= -github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= +github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1279,10 +1303,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.2 h1:+jQXlF3scKIcSEKkdHzXhCTDLPFi5r1wnK6yPS+49Gw= -github.com/pelletier/go-toml/v2 v2.0.2/go.mod h1:MovirKjgVRESsAvNZlAjtFwV867yGuwRkXbG66OzopI= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= @@ -1297,10 +1319,12 @@ github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6J github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= +github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= @@ -1320,8 +1344,8 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1331,8 +1355,8 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= -github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1348,18 +1372,28 @@ github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/psiemens/sconfig v0.1.0 h1:xfWqW+TRpih7mXZIqKYTmpRhlZLQ1kbxV8EjllPv76s= github.com/psiemens/sconfig v0.1.0/go.mod h1:+MLKqdledP/8G3rOBpknbLh0IclCf4WneJUtS26JB2U= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U= +github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= +github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E= +github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= +github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0= +github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA= +github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= +github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a h1:s7GrsqeorVkFR1vGmQ6WVL9nup0eyQCC+YVUeSQLH/Q= -github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= @@ -1375,8 +1409,9 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8= github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko= +github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= +github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sethvargo/go-retry v0.2.3 h1:oYlgvIvsju3jNbottWABtbnoLC+GDtLdBHxKWxQm/iU= @@ -1413,25 +1448,28 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/slok/go-http-metrics v0.10.0 h1:rh0LaYEKza5eaYRGDXujKrOln57nHBi4TtVhmNEpbgM= github.com/slok/go-http-metrics v0.10.0/go.mod h1:lFqdaS4kWMfUKCSukjC47PdCeTk+hXDUVm8kLHRqJ38= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= +github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= -github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.9.0 h1:sFSLUHgxdnN32Qy38hK3QkYBFXZj9DKjVjCUCtD7juY= -github.com/spf13/afero v1.9.0/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= @@ -1448,8 +1486,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= -github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= @@ -1469,13 +1507,14 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= -github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/supranational/blst v0.3.4/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/supranational/blst v0.3.10 h1:CMciDZ/h4pXDDXQASe8ZGTNKUiVNxVVA5hpci2Uuhuk= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= @@ -1500,6 +1539,7 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -1508,10 +1548,10 @@ github.com/vmihailenco/msgpack/v4 v4.3.11 h1:Q47CePddpNGNhk4GCnAx9DDtASi2rasatE0 github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA= -github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= -github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= +github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= @@ -1521,8 +1561,6 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= -github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= -github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -1538,8 +1576,10 @@ github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPR github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.0/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1558,42 +1598,45 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.8.0 h1:zcvBFizPbpa1q7FehvFiHbQwGzmPILebO0tyqIR5Djg= -go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 h1:ao8CJIShCaIbaMsGxy+jp2YHSudketpDgDRcbirov78= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0 h1:LrHL1A3KqIgAgi6mK7Q0aczmzU414AONAGT5xtnp+uo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0/go.mod h1:w8aZL87GMOvOBa2lU/JlVXE1q4chk/0FX+8ai4513bw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.8.0 h1:00hCSGLIxdYK/Z7r8GkaX0QIlfvgU3tmnLlQvcnix6U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.8.0/go.mod h1:twhIvtDQW2sWP1O2cT1N8nkSBgKCRZv2z6COTTBrf8Q= -go.opentelemetry.io/otel/sdk v1.8.0 h1:xwu69/fNuwbSHWe/0PGS888RmjWY181OmcXDQKu7ZQk= -go.opentelemetry.io/otel/sdk v1.8.0/go.mod h1:uPSfc+yfDH2StDM/Rm35WE8gXSNdvCg023J6HeGNO0c= -go.opentelemetry.io/otel/trace v1.8.0 h1:cSy0DF9eGI5WIfNwZ1q2iUyGj00tGzP24dE1lOlHrfY= -go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.18.0 h1:W5hyXNComRa23tGpKwG+FRAc4rfF6ZUg1JReK+QHS80= -go.opentelemetry.io/proto/otlp v0.18.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE= -go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= -go.uber.org/fx v1.18.2 h1:bUNI6oShr+OVFQeU8cDNbnN7VFsu+SsjHzUF51V/GAU= -go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= +go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= +go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY= +go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -1624,6 +1667,7 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1639,8 +1683,8 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1655,8 +1699,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w= -golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1684,8 +1728,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1742,15 +1786,14 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1786,8 +1829,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1815,6 +1858,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1823,12 +1867,14 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1850,7 +1896,10 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1881,6 +1930,7 @@ golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1894,16 +1944,19 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1913,14 +1966,14 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1976,6 +2029,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1989,9 +2043,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2000,9 +2053,9 @@ golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= -gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= +gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -2038,6 +2091,7 @@ google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6 google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= @@ -2123,7 +2177,10 @@ google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -2183,8 +2240,8 @@ google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5 google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= @@ -2217,11 +2274,13 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= -gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -2252,11 +2311,11 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= -lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= +nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= +nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/insecure/cmd/corrupted_builder.go b/insecure/cmd/corrupted_builder.go index b2791075934..34de857fda5 100644 --- a/insecure/cmd/corrupted_builder.go +++ b/insecure/cmd/corrupted_builder.go @@ -11,8 +11,11 @@ import ( "github.com/onflow/flow-go/insecure/corruptnet" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/p2pbuilder" + "github.com/onflow/flow-go/network/p2p/connection" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/utils/logging" ) @@ -70,25 +73,24 @@ func (cnb *CorruptedNodeBuilder) enqueueNetworkingLayer() { myAddr = cnb.FlowNodeBuilder.BaseConfig.BindAddr } - uniCfg := &p2pbuilder.UnicastConfig{ - StreamRetryInterval: cnb.UnicastCreateStreamRetryDelay, + uniCfg := &p2pconfig.UnicastConfig{ + StreamRetryInterval: cnb.FlowConfig.NetworkConfig.UnicastCreateStreamRetryDelay, RateLimiterDistributor: cnb.UnicastRateLimiterDistributor, } - connGaterCfg := &p2pbuilder.ConnectionGaterConfig{ + connGaterCfg := &p2pconfig.ConnectionGaterConfig{ InterceptPeerDialFilters: []p2p.PeerFilter{}, // disable connection gater onInterceptPeerDialFilters InterceptSecuredFilters: []p2p.PeerFilter{}, // disable connection gater onInterceptSecuredFilters } - peerManagerCfg := &p2pbuilder.PeerManagerConfig{ - ConnectionPruning: cnb.NetworkConnectionPruning, - UpdateInterval: cnb.PeerUpdateInterval, + peerManagerCfg := &p2pconfig.PeerManagerConfig{ + ConnectionPruning: cnb.FlowConfig.NetworkConfig.NetworkConnectionPruning, + UpdateInterval: cnb.FlowConfig.NetworkConfig.PeerUpdateInterval, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), } - cnb.GossipSubInspectorNotifDistributor = cmd.BuildGossipsubRPCValidationInspectorNotificationDisseminator(cnb.GossipSubRPCInspectorsConfig.GossipSubRPCInspectorNotificationCacheSize, cnb.MetricsRegisterer, cnb.Logger, cnb.MetricsEnabled) - // create default libp2p factory if corrupt node should enable the topic validator - libP2PNodeFactory := corruptlibp2p.NewCorruptLibP2PNodeFactory( + corruptLibp2pNode, err := corruptlibp2p.InitCorruptLibp2pNode( cnb.Logger, cnb.RootChainID, myAddr, @@ -102,24 +104,26 @@ func (cnb *CorruptedNodeBuilder) enqueueNetworkingLayer() { // run peer manager with the specified interval and let it also prune connections peerManagerCfg, uniCfg, - cnb.GossipSubConfig, + cnb.FlowConfig.NetworkConfig, + &p2p.DisallowListCacheConfig{ + MaxSize: cnb.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, + Metrics: metrics.DisallowListCacheMetricsFactory(cnb.HeroCacheMetricsFactory(), network.PrivateNetwork), + }, cnb.TopicValidatorDisabled, cnb.WithPubSubMessageSigning, cnb.WithPubSubStrictSignatureVerification, ) - - libp2pNode, err := libP2PNodeFactory() if err != nil { return nil, fmt.Errorf("failed to create libp2p node: %w", err) } - cnb.LibP2PNode = libp2pNode + cnb.LibP2PNode = corruptLibp2pNode cnb.Logger.Info(). Hex("node_id", logging.ID(cnb.NodeID)). Str("address", myAddr). Bool("topic_validator_disabled", cnb.TopicValidatorDisabled). Msg("corrupted libp2p node initialized") - return libp2pNode, nil + return corruptLibp2pNode, nil }) cnb.FlowNodeBuilder.OverrideComponent(cmd.NetworkComponent, func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { myAddr := cnb.FlowNodeBuilder.NodeConfig.Me.Address() diff --git a/insecure/cmd/mods_override.sh b/insecure/cmd/mods_override.sh index 6f6b4d4a6a7..ef0a732da73 100755 --- a/insecure/cmd/mods_override.sh +++ b/insecure/cmd/mods_override.sh @@ -6,7 +6,7 @@ cp ./go.mod ./go2.mod cp ./go.sum ./go2.sum # inject forked libp2p-pubsub into main module to allow building corrupt Docker images -echo "require github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee" >> ./go.mod +echo "require github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg.0.20230703223453-544e2fe28a26" >> ./go.mod # update go.sum since added new dependency go mod tidy diff --git a/insecure/corruptlibp2p/fixtures.go b/insecure/corruptlibp2p/fixtures.go index 599d1bcefe1..27a8e0aa2ba 100644 --- a/insecure/corruptlibp2p/fixtures.go +++ b/insecure/corruptlibp2p/fixtures.go @@ -29,14 +29,13 @@ func GossipSubCtrlFixture(opts ...GossipSubCtrlOption) *pubsubpb.ControlMessage } // WithIHave adds iHave control messages of the given size and number to the control message. -func WithIHave(msgCount int, msgSize int) GossipSubCtrlOption { +func WithIHave(msgCount, msgSize int, topicId string) GossipSubCtrlOption { return func(msg *pubsubpb.ControlMessage) { iHaves := make([]*pubsubpb.ControlIHave, msgCount) for i := 0; i < msgCount; i++ { - topicId := GossipSubTopicIdFixture() iHaves[i] = &pubsubpb.ControlIHave{ TopicID: &topicId, - MessageIDs: gossipSubMessageIdsFixture(msgSize), + MessageIDs: GossipSubMessageIdsFixture(msgSize), } } msg.Ihave = iHaves @@ -44,12 +43,12 @@ func WithIHave(msgCount int, msgSize int) GossipSubCtrlOption { } // WithIWant adds iWant control messages of the given size and number to the control message. -func WithIWant(msgCount int, msgSize int) GossipSubCtrlOption { +func WithIWant(msgCount, msgSize int) GossipSubCtrlOption { return func(msg *pubsubpb.ControlMessage) { iWants := make([]*pubsubpb.ControlIWant, msgCount) for i := 0; i < msgCount; i++ { iWants[i] = &pubsubpb.ControlIWant{ - MessageIDs: gossipSubMessageIdsFixture(msgSize), + MessageIDs: GossipSubMessageIdsFixture(msgSize), } } msg.Iwant = iWants @@ -94,8 +93,8 @@ func GossipSubTopicIdFixture() string { return unittest.GenerateRandomStringWithLen(topicIDFixtureLen) } -// gossipSubMessageIdsFixture returns a slice of random gossipSub message IDs of the given size. -func gossipSubMessageIdsFixture(count int) []string { +// GossipSubMessageIdsFixture returns a slice of random gossipSub message IDs of the given size. +func GossipSubMessageIdsFixture(count int) []string { msgIds := make([]string, count) for i := 0; i < count; i++ { msgIds[i] = gossipSubMessageIdFixture() diff --git a/insecure/corruptlibp2p/gossipsub_spammer.go b/insecure/corruptlibp2p/gossipsub_spammer.go index d3071802ad3..d38e1dfa12b 100644 --- a/insecure/corruptlibp2p/gossipsub_spammer.go +++ b/insecure/corruptlibp2p/gossipsub_spammer.go @@ -8,10 +8,12 @@ import ( pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" + corrupt "github.com/yhassanzadeh13/go-libp2p-pubsub" "github.com/onflow/flow-go/insecure/internal" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/network/p2p" p2ptest "github.com/onflow/flow-go/network/p2p/test" ) @@ -21,22 +23,48 @@ import ( type GossipSubRouterSpammer struct { router *atomicRouter SpammerNode p2p.LibP2PNode + SpammerId flow.Identity +} + +// NewGossipSubRouterSpammer creates a new GossipSubRouterSpammer. +// Args: +// - t: the test object. +// - sporkId: the spork node's ID. +// - role: the role of the spork node. +// - provider: the identity provider. +// Returns: +// - the GossipSubRouterSpammer. +func NewGossipSubRouterSpammer(t *testing.T, sporkId flow.Identifier, role flow.Role, provider module.IdentityProvider) *GossipSubRouterSpammer { + return NewGossipSubRouterSpammerWithRpcInspector(t, sporkId, role, provider, func(id peer.ID, rpc *corrupt.RPC) error { + return nil // no-op + }) } -// NewGossipSubRouterSpammer is the main method tests call for spamming attacks. -func NewGossipSubRouterSpammer(t *testing.T, sporkId flow.Identifier, role flow.Role) *GossipSubRouterSpammer { - spammerNode, router := createSpammerNode(t, sporkId, role) +// NewGossipSubRouterSpammerWithRpcInspector creates a new GossipSubRouterSpammer with a custom RPC inspector. +// The RPC inspector is called before each incoming RPC is processed by the router. +// If the inspector returns an error, the RPC is dropped. +// Args: +// - t: the test object. +// - sporkId: the spork node's ID. +// - role: the role of the spork node. +// - provider: the identity provider. +// - inspector: the RPC inspector. +// Returns: +// - the GossipSubRouterSpammer. +func NewGossipSubRouterSpammerWithRpcInspector(t *testing.T, sporkId flow.Identifier, role flow.Role, provider module.IdentityProvider, inspector func(id peer.ID, rpc *corrupt.RPC) error) *GossipSubRouterSpammer { + spammerNode, spammerId, router := newSpammerNodeWithRpcInspector(t, sporkId, role, provider, inspector) return &GossipSubRouterSpammer{ router: router, SpammerNode: spammerNode, + SpammerId: spammerId, } } // SpamControlMessage spams the victim with junk control messages. // ctlMessages is the list of spam messages to send to the victim node. -func (s *GossipSubRouterSpammer) SpamControlMessage(t *testing.T, victim p2p.LibP2PNode, ctlMessages []pb.ControlMessage) { +func (s *GossipSubRouterSpammer) SpamControlMessage(t *testing.T, victim p2p.LibP2PNode, ctlMessages []pb.ControlMessage, msgs ...*pb.Message) { for _, ctlMessage := range ctlMessages { - require.True(t, s.router.Get().SendControl(victim.Host().ID(), &ctlMessage)) + require.True(t, s.router.Get().SendControl(victim.Host().ID(), &ctlMessage, msgs...)) } } @@ -61,23 +89,39 @@ func (s *GossipSubRouterSpammer) Start(t *testing.T) { s.router.set(s.router.Get()) } -func createSpammerNode(t *testing.T, sporkId flow.Identifier, role flow.Role) (p2p.LibP2PNode, *atomicRouter) { +// newSpammerNodeWithRpcInspector creates a new spammer node, which is capable of sending spam control and actual messages to other nodes. +// It also creates a new atomic router that allows us to set the router to a new instance of the corrupt router. +// Args: +// - sporkId: the spork id of the spammer node. +// - role: the role of the spammer node. +// - provider: the identity provider of the spammer node. +// - inspector: the inspector function that is called when a message is received by the spammer node. +// Returns: +// - p2p.LibP2PNode: the spammer node. +// - flow.Identity: the identity of the spammer node. +// - *atomicRouter: the atomic router that allows us to set the router to a new instance of the corrupt router. +func newSpammerNodeWithRpcInspector( + t *testing.T, + sporkId flow.Identifier, + role flow.Role, + provider module.IdentityProvider, + inspector func(id peer.ID, rpc *corrupt.RPC) error) (p2p.LibP2PNode, flow.Identity, *atomicRouter) { router := newAtomicRouter() - spammerNode, _ := p2ptest.NodeFixture( - t, - sporkId, - t.Name(), - p2ptest.WithRole(role), + var opts []p2ptest.NodeFixtureParameterOption + opts = append(opts, p2ptest.WithRole(role), internal.WithCorruptGossipSub(CorruptGossipSubFactory(func(r *corrupt.GossipSubRouter) { require.NotNil(t, r) router.set(r) }), - CorruptGossipSubConfigFactoryWithInspector(func(id peer.ID, rpc *corrupt.RPC) error { - // here we can inspect the incoming RPC message to the spammer node - return nil - })), + CorruptGossipSubConfigFactoryWithInspector(inspector))) + spammerNode, spammerId := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + provider, + opts..., ) - return spammerNode, router + return spammerNode, spammerId, router } // atomicRouter is a wrapper around the corrupt.GossipSubRouter that allows atomic access to the router. diff --git a/insecure/corruptlibp2p/libp2p_node_factory.go b/insecure/corruptlibp2p/libp2p_node_factory.go index f7576b11057..43096e01f98 100644 --- a/insecure/corruptlibp2p/libp2p_node_factory.go +++ b/insecure/corruptlibp2p/libp2p_node_factory.go @@ -13,75 +13,101 @@ import ( fcrypto "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/distributor" "github.com/onflow/flow-go/network/p2p/p2pbuilder" - inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" ) -// NewCorruptLibP2PNodeFactory wrapper around the original DefaultLibP2PNodeFactory. Nodes returned from this factory func will be corrupted libp2p nodes. -func NewCorruptLibP2PNodeFactory( +// InitCorruptLibp2pNode initializes and returns a corrupt libp2p node that should only be used for BFT testing in +// the BFT testnet. This node is corrupt in the sense that it uses a forked version of the go-libp2p-pubsub library and +// is not compatible with the go-libp2p-pubsub library used by the other nodes in the network. This node should only be +// used for testing purposes. +// Args: +// - log: logger +// - chainID: chain id of the network this node is being used for (should be BFT testnet) +// - address: address of the node in the form of /ip4/ ... /tcp/ ... /p2p/ ... (see libp2p documentation for more info) +// - flowKey: private key of the node used for signing messages and establishing secure connections +// - sporkId: spork id of the network this node is being used for. +// - idProvider: identity provider used for translating peer ids to flow ids. +// - metricsCfg: metrics configuration used for initializing the metrics collector +// - resolver: resolver used for resolving multiaddresses to ip addresses +// - role: role of the node (a valid Flow role). +// - connGaterCfg: connection gater configuration used for initializing the connection gater +// - peerManagerCfg: peer manager configuration used for initializing the peer manager +// - uniCfg: unicast configuration used for initializing the unicast +// - gossipSubCfg: gossipsub configuration used for initializing the gossipsub +// - topicValidatorDisabled: whether or not topic validator is disabled +// - withMessageSigning: whether or not message signing is enabled +// - withStrictSignatureVerification: whether or not strict signature verification is enabled +// Returns: +// - p2p.LibP2PNode: initialized corrupt libp2p node +// - error: error if any. Any error returned from this function is fatal. +func InitCorruptLibp2pNode( log zerolog.Logger, chainID flow.ChainID, address string, flowKey fcrypto.PrivateKey, sporkId flow.Identifier, idProvider module.IdentityProvider, - metrics module.LibP2PMetrics, + metricsCfg module.NetworkMetrics, resolver madns.BasicResolver, role string, - connGaterCfg *p2pbuilder.ConnectionGaterConfig, - peerManagerCfg *p2pbuilder.PeerManagerConfig, - uniCfg *p2pbuilder.UnicastConfig, - gossipSubCfg *p2pbuilder.GossipSubConfig, + connGaterCfg *p2pconfig.ConnectionGaterConfig, + peerManagerCfg *p2pconfig.PeerManagerConfig, + uniCfg *p2pconfig.UnicastConfig, + netConfig *netconf.Config, + disallowListCacheCfg *p2p.DisallowListCacheConfig, topicValidatorDisabled, withMessageSigning, withStrictSignatureVerification bool, -) p2p.LibP2PFactoryFunc { - return func() (p2p.LibP2PNode, error) { - if chainID != flow.BftTestnet { - panic("illegal chain id for using corrupt libp2p node") - } +) (p2p.LibP2PNode, error) { + if chainID != flow.BftTestnet { + panic("illegal chain id for using corrupt libp2p node") + } - rpcInspectorBuilder := inspectorbuilder.NewGossipSubInspectorBuilder(log, sporkId, inspectorbuilder.DefaultGossipSubRPCInspectorsConfig(), distributor.DefaultGossipSubInspectorNotificationDistributor(log)) - rpcInspectors, err := rpcInspectorBuilder.Build() - if err != nil { - return nil, fmt.Errorf("failed to create gossipsub rpc inspectors for public libp2p node: %w", err) - } - gossipSubCfg.RPCInspectors = rpcInspectors + metCfg := &p2pconfig.MetricsConfig{ + HeroCacheFactory: metrics.NewNoopHeroCacheMetricsFactory(), + Metrics: metricsCfg, + } - builder, err := p2pbuilder.DefaultNodeBuilder( - log, - address, - flowKey, - sporkId, - idProvider, - metrics, - resolver, - role, - connGaterCfg, - peerManagerCfg, - gossipSubCfg, - p2pbuilder.DefaultResourceManagerConfig(), - uniCfg) + builder, err := p2pbuilder.DefaultNodeBuilder( + log, + address, + network.PrivateNetwork, + flowKey, + sporkId, + idProvider, + metCfg, + resolver, + role, + connGaterCfg, + peerManagerCfg, + &netConfig.GossipSubConfig, + &netConfig.GossipSubRPCInspectorsConfig, + &netConfig.ResourceManagerConfig, + uniCfg, + &netConfig.ConnectionManagerConfig, + disallowListCacheCfg) - if err != nil { - return nil, fmt.Errorf("could not create corrupt libp2p node builder: %w", err) - } - if topicValidatorDisabled { - builder.SetCreateNode(NewCorruptLibP2PNode) - } - - overrideWithCorruptGossipSub(builder, WithMessageSigning(withMessageSigning), WithStrictSignatureVerification(withStrictSignatureVerification)) - return builder.Build() + if err != nil { + return nil, fmt.Errorf("could not create corrupt libp2p node builder: %w", err) } + if topicValidatorDisabled { + builder.SetCreateNode(NewCorruptLibP2PNode) + } + + overrideWithCorruptGossipSub(builder, WithMessageSigning(withMessageSigning), WithStrictSignatureVerification(withStrictSignatureVerification)) + return builder.Build() } // CorruptGossipSubFactory returns a factory function that creates a new instance of the forked gossipsub module from // github.com/yhassanzadeh13/go-libp2p-pubsub for the purpose of BFT testing and attack vector implementation. func CorruptGossipSubFactory(routerOpts ...func(*corrupt.GossipSubRouter)) p2p.GossipSubFactoryFunc { - factory := func(ctx context.Context, logger zerolog.Logger, host host.Host, cfg p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error) { - adapter, router, err := NewCorruptGossipSubAdapter(ctx, logger, host, cfg) + factory := func(ctx context.Context, logger zerolog.Logger, host host.Host, cfg p2p.PubSubAdapterConfig, clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error) { + adapter, router, err := NewCorruptGossipSubAdapter(ctx, logger, host, cfg, clusterChangeConsumer) for _, opt := range routerOpts { opt(router) } diff --git a/insecure/corruptlibp2p/p2p_node.go b/insecure/corruptlibp2p/p2p_node.go index 143e1a9e938..b3fbd0cb36d 100644 --- a/insecure/corruptlibp2p/p2p_node.go +++ b/insecure/corruptlibp2p/p2p_node.go @@ -51,7 +51,12 @@ func (n *CorruptP2PNode) Subscribe(topic channels.Topic, _ p2p.TopicValidatorFun } // NewCorruptLibP2PNode returns corrupted libP2PNode that will subscribe to topics using the AcceptAllTopicValidator. -func NewCorruptLibP2PNode(logger zerolog.Logger, host host.Host, pCache p2p.ProtocolPeerCache, peerManager p2p.PeerManager) p2p.LibP2PNode { - node := p2pnode.NewNode(logger, host, pCache, peerManager) +func NewCorruptLibP2PNode( + logger zerolog.Logger, + host host.Host, + pCache p2p.ProtocolPeerCache, + peerManager p2p.PeerManager, + disallowListCacheCfg *p2p.DisallowListCacheConfig) p2p.LibP2PNode { + node := p2pnode.NewNode(logger, host, pCache, peerManager, disallowListCacheCfg) return &CorruptP2PNode{Node: node, logger: logger, codec: cbor.NewCodec()} } diff --git a/insecure/corruptlibp2p/pubsub_adapter.go b/insecure/corruptlibp2p/pubsub_adapter.go index c059bb0e3f1..f9d08eed50e 100644 --- a/insecure/corruptlibp2p/pubsub_adapter.go +++ b/insecure/corruptlibp2p/pubsub_adapter.go @@ -11,6 +11,7 @@ import ( corrupt "github.com/yhassanzadeh13/go-libp2p-pubsub" "github.com/onflow/flow-go/insecure/internal" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" @@ -28,9 +29,11 @@ import ( // totally separated from the rest of the codebase. type CorruptGossipSubAdapter struct { component.Component - gossipSub *corrupt.PubSub - router *corrupt.GossipSubRouter - logger zerolog.Logger + gossipSub *corrupt.PubSub + router *corrupt.GossipSubRouter + logger zerolog.Logger + clusterChangeConsumer p2p.CollectionClusterChangesConsumer + peerScoreExposer p2p.PeerScoreExposer } var _ p2p.PubSubAdapter = (*CorruptGossipSubAdapter)(nil) @@ -104,7 +107,26 @@ func (c *CorruptGossipSubAdapter) ListPeers(topic string) []peer.ID { return c.gossipSub.ListPeers(topic) } -func NewCorruptGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, *corrupt.GossipSubRouter, error) { +func (c *CorruptGossipSubAdapter) ActiveClustersChanged(lst flow.ChainIDList) { + c.clusterChangeConsumer.ActiveClustersChanged(lst) +} + +// PeerScoreExposer returns the peer score exposer for the gossipsub adapter. The exposer is a read-only interface +// for querying peer scores and returns the local scoring table of the underlying gossipsub node. +// The exposer is only available if the gossipsub adapter was configured with a score tracer. +// If the gossipsub adapter was not configured with a score tracer, the exposer will be nil. +// Args: +// +// None. +// +// Returns: +// +// The peer score exposer for the gossipsub adapter. +func (c *CorruptGossipSubAdapter) PeerScoreExposer() p2p.PeerScoreExposer { + return c.peerScoreExposer +} + +func NewCorruptGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig, clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, *corrupt.GossipSubRouter, error) { gossipSubConfig, ok := cfg.(*CorruptPubSubAdapterConfig) if !ok { return nil, nil, fmt.Errorf("invalid gossipsub config type: %T", cfg) @@ -119,18 +141,34 @@ func NewCorruptGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h ho return nil, nil, fmt.Errorf("failed to create corrupt gossipsub: %w", err) } - builder := component.NewComponentManagerBuilder(). - AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - ready() - <-ctx.Done() - }).Build() - + builder := component.NewComponentManagerBuilder() adapter := &CorruptGossipSubAdapter{ - Component: builder, - gossipSub: gossipSub, - router: router, - logger: logger, + gossipSub: gossipSub, + router: router, + logger: logger, + clusterChangeConsumer: clusterChangeConsumer, + } + + if scoreTracer := gossipSubConfig.ScoreTracer(); scoreTracer != nil { + builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + logger.Debug().Str("component", "corrupt-gossipsub_score_tracer").Msg("starting score tracer") + scoreTracer.Start(ctx) + logger.Debug().Str("component", "corrupt-gossipsub_score_tracer").Msg("score tracer started") + + <-scoreTracer.Done() + logger.Debug().Str("component", "corrupt-gossipsub_score_tracer").Msg("score tracer stopped") + }) + adapter.peerScoreExposer = scoreTracer } + builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + // it is likely that this adapter is configured without a score tracer, so we need to + // wait for the context to be done in order to prevent immature shutdown. + <-ctx.Done() + }) + + adapter.Component = builder.Build() return adapter, router, nil } diff --git a/insecure/corruptlibp2p/pubsub_adapter_config.go b/insecure/corruptlibp2p/pubsub_adapter_config.go index e9b7f65d6fe..1bae78dd872 100644 --- a/insecure/corruptlibp2p/pubsub_adapter_config.go +++ b/insecure/corruptlibp2p/pubsub_adapter_config.go @@ -24,6 +24,7 @@ type CorruptPubSubAdapterConfig struct { inspector func(peer.ID, *corrupt.RPC) error withMessageSigning bool withStrictSignatureVerification bool + scoreTracer p2p.PeerScoreTracer } type CorruptPubSubAdapterConfigOption func(config *CorruptPubSubAdapterConfig) @@ -78,12 +79,56 @@ func (c *CorruptPubSubAdapterConfig) WithSubscriptionFilter(filter p2p.Subscript c.options = append(c.options, corrupt.WithSubscriptionFilter(filter)) } -func (c *CorruptPubSubAdapterConfig) WithScoreOption(_ p2p.ScoreOptionBuilder) { - // CorruptPubSub does not support score options. This is a no-op. -} - -func (c *CorruptPubSubAdapterConfig) WithAppSpecificRpcInspectors(_ ...p2p.GossipSubRPCInspector) { - // CorruptPubSub receives its inspector at a different time than the original pubsub (i.e., at creation time). +func (c *CorruptPubSubAdapterConfig) WithScoreOption(option p2p.ScoreOptionBuilder) { + params, thresholds := option.BuildFlowPubSubScoreOption() + // convert flow pubsub score option to corrupt pubsub score option + corruptParams := &corrupt.PeerScoreParams{ + SkipAtomicValidation: params.SkipAtomicValidation, + TopicScoreCap: params.TopicScoreCap, + AppSpecificScore: params.AppSpecificScore, + AppSpecificWeight: params.AppSpecificWeight, + IPColocationFactorWeight: params.IPColocationFactorWeight, + IPColocationFactorThreshold: params.IPColocationFactorThreshold, + IPColocationFactorWhitelist: params.IPColocationFactorWhitelist, + BehaviourPenaltyWeight: params.BehaviourPenaltyWeight, + BehaviourPenaltyThreshold: params.BehaviourPenaltyThreshold, + BehaviourPenaltyDecay: params.BehaviourPenaltyDecay, + DecayInterval: params.DecayInterval, + DecayToZero: params.DecayToZero, + RetainScore: params.RetainScore, + SeenMsgTTL: params.SeenMsgTTL, + } + corruptThresholds := &corrupt.PeerScoreThresholds{ + SkipAtomicValidation: thresholds.SkipAtomicValidation, + GossipThreshold: thresholds.GossipThreshold, + PublishThreshold: thresholds.PublishThreshold, + GraylistThreshold: thresholds.GraylistThreshold, + AcceptPXThreshold: thresholds.AcceptPXThreshold, + OpportunisticGraftThreshold: thresholds.OpportunisticGraftThreshold, + } + for topic, topicParams := range params.Topics { + corruptParams.Topics[topic] = &corrupt.TopicScoreParams{ + SkipAtomicValidation: topicParams.SkipAtomicValidation, + TopicWeight: topicParams.TopicWeight, + TimeInMeshWeight: topicParams.TimeInMeshWeight, + TimeInMeshQuantum: topicParams.TimeInMeshQuantum, + TimeInMeshCap: topicParams.TimeInMeshCap, + FirstMessageDeliveriesWeight: topicParams.FirstMessageDeliveriesWeight, + FirstMessageDeliveriesDecay: topicParams.FirstMessageDeliveriesDecay, + FirstMessageDeliveriesCap: topicParams.FirstMessageDeliveriesCap, + MeshMessageDeliveriesWeight: topicParams.MeshMessageDeliveriesWeight, + MeshMessageDeliveriesDecay: topicParams.MeshMessageDeliveriesDecay, + MeshMessageDeliveriesCap: topicParams.MeshMessageDeliveriesCap, + MeshMessageDeliveriesThreshold: topicParams.MeshMessageDeliveriesThreshold, + MeshMessageDeliveriesWindow: topicParams.MeshMessageDeliveriesWindow, + MeshMessageDeliveriesActivation: topicParams.MeshMessageDeliveriesActivation, + MeshFailurePenaltyWeight: topicParams.MeshFailurePenaltyWeight, + MeshFailurePenaltyDecay: topicParams.MeshFailurePenaltyDecay, + InvalidMessageDeliveriesWeight: topicParams.InvalidMessageDeliveriesWeight, + InvalidMessageDeliveriesDecay: topicParams.InvalidMessageDeliveriesDecay, + } + } + c.options = append(c.options, corrupt.WithPeerScore(corruptParams, corruptThresholds)) } func (c *CorruptPubSubAdapterConfig) WithTracer(_ p2p.PubSubTracer) { @@ -96,8 +141,20 @@ func (c *CorruptPubSubAdapterConfig) WithMessageIdFunction(f func([]byte) string return f(pmsg.Data) })) } -func (c *CorruptPubSubAdapterConfig) WithScoreTracer(_ p2p.PeerScoreTracer) { - // CorruptPubSub does not support score tracer. This is a no-op. + +func (c *CorruptPubSubAdapterConfig) WithScoreTracer(tracer p2p.PeerScoreTracer) { + c.scoreTracer = tracer + c.options = append(c.options, corrupt.WithPeerScoreInspect(func(snapshot map[peer.ID]*corrupt.PeerScoreSnapshot) { + tracer.UpdatePeerScoreSnapshots(convertPeerScoreSnapshots(snapshot)) + }, tracer.UpdateInterval())) +} + +func (c *CorruptPubSubAdapterConfig) ScoreTracer() p2p.PeerScoreTracer { + return c.scoreTracer +} + +func (c *CorruptPubSubAdapterConfig) WithInspectorSuite(_ p2p.GossipSubInspectorSuite) { + // CorruptPubSub does not support inspector suite. This is a no-op. } func (c *CorruptPubSubAdapterConfig) Build() []corrupt.Option { @@ -111,3 +168,43 @@ func defaultCorruptPubsubOptions(base *p2p.BasePubSubAdapterConfig, withMessageS corrupt.WithMaxMessageSize(base.MaxMessageSize), } } + +// convertPeerScoreSnapshots converts a libp2p pubsub peer score snapshot to a Flow peer score snapshot. +// Args: +// - snapshot: the libp2p pubsub peer score snapshot. +// +// Returns: +// - map[peer.ID]*p2p.PeerScoreSnapshot: the Flow peer score snapshot. +func convertPeerScoreSnapshots(snapshot map[peer.ID]*corrupt.PeerScoreSnapshot) map[peer.ID]*p2p.PeerScoreSnapshot { + newSnapshot := make(map[peer.ID]*p2p.PeerScoreSnapshot) + for id, snap := range snapshot { + newSnapshot[id] = &p2p.PeerScoreSnapshot{ + Topics: convertTopicScoreSnapshot(snap.Topics), + Score: snap.Score, + AppSpecificScore: snap.AppSpecificScore, + BehaviourPenalty: snap.BehaviourPenalty, + IPColocationFactor: snap.IPColocationFactor, + } + } + return newSnapshot +} + +// convertTopicScoreSnapshot converts a libp2p pubsub topic score snapshot to a Flow topic score snapshot. +// Args: +// - snapshot: the libp2p pubsub topic score snapshot. +// +// Returns: +// - map[string]*p2p.TopicScoreSnapshot: the Flow topic score snapshot. +func convertTopicScoreSnapshot(snapshot map[string]*corrupt.TopicScoreSnapshot) map[string]*p2p.TopicScoreSnapshot { + newSnapshot := make(map[string]*p2p.TopicScoreSnapshot) + for topic, snap := range snapshot { + newSnapshot[topic] = &p2p.TopicScoreSnapshot{ + TimeInMesh: snap.TimeInMesh, + FirstMessageDeliveries: snap.FirstMessageDeliveries, + MeshMessageDeliveries: snap.MeshMessageDeliveries, + InvalidMessageDeliveries: snap.InvalidMessageDeliveries, + } + } + + return newSnapshot +} diff --git a/insecure/corruptlibp2p/spam_test.go b/insecure/corruptlibp2p/spam_test.go index c99c07f308f..2886b598c66 100644 --- a/insecure/corruptlibp2p/spam_test.go +++ b/insecure/corruptlibp2p/spam_test.go @@ -2,6 +2,7 @@ package corruptlibp2p_test import ( "context" + "fmt" "sync" "testing" "time" @@ -30,17 +31,18 @@ func TestSpam_IHave(t *testing.T) { const messagesToSpam = 3 sporkId := unittest.IdentifierFixture() role := flow.RoleConsensus - - gsrSpammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkId, role) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) + gsrSpammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkId, role, idProvider) allSpamIHavesReceived := sync.WaitGroup{} allSpamIHavesReceived.Add(messagesToSpam) var iHaveReceivedCtlMsgs []pb.ControlMessage - victimNode, _ := p2ptest.NodeFixture( + victimNode, victimIdentity := p2ptest.NodeFixture( t, sporkId, t.Name(), + idProvider, p2ptest.WithRole(role), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(func(id peer.ID, rpc *corrupt.RPC) error { @@ -54,7 +56,7 @@ func TestSpam_IHave(t *testing.T) { return nil })), ) - + idProvider.SetIdentities(flow.IdentityList{&victimIdentity, &gsrSpammer.SpammerId}) // starts nodes ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) @@ -68,14 +70,14 @@ func TestSpam_IHave(t *testing.T) { // prior to the test we should ensure that spammer and victim connect. // this is vital as the spammer will circumvent the normal pubsub subscription mechanism and send iHAVE messages directly to the victim. // without a prior connection established, directly spamming pubsub messages may cause a race condition in the pubsub implementation. - p2ptest.EnsureConnected(t, ctx, nodes) - p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - return unittest.ProposalFixture(), blockTopic + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) // prepare to spam - generate iHAVE control messages - iHaveSentCtlMsgs := gsrSpammer.GenerateCtlMessages(messagesToSpam, corruptlibp2p.WithIHave(messagesToSpam, 5)) + iHaveSentCtlMsgs := gsrSpammer.GenerateCtlMessages(messagesToSpam, corruptlibp2p.WithIHave(messagesToSpam, 5, fmt.Sprintf("%s/%s", channels.PushBlocks, sporkId))) // start spamming the victim peer gsrSpammer.SpamControlMessage(t, victimNode, iHaveSentCtlMsgs) diff --git a/insecure/corruptnet/conduit.go b/insecure/corruptnet/conduit.go index 418a392ba8b..eb38cad9c0e 100644 --- a/insecure/corruptnet/conduit.go +++ b/insecure/corruptnet/conduit.go @@ -20,7 +20,14 @@ type Conduit struct { egressController insecure.EgressController } -var _ network.Conduit = &Conduit{} +// ReportMisbehavior reports the misbehavior of a node on sending a message to the current node that appears valid +// based on the networking layer but is considered invalid by the current node based on the Flow protocol. +// This method is a no-op in the test helper implementation. +func (c *Conduit) ReportMisbehavior(_ network.MisbehaviorReport) { + // no-op +} + +var _ network.Conduit = (*Conduit)(nil) // Publish sends the incoming events as publish events to the controller of this conduit (i.e., its factory) to handle. func (c *Conduit) Publish(event interface{}, targetIDs ...flow.Identifier) error { diff --git a/insecure/corruptnet/network.go b/insecure/corruptnet/network.go index 8a45d603ab5..14486a1c286 100644 --- a/insecure/corruptnet/network.go +++ b/insecure/corruptnet/network.go @@ -63,10 +63,10 @@ type Network struct { approvalHasher hash.Hasher } -var _ flownet.Network = &Network{} -var _ insecure.EgressController = &Network{} -var _ insecure.IngressController = &Network{} -var _ insecure.CorruptNetworkServer = &Network{} +var _ flownet.Network = (*Network)(nil) +var _ insecure.EgressController = (*Network)(nil) +var _ insecure.IngressController = (*Network)(nil) +var _ insecure.CorruptNetworkServer = (*Network)(nil) func NewCorruptNetwork( logger zerolog.Logger, diff --git a/insecure/go.mod b/insecure/go.mod index 2cb2fb0b401..b727c8ecf6c 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -1,30 +1,30 @@ module github.com/onflow/flow-go/insecure -go 1.19 +go 1.20 require ( - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.3 github.com/hashicorp/go-multierror v1.1.1 github.com/ipfs/go-datastore v0.6.0 - github.com/libp2p/go-libp2p v0.24.2 - github.com/libp2p/go-libp2p-pubsub v0.8.2 + github.com/libp2p/go-libp2p v0.28.1 + github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/multiformats/go-multiaddr-dns v0.3.1 - github.com/onflow/flow-go v0.29.8 - github.com/onflow/flow-go/crypto v0.24.7 + github.com/onflow/flow-go v0.31.1-0.20230718164039-e3411eff1e9d + github.com/onflow/flow-go/crypto v0.24.9 github.com/rs/zerolog v1.29.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.2 - github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee - go.uber.org/atomic v1.10.0 - google.golang.org/grpc v1.53.0 + github.com/stretchr/testify v1.8.4 + github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg.0.20230703223453-544e2fe28a26 + go.uber.org/atomic v1.11.0 + google.golang.org/grpc v1.56.1 google.golang.org/protobuf v1.30.0 ) require ( cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute v1.19.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.12.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect cloud.google.com/go/storage v1.28.1 // indirect github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect @@ -42,23 +42,24 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect github.com/aws/smithy-go v1.13.5 // indirect - github.com/benbjohnson/clock v1.3.0 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.3.0 // indirect + github.com/bits-and-blooms/bitset v1.5.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/containerd/cgroups v1.0.4 // indirect + github.com/containerd/cgroups v1.1.0 // indirect + github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect - github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -67,29 +68,32 @@ require ( github.com/ethereum/go-ethereum v1.9.13 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/fxamacker/cbor/v2 v2.4.1-0.20220515183430-ad2eae63303f // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c // indirect github.com/fxamacker/circlehash v0.3.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gammazero/deque v0.1.0 // indirect github.com/gammazero/workerpool v1.1.2 // indirect - github.com/ghodss/yaml v1.0.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect - github.com/go-test/deep v1.0.8 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.1 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/go-test/deep v1.1.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.0.0 // indirect + github.com/golang/glog v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 // indirect + github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.7.1 // indirect @@ -98,128 +102,130 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-20200501113911-9a95f0fdbfea // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huin/goupnp v1.0.3 // indirect + github.com/huin/goupnp v1.2.0 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/go-block-format v0.0.3 // indirect + github.com/ipfs/boxo v0.10.0 // indirect + github.com/ipfs/go-block-format v0.1.2 // indirect github.com/ipfs/go-blockservice v0.4.0 // indirect - github.com/ipfs/go-cid v0.3.2 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-ds-badger2 v0.1.3 // indirect github.com/ipfs/go-fetcher v1.5.0 // indirect - github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect + github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect - github.com/ipfs/go-ipfs-pq v0.0.2 // indirect + github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-provider v0.7.0 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect - github.com/ipfs/go-ipld-format v0.3.0 // indirect - github.com/ipfs/go-ipns v0.2.0 // indirect + github.com/ipfs/go-ipld-format v0.5.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect - github.com/ipfs/go-peertaskqueue v0.7.0 // indirect + github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-verifcid v0.0.1 // indirect - github.com/ipld/go-ipld-prime v0.14.1 // indirect + github.com/ipld/go-ipld-prime v0.20.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/go-bindata v3.23.0+incompatible // indirect - github.com/klauspost/compress v1.15.13 // indirect - github.com/klauspost/cpuid/v2 v2.2.2 // indirect - github.com/koron/go-ssdp v0.0.3 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/koron/go-ssdp v0.0.4 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/libp2p/go-addr-util v0.1.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect - github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect + github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect github.com/libp2p/go-libp2p-core v0.20.1 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.19.0 // indirect - github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.24.2 // indirect + github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-msgio v0.2.0 // indirect - github.com/libp2p/go-nat v0.1.0 // indirect + github.com/libp2p/go-msgio v0.3.0 // indirect + github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect - github.com/libp2p/go-openssl v0.1.0 // indirect - github.com/libp2p/go-reuseport v0.2.0 // indirect + github.com/libp2p/go-reuseport v0.3.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.0 // indirect - github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/lucas-clemente/quic-go v0.31.1 // indirect + github.com/logrusorgru/aurora/v4 v4.0.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c // indirect - github.com/magiconair/properties v1.8.6 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect - github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect - github.com/mattn/go-pointer v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/miekg/dns v1.1.50 // indirect + github.com/miekg/dns v1.1.54 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr v0.8.0 // indirect + github.com/multiformats/go-multiaddr v0.9.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect - github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multicodec v0.7.0 // indirect - github.com/multiformats/go-multihash v0.2.1 // indirect - github.com/multiformats/go-multistream v0.3.3 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multicodec v0.9.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onflow/atree v0.5.0 // indirect - github.com/onflow/cadence v0.38.1 // indirect - github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 // indirect - github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 // indirect - github.com/onflow/flow-ft/lib/go/contracts v0.5.0 // indirect - github.com/onflow/flow-go-sdk v0.40.0 // indirect - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230330183547-d0dd18f6f20d // indirect - github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 // indirect + github.com/onflow/atree v0.6.0 // indirect + github.com/onflow/cadence v0.39.14 // indirect + github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d // indirect + github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 // indirect + github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect + github.com/onflow/flow-go-sdk v0.41.9 // indirect + github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce // indirect + github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d // indirect github.com/onflow/sdks v0.5.0 // indirect - github.com/onsi/ginkgo/v2 v2.6.1 // indirect + github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d // indirect + github.com/onsi/ginkgo/v2 v2.9.7 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect + github.com/polydawn/refmt v0.89.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/psiemens/sconfig v0.1.0 // indirect + github.com/quic-go/qpack v0.4.0 // indirect + github.com/quic-go/qtls-go1-19 v0.3.2 // indirect + github.com/quic-go/qtls-go1-20 v0.2.2 // indirect + github.com/quic-go/quic-go v0.33.0 // indirect + github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect - github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/rs/cors v1.8.0 // indirect - github.com/schollz/progressbar/v3 v3.8.3 // indirect + github.com/schollz/progressbar/v3 v3.13.1 // indirect github.com/sethvargo/go-retry v0.2.3 // indirect github.com/shirou/gopsutil/v3 v3.22.2 // indirect github.com/slok/go-http-metrics v0.10.0 // indirect - github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect + github.com/sony/gobreaker v0.5.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.9.0 // indirect + github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/cobra v1.6.1 // indirect + github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/viper v1.12.0 // indirect + github.com/spf13/viper v1.15.0 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/subosito/gotenv v1.4.0 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect @@ -228,43 +234,43 @@ require ( github.com/vmihailenco/msgpack/v4 v4.3.11 // indirect github.com/vmihailenco/tagparser v0.1.1 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect - github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.8.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.8.0 // indirect - go.opentelemetry.io/otel/sdk v1.8.0 // indirect - go.opentelemetry.io/otel/trace v1.8.0 // indirect - go.opentelemetry.io/proto/otlp v0.18.0 // indirect - go.uber.org/dig v1.15.0 // indirect - go.uber.org/fx v1.18.2 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.opentelemetry.io/otel v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/sdk v1.16.0 // indirect + go.opentelemetry.io/otel/trace v1.16.0 // indirect + go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.uber.org/dig v1.17.0 // indirect + go.uber.org/fx v1.19.2 // indirect + go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.4.0 // indirect - golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/oauth2 v0.6.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/term v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect + golang.org/x/time v0.1.0 // indirect + golang.org/x/tools v0.9.1 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 // indirect - gopkg.in/ini.v1 v1.66.6 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/blake3 v1.1.7 // indirect - nhooyr.io/websocket v1.8.6 // indirect + lukechampine.com/blake3 v1.2.1 // indirect + nhooyr.io/websocket v1.8.7 // indirect ) replace github.com/onflow/flow-go => ../ diff --git a/insecure/go.sum b/insecure/go.sum index 68ceeb3ef8d..308ac3ab9f3 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -19,6 +19,16 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -27,14 +37,15 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= -cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/kms v1.0.0/go.mod h1:nhUehi+w7zht2XrUfvTRNpxrfayBHqP4lu2NSywui/0= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/profiler v0.3.0 h1:R6y/xAeifaUXxd2x6w+jIwKxoKl8Cv5HJvcvASTPWJo= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -85,6 +96,7 @@ github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -147,15 +159,16 @@ github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAm github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.3.0 h1:h7mv5q31cthBTd7V4kLAZaIThj1e8vPGcSqpPue9KVI= -github.com/bits-and-blooms/bitset v1.3.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= +github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= @@ -178,12 +191,14 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bytecodealliance/wasmtime-go v0.22.0/go.mod h1:q320gUxqyI8yB+ZqRuaJOEnGkAnHh6WtJjMaT2CW4wI= +github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -211,12 +226,13 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -232,7 +248,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -243,9 +258,9 @@ github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQY github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= @@ -257,8 +272,9 @@ github.com/dgraph-io/badger/v2 v2.2007.3/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDm github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= @@ -291,9 +307,11 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= github.com/ethereum/go-ethereum v1.9.13 h1:rOPqjSngvs1VSYH2H+PMPiWt4VEulvNRbFgqiGqJM3E= github.com/ethereum/go-ethereum v1.9.13/go.mod h1:qwN9d1GLyDh0N7Ab8bMGd0H9knaji2jOBm2RrMGjXls= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -303,27 +321,30 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/fxamacker/cbor/v2 v2.4.1-0.20220515183430-ad2eae63303f h1:dxTR4AaxCwuQv9LAVTAC2r1szlS+epeuPT5ClLKT6ZY= -github.com/fxamacker/cbor/v2 v2.4.1-0.20220515183430-ad2eae63303f/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fxamacker/cbor/v2 v2.2.1-0.20210927235116-3d6d5d1de29b/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c h1:5tm/Wbs9d9r+qZaUFXk59CWDD0+77PBqDREffYkyi5c= +github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/circlehash v0.1.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM= github.com/fxamacker/circlehash v0.3.0 h1:XKdvTtIJV9t7DDUtsf0RIpC1OcxZtPbmgIH7ekx28WA= github.com/fxamacker/circlehash v0.3.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gammazero/deque v0.1.0 h1:f9LnNmq66VDeuAlSAapemq/U7hJ2jpIWa4c09q8Dlik= github.com/gammazero/deque v0.1.0/go.mod h1:KQw7vFau1hHuM8xmI9RbgKFbAsQFWmBpqQ2KenFLk6M= github.com/gammazero/workerpool v1.1.2 h1:vuioDQbgrz4HoaCi2q1HLlOXdpbap5AET7xu5/qj87g= github.com/gammazero/workerpool v1.1.2/go.mod h1:UelbXcO0zCIGFcufcirHhq2/xtLXJdQ29qZNlXG9OjQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -349,29 +370,35 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= +github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= @@ -390,9 +417,11 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -409,6 +438,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -428,8 +458,10 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -463,6 +495,7 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -474,8 +507,13 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 h1:wORs2YN3R3ona/CXYuTvLM31QlgoNKHvlCNuArCDDCU= -github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -489,11 +527,13 @@ github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -521,8 +561,9 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -548,6 +589,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= +github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -558,29 +601,32 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= -github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= +github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= +github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= +github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUiJ2teMvI= github.com/ipfs/go-bitswap v0.5.0/go.mod h1:WwyyYD33RHCpczgHjpx+xjWYIy8l41K+l5EMy4/ctSM= github.com/ipfs/go-bitswap v0.9.0 h1:/dZi/XhUN/aIk78pI4kaZrilUglJ+7/SCmOHWIpiy8E= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= +github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= +github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-blockservice v0.2.0/go.mod h1:Vzvj2fAnbbyly4+T7D5+p9n3+ZKVHA2bRMMo1QoILtQ= github.com/ipfs/go-blockservice v0.4.0 h1:7MUijAW5SqdsqEW/EhnNFRJXVF8mGU5aGhZ3CQaCWbY= @@ -593,8 +639,8 @@ github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67Fexh github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= -github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= @@ -625,8 +671,8 @@ github.com/ipfs/go-fetcher v1.5.0/go.mod h1:5pDZ0393oRF/fHiLmtFZtpMNBQfHOYNPtryW github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blockstore v0.2.0/go.mod h1:SNeEpz/ICnMYZQYr7KNZTjdn7tEPB/99xpe8xI1RW7o= -github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= -github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= +github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12XwJyvYvMM= +github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -644,8 +690,9 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAz github.com/ipfs/go-ipfs-exchange-offline v0.1.0/go.mod h1:YdJXa+yPF1na+gfYHYejtLwHFpuKv22eatApNiSfanM= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= -github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= +github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= +github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= github.com/ipfs/go-ipfs-provider v0.7.0 h1:5GpHv46eIS8h2mbbKg1ckU5paajDYJtE4GA/SBepOQg= github.com/ipfs/go-ipfs-provider v0.7.0/go.mod h1:mgjsWgDt9j19N1REPxRa31p+eRIQmjNt5McNdQQ5CsA= github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= @@ -654,10 +701,8 @@ github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FF github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-format v0.3.0 h1:Mwm2oRLzIuUwEPewWAWyMuuBQUsn3awfFEYVb8akMOQ= -github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipns v0.2.0 h1:BgmNtQhqOw5XEZ8RAfWEpK4DhqaYiuP6h71MhIp7xXU= -github.com/ipfs/go-ipns v0.2.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= +github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds= +github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= @@ -677,13 +722,14 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= -github.com/ipfs/go-peertaskqueue v0.7.0 h1:VyO6G4sbzX80K58N60cCaHsSsypbUNs1GjO5seGNsQ0= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= +github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= +github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.14.1 h1:n9obcUnuqPK34HlfbiB+o9GhXE/x59uue4z9YTsaoj4= -github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= +github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -722,9 +768,11 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/go-bindata v3.23.0+incompatible h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8= github.com/kevinburke/go-bindata v3.23.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -735,37 +783,36 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= -github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= -github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= -github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= -github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= +github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= +github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= github.com/libp2p/go-addr-util v0.1.0 h1:acKsntI33w2bTU7tC9a0SaPimJGfSI0bFKC18ChxeVI= @@ -792,10 +839,10 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= -github.com/libp2p/go-libp2p v0.24.2 h1:iMViPIcLY0D6zr/f+1Yq9EavCZu2i7eDstsr1nEwSAk= -github.com/libp2p/go-libp2p v0.24.2/go.mod h1:WuxtL2V8yGjam03D93ZBC19tvOUiPpewYv1xdFGWu1k= -github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= -github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= +github.com/libp2p/go-libp2p v0.28.1 h1:YurK+ZAI6cKfASLJBVFkpVBdl3wGhFi6fusOt725ii8= +github.com/libp2p/go-libp2p v0.28.1/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= +github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= +github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI= @@ -839,10 +886,10 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-kad-dht v0.19.0 h1:2HuiInHZTm9ZvQajaqdaPLHr0PCKKigWiflakimttE0= -github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= -github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= -github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= +github.com/libp2p/go-libp2p-kad-dht v0.24.2 h1:zd7myKBKCmtZBhI3I0zm8xBkb28v3gmSEtQfBdAdFwc= +github.com/libp2p/go-libp2p-kad-dht v0.24.2/go.mod h1:BShPzRbK6+fN3hk8a0WGAYKpb8m4k+DtchkqouGTrSg= +github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= +github.com/libp2p/go-libp2p-kbucket v0.6.3/go.mod h1:RCseT7AH6eJWxxk2ol03xtP9pEHetYSPXOaJnOiD8i0= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= @@ -867,8 +914,8 @@ github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRj github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= -github.com/libp2p/go-libp2p-pubsub v0.8.2 h1:QLGUmkgKmwEVxVDYGsqc5t9CykOMY2Y21cXQHjR462I= -github.com/libp2p/go-libp2p-pubsub v0.8.2/go.mod h1:e4kT+DYjzPUYGZeWk4I+oxCSYTXizzXii5LDRRhjKSw= +github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= +github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc= github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= @@ -921,13 +968,13 @@ github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= -github.com/libp2p/go-msgio v0.2.0 h1:W6shmB+FeynDrUVl2dgFQvzfBZcXiyqY4VmpQLu9FqU= -github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= +github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= +github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= -github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= -github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= +github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= +github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= @@ -939,12 +986,10 @@ github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= -github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= -github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= -github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= +github.com/libp2p/go-reuseport v0.3.0 h1:iiZslO5byUYZEg9iCwJGf5h+sf1Agmqx2V2FDjPyvUw= +github.com/libp2p/go-reuseport v0.3.0/go.mod h1:laea40AimhtfEqysZ71UpYj4S+R9VpH8PgqLo7L+SwI= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= @@ -977,37 +1022,30 @@ github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rB github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= -github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= +github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= -github.com/lucas-clemente/quic-go v0.31.1 h1:O8Od7hfioqq0PMYHDyBkxU2aA7iZ2W9pjbrWuja2YR4= -github.com/lucas-clemente/quic-go v0.31.1/go.mod h1:0wFbizLgYzqHqtlyxyCaJKlE7bYgE6JQ+54TLd/Dq2g= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c h1:OqVcb1Dkheracn4fgCjxlfhuSnM8jmPbrWkJbRIC4fo= -github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c/go.mod h1:5/Yq7mnb+VdE44ff+FL8LSOPEquOVqm/7Hz40U4VUZo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= -github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= -github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= -github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/marten-seemann/webtransport-go v0.4.3 h1:vkt5o/Ci+luknRteWdYGYH1KcB7ziup+J+1PzZJIvmg= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -1019,19 +1057,23 @@ github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXT github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= -github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1042,8 +1084,8 @@ github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= +github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -1056,8 +1098,9 @@ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= @@ -1100,8 +1143,8 @@ github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= -github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= -github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= +github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -1120,11 +1163,10 @@ github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysj github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= -github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= -github.com/multiformats/go-multicodec v0.7.0 h1:rTUjGOwjlhGHbEMbPoSUJowG1spZTVsITRANCjKTUAQ= -github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= +github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1133,16 +1175,15 @@ github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.0.16/go.mod h1:zhfEIgVnB/rPMfxgFw15ZmGoNaKyNUIE4IWHG/kC+Ag= -github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= github.com/multiformats/go-multistream v0.2.0/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= github.com/multiformats/go-multistream v0.2.2/go.mod h1:UIcnm7Zuo8HKG+HkWgfQsGL+/MIEhyTqbODbIUwSXKs= -github.com/multiformats/go-multistream v0.3.3 h1:d5PZpjwRgVlbwfdTDjife7XszfZd8KYWfROYFlGcR8o= -github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= +github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= +github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -1171,41 +1212,50 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onflow/atree v0.5.0 h1:y3lh8hY2fUo8KVE2ALVcz0EiNTq0tXJ6YTXKYVDA+3E= -github.com/onflow/atree v0.5.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= -github.com/onflow/cadence v0.38.1 h1:8YpnE1ixAGB8hF3t+slkHGhjfIBJ95dqUS+sEHrM2kY= -github.com/onflow/cadence v0.38.1/go.mod h1:SpfjNhPsJxGIHbOthE9JD/e8JFaFY73joYLPsov+PY4= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 h1:9QEI+C9k/Cx/TRC3SCAHmNQqV7UlLG0DHQewTl8Lg6w= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1/go.mod h1:xiSs5IkirazpG89H5jH8xVUKNPlCZqUhLH4+vikQVS4= -github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 h1:dhXSFiBkS6Q3XmBctJAfwR4XPkgBT7VNx08F/zTBgkM= -github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1/go.mod h1:cBimYbTvHK77lclJ1JyhvmKAB9KDzCeWm7OW1EeQSr0= -github.com/onflow/flow-ft/lib/go/contracts v0.5.0 h1:Cg4gHGVblxcejfNNG5Mfj98Wf4zbY76O0Y28QB0766A= -github.com/onflow/flow-ft/lib/go/contracts v0.5.0/go.mod h1:1zoTjp1KzNnOPkyqKmWKerUyf0gciw+e6tAEt0Ks3JE= -github.com/onflow/flow-go-sdk v0.40.0 h1:s8uwoyTquN8tjdXpqGmNkXTjf79yUII8JExc5QEl4Xw= -github.com/onflow/flow-go-sdk v0.40.0/go.mod h1:34dxXk9Hp/bQw6Zy6+H44Xo0kQU+aJyQoqdDxq00rJM= -github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= -github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230330183547-d0dd18f6f20d h1:Wl8bE1YeZEcRNnCpxw2rikOEaivuYKDrnJd2vsfIWoA= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230330183547-d0dd18f6f20d/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= -github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8/go.mod h1:73C8FlT4L/Qe4Cf5iXUNL8b2pvu4zs5dJMMJ5V2TjUI= +github.com/onflow/atree v0.1.0-beta1.0.20211027184039-559ee654ece9/go.mod h1:+6x071HgCF/0v5hQcaE5qqjc2UqN5gCU8h5Mk6uqpOg= +github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= +github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= +github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= +github.com/onflow/cadence v0.39.14 h1:YoR3YFUga49rqzVY1xwI6I2ZDBmvwGh13jENncsleC8= +github.com/onflow/cadence v0.39.14/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d h1:B7PdhdUNkve5MVrekWDuQf84XsGBxNZ/D3x+QQ8XeVs= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d/go.mod h1:xAiV/7TKhw863r6iO3CS5RnQ4F+pBY1TxD272BsILlo= +github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= +github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= +github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= +github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= +github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= +github.com/onflow/flow-go-sdk v0.41.9 h1:cyplhhhc0RnfOAan2t7I/7C9g1hVGDDLUhWj6ZHAkk4= +github.com/onflow/flow-go-sdk v0.41.9/go.mod h1:e9Q5TITCy7g08lkdQJxP8fAKBnBoC5FjALvUKr36j4I= +github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow-go/crypto v0.24.9 h1:0EQp+kSZYJepMIiSypfJVe7tzsPcb6UXOdOtsTCDhBs= +github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= +github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= +github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= +github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= github.com/onflow/sdks v0.5.0/go.mod h1:F0dj0EyHC55kknLkeD10js4mo14yTdMotnWMslPirrU= +github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d h1:gAEqYPn3DS83rHIKEpsajnppVD1+zwuYPFyeDVFaQvg= +github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d/go.mod h1:iMC8gkLqu4nkbkAla5HkSBb+FGyQOZiWz3DYm2wSXCk= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.6.1 h1:1xQPCjcqYw/J5LchOcp4/2q/jzJFjiAOc25chhnDw+Q= -github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= +github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1227,10 +1277,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.2 h1:+jQXlF3scKIcSEKkdHzXhCTDLPFi5r1wnK6yPS+49Gw= -github.com/pelletier/go-toml/v2 v2.0.2/go.mod h1:MovirKjgVRESsAvNZlAjtFwV867yGuwRkXbG66OzopI= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= @@ -1243,10 +1291,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= +github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= @@ -1266,8 +1316,8 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1277,8 +1327,8 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= -github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1294,18 +1344,28 @@ github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/psiemens/sconfig v0.1.0 h1:xfWqW+TRpih7mXZIqKYTmpRhlZLQ1kbxV8EjllPv76s= github.com/psiemens/sconfig v0.1.0/go.mod h1:+MLKqdledP/8G3rOBpknbLh0IclCf4WneJUtS26JB2U= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U= +github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= +github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E= +github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= +github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0= +github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA= +github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= +github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a h1:s7GrsqeorVkFR1vGmQ6WVL9nup0eyQCC+YVUeSQLH/Q= -github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= @@ -1321,8 +1381,9 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8= github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko= +github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= +github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sethvargo/go-retry v0.2.3 h1:oYlgvIvsju3jNbottWABtbnoLC+GDtLdBHxKWxQm/iU= @@ -1359,32 +1420,35 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/slok/go-http-metrics v0.10.0 h1:rh0LaYEKza5eaYRGDXujKrOln57nHBi4TtVhmNEpbgM= github.com/slok/go-http-metrics v0.10.0/go.mod h1:lFqdaS4kWMfUKCSukjC47PdCeTk+hXDUVm8kLHRqJ38= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= +github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= -github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.9.0 h1:sFSLUHgxdnN32Qy38hK3QkYBFXZj9DKjVjCUCtD7juY= -github.com/spf13/afero v1.9.0/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1394,8 +1458,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= -github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= @@ -1415,13 +1479,14 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= -github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/supranational/blst v0.3.4/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/supranational/blst v0.3.10 h1:CMciDZ/h4pXDDXQASe8ZGTNKUiVNxVVA5hpci2Uuhuk= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= @@ -1446,6 +1511,7 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -1454,10 +1520,10 @@ github.com/vmihailenco/msgpack/v4 v4.3.11 h1:Q47CePddpNGNhk4GCnAx9DDtASi2rasatE0 github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA= -github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= -github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= +github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= @@ -1467,16 +1533,14 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= -github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= -github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee h1:yFB2xjfswpuRh8FHagdBMKcBMltjr5u/XKzX6fkJO5E= -github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee/go.mod h1:Tylw4k1H86gbJx84i3r7qahN/mBaeMpUBvHY0Igshfw= +github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg.0.20230703223453-544e2fe28a26 h1:C7wI5fYoMlSMEGEVi/PH3Toh9TzpIWlvX9DTLIco52Y= +github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg.0.20230703223453-544e2fe28a26/go.mod h1:bZmV+V29p09ee2aWv/1WCAfHKIwWlwYmNeMspQ2CzJc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1486,8 +1550,10 @@ github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPR github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.0/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1506,42 +1572,45 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.8.0 h1:zcvBFizPbpa1q7FehvFiHbQwGzmPILebO0tyqIR5Djg= -go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 h1:ao8CJIShCaIbaMsGxy+jp2YHSudketpDgDRcbirov78= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0 h1:LrHL1A3KqIgAgi6mK7Q0aczmzU414AONAGT5xtnp+uo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0/go.mod h1:w8aZL87GMOvOBa2lU/JlVXE1q4chk/0FX+8ai4513bw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.8.0 h1:00hCSGLIxdYK/Z7r8GkaX0QIlfvgU3tmnLlQvcnix6U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.8.0/go.mod h1:twhIvtDQW2sWP1O2cT1N8nkSBgKCRZv2z6COTTBrf8Q= -go.opentelemetry.io/otel/sdk v1.8.0 h1:xwu69/fNuwbSHWe/0PGS888RmjWY181OmcXDQKu7ZQk= -go.opentelemetry.io/otel/sdk v1.8.0/go.mod h1:uPSfc+yfDH2StDM/Rm35WE8gXSNdvCg023J6HeGNO0c= -go.opentelemetry.io/otel/trace v1.8.0 h1:cSy0DF9eGI5WIfNwZ1q2iUyGj00tGzP24dE1lOlHrfY= -go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.18.0 h1:W5hyXNComRa23tGpKwG+FRAc4rfF6ZUg1JReK+QHS80= -go.opentelemetry.io/proto/otlp v0.18.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE= -go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= -go.uber.org/fx v1.18.2 h1:bUNI6oShr+OVFQeU8cDNbnN7VFsu+SsjHzUF51V/GAU= -go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= +go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= +go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY= +go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -1572,6 +1641,7 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1587,9 +1657,12 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= @@ -1600,8 +1673,9 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w= -golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1616,6 +1690,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -1627,8 +1702,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1684,10 +1759,10 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1699,9 +1774,15 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1714,8 +1795,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1743,6 +1824,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1751,12 +1833,14 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1778,43 +1862,57 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1823,21 +1921,23 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1884,6 +1984,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1893,17 +1994,24 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= +gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -1927,6 +2035,17 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -1982,10 +2101,34 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2013,10 +2156,17 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -2047,11 +2197,13 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= -gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -2066,7 +2218,6 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= @@ -2082,13 +2233,15 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= -lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= +nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= +nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= +pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/insecure/rpc_inspector/metrics_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/metrics_inspector_test.go similarity index 91% rename from insecure/rpc_inspector/metrics_inspector_test.go rename to insecure/integration/functional/test/gossipsub/rpc_inspector/metrics_inspector_test.go index 4b7147d946b..c62cfba2f8b 100644 --- a/insecure/rpc_inspector/metrics_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/metrics_inspector_test.go @@ -28,7 +28,8 @@ func TestMetricsInspector_ObserveRPC(t *testing.T) { t.Parallel() role := flow.RoleConsensus sporkID := unittest.IdentifierFixture() - spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) @@ -56,14 +57,16 @@ func TestMetricsInspector_ObserveRPC(t *testing.T) { }) metricsInspector := inspector.NewControlMsgMetricsInspector(unittest.Logger(), mockMetricsObserver, 2) corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(metricsInspector) - victimNode, _ := p2ptest.NodeFixture( + victimNode, victimIdentity := p2ptest.NodeFixture( t, sporkID, t.Name(), + idProvider, p2ptest.WithRole(role), internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), ) + idProvider.SetIdentities(flow.IdentityList{&victimIdentity, &spammer.SpammerId}) metricsInspector.Start(signalerCtx) nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) @@ -73,7 +76,7 @@ func TestMetricsInspector_ObserveRPC(t *testing.T) { ctlMsgs := spammer.GenerateCtlMessages(controlMessageCount, corruptlibp2p.WithGraft(messageCount, channels.PushBlocks.String()), corruptlibp2p.WithPrune(messageCount, channels.PushBlocks.String()), - corruptlibp2p.WithIHave(messageCount, 1000)) + corruptlibp2p.WithIHave(messageCount, 1000, channels.PushBlocks.String())) // start spamming the victim peer spammer.SpamControlMessage(t, victimNode, ctlMsgs) diff --git a/insecure/rpc_inspector/utils.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go similarity index 83% rename from insecure/rpc_inspector/utils.go rename to insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go index 02cf9492f7c..2307c57f0ab 100644 --- a/insecure/rpc_inspector/utils.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go @@ -19,10 +19,10 @@ func startNodesAndEnsureConnected(t *testing.T, ctx irrecoverable.SignalerContex // prior to the test we should ensure that spammer and victim connect. // this is vital as the spammer will circumvent the normal pubsub subscription mechanism and send iHAVE messages directly to the victim. // without a prior connection established, directly spamming pubsub messages may cause a race condition in the pubsub implementation. - p2ptest.EnsureConnected(t, ctx, nodes) - p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkID) - return unittest.ProposalFixture(), blockTopic + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkID) + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) } diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go new file mode 100644 index 00000000000..3dd873d0e7c --- /dev/null +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -0,0 +1,1250 @@ +package rpc_inspector + +import ( + "context" + "fmt" + "math/rand" + "os" + "testing" + "time" + + pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog" + mockery "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/config" + "github.com/onflow/flow-go/insecure/corruptlibp2p" + "github.com/onflow/flow-go/insecure/internal" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/inspector/validation" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" + mockp2p "github.com/onflow/flow-go/network/p2p/mock" + "github.com/onflow/flow-go/network/p2p/p2pconf" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestValidationInspector_SafetyThreshold ensures that when RPC control message count is below the configured safety threshold the control message validation inspector +// does not return any errors and validation is skipped. +func TestValidationInspector_SafetyThreshold(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus + sporkID := unittest.IdentifierFixture() + // if GRAFT/PRUNE message count is lower than safety threshold the RPC validation should pass + safetyThreshold := uint64(10) + // create our RPC validation inspector + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) + inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.GraftLimits.SafetyThreshold = safetyThreshold + inspectorConfig.PruneLimits.SafetyThreshold = safetyThreshold + + // expected log message logged when valid number GRAFT control messages spammed under safety threshold + graftExpectedMessageStr := fmt.Sprintf("control message %s inspection passed 5 is below configured safety threshold", p2pmsg.CtrlMsgGraft) + // expected log message logged when valid number PRUNE control messages spammed under safety threshold + pruneExpectedMessageStr := fmt.Sprintf("control message %s inspection passed 5 is below configured safety threshold", p2pmsg.CtrlMsgGraft) + graftInfoLogsReceived := atomic.NewInt64(0) + pruneInfoLogsReceived := atomic.NewInt64(0) + // setup logger hook, we expect info log validation is skipped + hook := zerolog.HookFunc(func(e *zerolog.Event, level zerolog.Level, message string) { + if level == zerolog.TraceLevel { + if message == graftExpectedMessageStr { + graftInfoLogsReceived.Inc() + } + + if message == pruneExpectedMessageStr { + pruneInfoLogsReceived.Inc() + } + } + }) + logger := zerolog.New(os.Stdout).Hook(hook) + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + validationInspector, err := validation.NewControlMsgValidationInspector( + logger, + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + + messageCount := 5 + controlMessageCount := int64(2) + + defer distributor.AssertNotCalled(t, "Distribute", mockery.Anything) + + validationInspector.Start(signalerCtx) + + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + spammer.Start(t) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) + // prepare to spam - generate control messages + ctlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), + corruptlibp2p.WithGraft(messageCount, channels.PushBlocks.String()), + corruptlibp2p.WithGraft(messageCount, channels.PushBlocks.String()), + corruptlibp2p.WithIHave(messageCount, 1000, channels.PushBlocks.String())) + + // start spamming the victim peer + spammer.SpamControlMessage(t, victimNode, ctlMsgs) + + // eventually we should receive 2 info logs each for GRAFT inspection and PRUNE inspection + require.Eventually(t, func() bool { + return graftInfoLogsReceived.Load() == controlMessageCount && pruneInfoLogsReceived.Load() == controlMessageCount + }, 2*time.Second, 10*time.Millisecond) +} + +// TestValidationInspector_HardThreshold_Detection ensures that when RPC control message count is above the configured hard threshold an invalid control message +// notification is disseminated with the expected error. +func TestValidationInspector_HardThreshold_Detection(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus + sporkID := unittest.IdentifierFixture() + // if GRAFT/PRUNE message count is higher than hard threshold the RPC validation should fail and expected error should be returned + hardThreshold := uint64(10) + // create our RPC validation inspector + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) + inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.GraftLimits.HardThreshold = hardThreshold + inspectorConfig.PruneLimits.HardThreshold = hardThreshold + + messageCount := 50 + controlMessageCount := int64(1) + count := atomic.NewInt64(0) + invGraftNotifCount := atomic.NewUint64(0) + invPruneNotifCount := atomic.NewUint64(0) + done := make(chan struct{}) + // ensure expected notifications are disseminated with expected error + inspectDisseminatedNotif := func(spammer *corruptlibp2p.GossipSubRouterSpammer) func(args mockery.Arguments) { + return func(args mockery.Arguments) { + count.Inc() + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) + require.Equal(t, uint64(messageCount), notification.Count) + switch notification.MsgType { + case p2pmsg.CtrlMsgGraft: + invGraftNotifCount.Inc() + case p2pmsg.CtrlMsgPrune: + invPruneNotifCount.Inc() + default: + require.Fail(t, "unexpected control message type") + } + if count.Load() == 2 { + close(done) + } + } + } + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(2, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + + validationInspector.Start(signalerCtx) + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + spammer.Start(t) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) + + // prepare to spam - generate control messages + graftCtlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(messageCount, channels.PushBlocks.String())) + pruneCtlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(messageCount, channels.PushBlocks.String())) + + // start spamming the victim peer + spammer.SpamControlMessage(t, victimNode, graftCtlMsgs) + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgs) + + unittest.RequireCloseBefore(t, done, 2*time.Second, "failed to inspect RPC messages on time") + // ensure we receive the expected number of invalid control message notifications for graft and prune control message types + require.Equal(t, uint64(1), invGraftNotifCount.Load()) + require.Equal(t, uint64(1), invPruneNotifCount.Load()) +} + +// TestValidationInspector_HardThresholdIHave_Detection ensures that when the ihave RPC control message count is above the configured hard threshold the control message validation inspector +// inspects a sample size of the ihave messages and returns the expected error when validation for a topic in that sample fails. +func TestValidationInspector_HardThresholdIHave_Detection(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus + sporkID := unittest.IdentifierFixture() + // create our RPC validation inspector + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) + inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.IHaveLimits.HardThreshold = 50 + inspectorConfig.IHaveInspectionMaxSampleSize = 100 + // set the sample size divisor to 2 which will force inspection of 50% of topic IDS + inspectorConfig.IHaveSyncInspectSampleSizePercentage = .5 + + unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", corruptlibp2p.GossipSubTopicIdFixture(), sporkID)) + messageCount := 100 + controlMessageCount := int64(1) + count := atomic.NewInt64(0) + done := make(chan struct{}) + + invIhaveNotifCount := atomic.NewUint64(0) + // ensure expected notifications are disseminated with expected error + inspectDisseminatedNotif := func(spammer *corruptlibp2p.GossipSubRouterSpammer) func(args mockery.Arguments) { + return func(args mockery.Arguments) { + count.Inc() + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) + require.Equal(t, uint64(messageCount), notification.Count) + require.True(t, channels.IsInvalidTopicErr(notification.Err)) + switch notification.MsgType { + case p2pmsg.CtrlMsgIHave: + invIhaveNotifCount.Inc() + default: + require.Fail(t, "unexpected control message type") + } + if count.Load() == 1 { + close(done) + } + } + } + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(1, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + + validationInspector.Start(signalerCtx) + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + spammer.Start(t) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) + + // add an unknown topic to each of our ihave control messages, this will ensure + // that whatever random sample of topic ids that are inspected cause validation + // to fail and a notification to be disseminated as expected. + ihaveCtlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithIHave(messageCount, 1000, unknownTopic.String())) + + // start spamming the victim peer + spammer.SpamControlMessage(t, victimNode, ihaveCtlMsgs) + + unittest.RequireCloseBefore(t, done, 2*time.Second, "failed to inspect RPC messages on time") + require.Equal(t, uint64(1), invIhaveNotifCount.Load()) +} + +// TestValidationInspector_RateLimitedPeer_Detection ensures that the control message validation inspector rate limits peers per control message type as expected and +// the expected invalid control message notification is disseminated with the expected error. +func TestValidationInspector_RateLimitedPeer_Detection(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus + sporkID := unittest.IdentifierFixture() + // create our RPC validation inspector + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) + inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig.NumberOfWorkers = 1 + + // here we set the message count to the amount of flow channels + // so that we can generate a valid ctl msg with all valid topics. + flowChannels := channels.Channels() + messageCount := flowChannels.Len() + controlMessageCount := int64(1) + + count := atomic.NewInt64(0) + invGraftNotifCount := atomic.NewUint64(0) + invPruneNotifCount := atomic.NewUint64(0) + done := make(chan struct{}) + // ensure expected notifications are disseminated with expected error + inspectDisseminatedNotif := func(spammer *corruptlibp2p.GossipSubRouterSpammer) func(args mockery.Arguments) { + return func(args mockery.Arguments) { + count.Inc() + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) + require.True(t, validation.IsErrRateLimitedControlMsg(notification.Err)) + require.Equal(t, uint64(messageCount), notification.Count) + switch notification.MsgType { + case p2pmsg.CtrlMsgGraft: + invGraftNotifCount.Inc() + case p2pmsg.CtrlMsgPrune: + invPruneNotifCount.Inc() + default: + require.Fail(t, "unexpected control message type") + } + if count.Load() == 4 { + close(done) + } + } + } + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(4, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + + validationInspector.Start(signalerCtx) + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + spammer.Start(t) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) + + // the first time we spam this message it will be processed completely so we need to ensure + // all topics are valid and no duplicates exists. + validCtlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), func(message *pb.ControlMessage) { + grafts := make([]*pb.ControlGraft, messageCount) + prunes := make([]*pb.ControlPrune, messageCount) + ihaves := make([]*pb.ControlIHave, messageCount) + for i := 0; i < messageCount; i++ { + topic := fmt.Sprintf("%s/%s", flowChannels[i].String(), sporkID) + grafts[i] = &pb.ControlGraft{TopicID: &topic} + prunes[i] = &pb.ControlPrune{TopicID: &topic} + ihaves[i] = &pb.ControlIHave{TopicID: &topic, MessageIDs: corruptlibp2p.GossipSubMessageIdsFixture(messageCount)} + } + message.Graft = grafts + message.Prune = prunes + }) + + // start spamming the victim peer + for i := 0; i < 3; i++ { + spammer.SpamControlMessage(t, victimNode, validCtlMsgs) + } + + unittest.RequireCloseBefore(t, done, 2*time.Second, "failed to inspect RPC messages on time") + // ensure we receive the expected number of invalid control message notifications for graft and prune control message types + require.Equal(t, uint64(2), invGraftNotifCount.Load()) + require.Equal(t, uint64(2), invPruneNotifCount.Load()) +} + +// TestValidationInspector_InvalidTopicId_Detection ensures that when an RPC control message contains an invalid topic ID an invalid control message +// notification is disseminated with the expected error. +// An invalid topic ID could have any of the following properties: +// - unknown topic: the topic is not a known Flow topic +// - malformed topic: topic is malformed in some way +// - invalid spork ID: spork ID prepended to topic and current spork ID do not match +func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus + sporkID := unittest.IdentifierFixture() + // if GRAFT/PRUNE message count is higher than hard threshold the RPC validation should fail and expected error should be returned + // create our RPC validation inspector + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) + inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + // set safety thresholds to 0 to force inspector to validate all control messages + inspectorConfig.PruneLimits.SafetyThreshold = 0 + inspectorConfig.GraftLimits.SafetyThreshold = 0 + // set hard threshold to 0 so that in the case of invalid cluster ID + // we force the inspector to return an error + inspectorConfig.ClusterPrefixHardThreshold = 0 + inspectorConfig.IHaveLimits.SafetyThreshold = 0 + inspectorConfig.IHaveLimits.HardThreshold = 50 + inspectorConfig.IHaveAsyncInspectSampleSizePercentage = .5 + inspectorConfig.IHaveInspectionMaxSampleSize = 100 + ihaveMessageCount := 100 + inspectorConfig.NumberOfWorkers = 1 + + // SafetyThreshold < messageCount < HardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked + // restricting the message count to 1 allows us to only aggregate a single error when the error is logged in the inspector. + messageCount := inspectorConfig.GraftLimits.SafetyThreshold + 1 + controlMessageCount := int64(1) + + count := atomic.NewUint64(0) + invGraftNotifCount := atomic.NewUint64(0) + invPruneNotifCount := atomic.NewUint64(0) + invIHaveNotifCount := atomic.NewUint64(0) + done := make(chan struct{}) + expectedNumOfTotalNotif := 9 + // ensure expected notifications are disseminated with expected error + inspectDisseminatedNotif := func(spammer *corruptlibp2p.GossipSubRouterSpammer) func(args mockery.Arguments) { + return func(args mockery.Arguments) { + count.Inc() + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) + require.True(t, channels.IsInvalidTopicErr(notification.Err)) + switch notification.MsgType { + case p2pmsg.CtrlMsgGraft: + invGraftNotifCount.Inc() + require.Equal(t, messageCount, notification.Count) + case p2pmsg.CtrlMsgPrune: + invPruneNotifCount.Inc() + require.Equal(t, messageCount, notification.Count) + case p2pmsg.CtrlMsgIHave: + require.Equal(t, uint64(ihaveMessageCount), notification.Count) + invIHaveNotifCount.Inc() + default: + require.Fail(t, "unexpected control message type") + } + if count.Load() == uint64(expectedNumOfTotalNotif) { + close(done) + } + } + } + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + + // create unknown topic + unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", corruptlibp2p.GossipSubTopicIdFixture(), sporkID)) + // create malformed topic + malformedTopic := channels.Topic("!@#$%^&**((") + // a topics spork ID is considered invalid if it does not match the current spork ID + invalidSporkIDTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, unittest.IdentifierFixture())) + + validationInspector.Start(signalerCtx) + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + spammer.Start(t) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) + + // prepare to spam - generate control messages + graftCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), unknownTopic.String())) + graftCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), malformedTopic.String())) + graftCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), invalidSporkIDTopic.String())) + + pruneCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(int(messageCount), unknownTopic.String())) + pruneCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(int(messageCount), malformedTopic.String())) + pruneCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(int(messageCount), invalidSporkIDTopic.String())) + + iHaveCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithIHave(ihaveMessageCount, 1000, unknownTopic.String())) + iHaveCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithIHave(ihaveMessageCount, 1000, malformedTopic.String())) + iHaveCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithIHave(ihaveMessageCount, 1000, invalidSporkIDTopic.String())) + + // spam the victim peer with invalid graft messages + spammer.SpamControlMessage(t, victimNode, graftCtlMsgsWithUnknownTopic) + spammer.SpamControlMessage(t, victimNode, graftCtlMsgsWithMalformedTopic) + spammer.SpamControlMessage(t, victimNode, graftCtlMsgsInvalidSporkIDTopic) + + // spam the victim peer with invalid prune messages + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsWithUnknownTopic) + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsWithMalformedTopic) + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsInvalidSporkIDTopic) + + // spam the victim peer with invalid ihave messages + spammer.SpamControlMessage(t, victimNode, iHaveCtlMsgsWithUnknownTopic) + spammer.SpamControlMessage(t, victimNode, iHaveCtlMsgsWithMalformedTopic) + spammer.SpamControlMessage(t, victimNode, iHaveCtlMsgsInvalidSporkIDTopic) + + unittest.RequireCloseBefore(t, done, 5*time.Second, "failed to inspect RPC messages on time") + + // ensure we receive the expected number of invalid control message notifications for graft and prune control message types + // we send 3 messages with 3 diff invalid topics + require.Equal(t, uint64(3), invGraftNotifCount.Load()) + require.Equal(t, uint64(3), invPruneNotifCount.Load()) + require.Equal(t, uint64(3), invIHaveNotifCount.Load()) +} + +// TestValidationInspector_DuplicateTopicId_Detection ensures that when an RPC control message contains a duplicate topic ID an invalid control message +// notification is disseminated with the expected error. +func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus + sporkID := unittest.IdentifierFixture() + // if GRAFT/PRUNE message count is higher than hard threshold the RPC validation should fail and expected error should be returned + // create our RPC validation inspector + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) + inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + // set safety thresholds to 0 to force inspector to validate all control messages + inspectorConfig.PruneLimits.SafetyThreshold = 0 + inspectorConfig.GraftLimits.SafetyThreshold = 0 + // set hard threshold to 0 so that in the case of invalid cluster ID + // we force the inspector to return an error + inspectorConfig.ClusterPrefixHardThreshold = 0 + inspectorConfig.NumberOfWorkers = 1 + + // SafetyThreshold < messageCount < HardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked + // restricting the message count to 1 allows us to only aggregate a single error when the error is logged in the inspector. + messageCount := inspectorConfig.GraftLimits.SafetyThreshold + 3 + controlMessageCount := int64(1) + + count := atomic.NewInt64(0) + done := make(chan struct{}) + expectedNumOfTotalNotif := 2 + invGraftNotifCount := atomic.NewUint64(0) + invPruneNotifCount := atomic.NewUint64(0) + inspectDisseminatedNotif := func(spammer *corruptlibp2p.GossipSubRouterSpammer) func(args mockery.Arguments) { + return func(args mockery.Arguments) { + count.Inc() + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) + require.True(t, validation.IsErrDuplicateTopic(notification.Err)) + require.Equal(t, messageCount, notification.Count) + switch notification.MsgType { + case p2pmsg.CtrlMsgGraft: + invGraftNotifCount.Inc() + case p2pmsg.CtrlMsgPrune: + invPruneNotifCount.Inc() + default: + require.Fail(t, "unexpected control message type") + } + if count.Load() == int64(expectedNumOfTotalNotif) { + close(done) + } + } + } + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + + // a topics spork ID is considered invalid if it does not match the current spork ID + duplicateTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, sporkID)) + + validationInspector.Start(signalerCtx) + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + spammer.Start(t) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) + + // prepare to spam - generate control messages + graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), duplicateTopic.String())) + + pruneCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(int(messageCount), duplicateTopic.String())) + + // start spamming the victim peer + spammer.SpamControlMessage(t, victimNode, graftCtlMsgsDuplicateTopic) + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsDuplicateTopic) + + unittest.RequireCloseBefore(t, done, 5*time.Second, "failed to inspect RPC messages on time") + // ensure we receive the expected number of invalid control message notifications for graft and prune control message types + require.Equal(t, uint64(1), invGraftNotifCount.Load()) + require.Equal(t, uint64(1), invPruneNotifCount.Load()) +} + +// TestValidationInspector_UnknownClusterId_Detection ensures that when an RPC control message contains a topic with an unknown cluster ID an invalid control message +// notification is disseminated with the expected error. +func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus + sporkID := unittest.IdentifierFixture() + // if GRAFT/PRUNE message count is higher than hard threshold the RPC validation should fail and expected error should be returned + // create our RPC validation inspector + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) + inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + // set safety thresholds to 0 to force inspector to validate all control messages + inspectorConfig.PruneLimits.SafetyThreshold = 0 + inspectorConfig.GraftLimits.SafetyThreshold = 0 + // set hard threshold to 0 so that in the case of invalid cluster ID + // we force the inspector to return an error + inspectorConfig.ClusterPrefixHardThreshold = 0 + inspectorConfig.NumberOfWorkers = 1 + + // SafetyThreshold < messageCount < HardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked + // restricting the message count to 1 allows us to only aggregate a single error when the error is logged in the inspector. + messageCount := inspectorConfig.GraftLimits.SafetyThreshold + 1 + controlMessageCount := int64(1) + + count := atomic.NewInt64(0) + done := make(chan struct{}) + expectedNumOfTotalNotif := 2 + invGraftNotifCount := atomic.NewUint64(0) + invPruneNotifCount := atomic.NewUint64(0) + inspectDisseminatedNotif := func(spammer *corruptlibp2p.GossipSubRouterSpammer) func(args mockery.Arguments) { + return func(args mockery.Arguments) { + count.Inc() + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) + require.True(t, channels.IsUnknownClusterIDErr(notification.Err)) + require.Equal(t, messageCount, notification.Count) + switch notification.MsgType { + case p2pmsg.CtrlMsgGraft: + invGraftNotifCount.Inc() + case p2pmsg.CtrlMsgPrune: + invPruneNotifCount.Inc() + default: + require.Fail(t, "unexpected control message type") + } + if count.Load() == int64(expectedNumOfTotalNotif) { + close(done) + } + } + } + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Times(3) + + // setup cluster prefixed topic with an invalid cluster ID + unknownClusterID := channels.Topic(channels.SyncCluster("unknown-cluster-ID")) + // consume cluster ID update so that active cluster IDs set + validationInspector.ActiveClustersChanged(flow.ChainIDList{"known-cluster-id"}) + + validationInspector.Start(signalerCtx) + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + spammer.Start(t) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) + + // prepare to spam - generate control messages + graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), unknownClusterID.String())) + pruneCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(int(messageCount), unknownClusterID.String())) + + // start spamming the victim peer + spammer.SpamControlMessage(t, victimNode, graftCtlMsgsDuplicateTopic) + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsDuplicateTopic) + + unittest.RequireCloseBefore(t, done, 5*time.Second, "failed to inspect RPC messages on time") + // ensure we receive the expected number of invalid control message notifications for graft and prune control message types + require.Equal(t, uint64(1), invGraftNotifCount.Load()) + require.Equal(t, uint64(1), invPruneNotifCount.Load()) +} + +// TestValidationInspector_ActiveClusterIdsNotSet_Graft_Detection ensures that an error is returned only after the cluster prefixed topics received for a peer exceed the configured +// cluster prefix hard threshold when the active cluster IDs not set and an invalid control message notification is disseminated with the expected error. +// This test involves Graft control messages. +func TestValidationInspector_ActiveClusterIdsNotSet_Graft_Detection(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus + sporkID := unittest.IdentifierFixture() + // if GRAFT/PRUNE message count is higher than hard threshold the RPC validation should fail and expected error should be returned + // create our RPC validation inspector + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) + inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig.GraftLimits.SafetyThreshold = 0 + inspectorConfig.ClusterPrefixHardThreshold = 5 + inspectorConfig.NumberOfWorkers = 1 + controlMessageCount := int64(10) + + count := atomic.NewInt64(0) + done := make(chan struct{}) + expectedNumOfTotalNotif := 5 + invGraftNotifCount := atomic.NewUint64(0) + inspectDisseminatedNotif := func(spammer *corruptlibp2p.GossipSubRouterSpammer) func(args mockery.Arguments) { + return func(args mockery.Arguments) { + count.Inc() + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.True(t, validation.IsErrActiveClusterIDsNotSet(notification.Err)) + require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) + switch notification.MsgType { + case p2pmsg.CtrlMsgGraft: + invGraftNotifCount.Inc() + default: + require.Fail(t, "unexpected control message type") + } + require.Equal(t, uint64(1), notification.Count) + if count.Load() == int64(expectedNumOfTotalNotif) { + close(done) + } + } + } + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Times(int(controlMessageCount + 1)) + + // we deliberately avoid setting the cluster IDs so that we eventually receive errors after we have exceeded the allowed cluster + // prefixed hard threshold + validationInspector.Start(signalerCtx) + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + spammer.Start(t) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) + // generate multiple control messages with GRAFT's for randomly generated + // cluster prefixed channels, this ensures we do not encounter duplicate topic ID errors + ctlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), + corruptlibp2p.WithGraft(1, randomClusterPrefixedTopic().String()), + ) + // start spamming the victim peer + spammer.SpamControlMessage(t, victimNode, ctlMsgs) + + unittest.RequireCloseBefore(t, done, 5*time.Second, "failed to inspect RPC messages on time") + // ensure we receive the expected number of invalid control message notifications for graft and prune control message types + require.Equal(t, uint64(5), invGraftNotifCount.Load()) +} + +// TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection ensures that an error is returned only after the cluster prefixed topics received for a peer exceed the configured +// cluster prefix hard threshold when the active cluster IDs not set and an invalid control message notification is disseminated with the expected error. +// This test involves Prune control messages. +func TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus + sporkID := unittest.IdentifierFixture() + // if GRAFT/PRUNE message count is higher than hard threshold the RPC validation should fail and expected error should be returned + // create our RPC validation inspector + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) + inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + inspectorConfig.PruneLimits.SafetyThreshold = 0 + inspectorConfig.ClusterPrefixHardThreshold = 5 + inspectorConfig.NumberOfWorkers = 1 + controlMessageCount := int64(10) + + count := atomic.NewInt64(0) + done := make(chan struct{}) + expectedNumOfTotalNotif := 5 + invPruneNotifCount := atomic.NewUint64(0) + inspectDisseminatedNotif := func(spammer *corruptlibp2p.GossipSubRouterSpammer) func(args mockery.Arguments) { + return func(args mockery.Arguments) { + count.Inc() + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.True(t, validation.IsErrActiveClusterIDsNotSet(notification.Err)) + require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) + switch notification.MsgType { + case p2pmsg.CtrlMsgPrune: + invPruneNotifCount.Inc() + default: + require.Fail(t, "unexpected control message type") + } + require.Equal(t, uint64(1), notification.Count) + if count.Load() == int64(expectedNumOfTotalNotif) { + close(done) + } + } + } + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Times(int(controlMessageCount + 1)) + + // we deliberately avoid setting the cluster IDs so that we eventually receive errors after we have exceeded the allowed cluster + // prefixed hard threshold + validationInspector.Start(signalerCtx) + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + spammer.Start(t) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) + // generate multiple control messages with GRAFT's for randomly generated + // cluster prefixed channels, this ensures we do not encounter duplicate topic ID errors + ctlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), + corruptlibp2p.WithPrune(1, randomClusterPrefixedTopic().String()), + ) + // start spamming the victim peer + spammer.SpamControlMessage(t, victimNode, ctlMsgs) + + unittest.RequireCloseBefore(t, done, 5*time.Second, "failed to inspect RPC messages on time") + // ensure we receive the expected number of invalid control message notifications for graft and prune control message types + require.Equal(t, uint64(5), invPruneNotifCount.Load()) +} + +// TestValidationInspector_UnstakedNode_Detection ensures that RPC control message inspector disseminates an invalid control message notification when an unstaked peer +// sends a control message for a cluster prefixed topic. +func TestValidationInspector_UnstakedNode_Detection(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus + sporkID := unittest.IdentifierFixture() + // if GRAFT/PRUNE message count is higher than hard threshold the RPC validation should fail and expected error should be returned + // create our RPC validation inspector + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) + inspectorConfig := flowConfig.NetworkConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + // set safety thresholds to 0 to force inspector to validate all control messages + inspectorConfig.PruneLimits.SafetyThreshold = 0 + inspectorConfig.GraftLimits.SafetyThreshold = 0 + // set hard threshold to 0 so that in the case of invalid cluster ID + // we force the inspector to return an error + inspectorConfig.ClusterPrefixHardThreshold = 0 + inspectorConfig.NumberOfWorkers = 1 + + // SafetyThreshold < messageCount < HardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked + // restricting the message count to 1 allows us to only aggregate a single error when the error is logged in the inspector. + messageCount := inspectorConfig.GraftLimits.SafetyThreshold + 1 + controlMessageCount := int64(1) + + count := atomic.NewInt64(0) + done := make(chan struct{}) + expectedNumOfTotalNotif := 2 + invGraftNotifCount := atomic.NewUint64(0) + invPruneNotifCount := atomic.NewUint64(0) + inspectDisseminatedNotif := func(spammer *corruptlibp2p.GossipSubRouterSpammer) func(args mockery.Arguments) { + return func(args mockery.Arguments) { + count.Inc() + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) + require.True(t, validation.IsErrUnstakedPeer(notification.Err)) + require.Equal(t, messageCount, notification.Count) + switch notification.MsgType { + case p2pmsg.CtrlMsgGraft: + invGraftNotifCount.Inc() + case p2pmsg.CtrlMsgPrune: + invPruneNotifCount.Inc() + default: + require.Fail(t, "unexpected control message type") + } + if count.Load() == int64(expectedNumOfTotalNotif) { + close(done) + } + } + } + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + withExpectedNotificationDissemination(expectedNumOfTotalNotif, inspectDisseminatedNotif)(distributor, spammer) + validationInspector, err := validation.NewControlMsgValidationInspector( + unittest.Logger(), + sporkID, + &inspectorConfig, + distributor, + metrics.NewNoopCollector(), + metrics.NewNoopCollector(), + idProvider, + metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(nil, false).Times(3) + + // setup cluster prefixed topic with an invalid cluster ID + clusterID := flow.ChainID("known-cluster-id") + clusterIDTopic := channels.Topic(channels.SyncCluster(clusterID)) + // consume cluster ID update so that active cluster IDs set + validationInspector.ActiveClustersChanged(flow.ChainIDList{clusterID}) + + validationInspector.Start(signalerCtx) + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + spammer.Start(t) + defer stopNodesAndInspector(t, cancel, nodes, validationInspector) + + // prepare to spam - generate control messages + graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), clusterIDTopic.String())) + pruneCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(int(messageCount), clusterIDTopic.String())) + + // start spamming the victim peer + spammer.SpamControlMessage(t, victimNode, graftCtlMsgsDuplicateTopic) + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsDuplicateTopic) + + unittest.RequireCloseBefore(t, done, 5*time.Second, "failed to inspect RPC messages on time") + // ensure we receive the expected number of invalid control message notifications for graft and prune control message types + require.Equal(t, uint64(1), invGraftNotifCount.Load()) + require.Equal(t, uint64(1), invPruneNotifCount.Load()) +} + +func randomClusterPrefixedTopic() channels.Topic { + return channels.Topic(channels.SyncCluster(flow.ChainID(fmt.Sprintf("%d", rand.Uint64())))) +} + +type onNotificationDissemination func(spammer *corruptlibp2p.GossipSubRouterSpammer) func(args mockery.Arguments) +type mockDistributorOption func(*mockp2p.GossipSubInspectorNotificationDistributor, *corruptlibp2p.GossipSubRouterSpammer) + +func withExpectedNotificationDissemination(expectedNumOfTotalNotif int, f onNotificationDissemination) mockDistributorOption { + return func(distributor *mockp2p.GossipSubInspectorNotificationDistributor, spammer *corruptlibp2p.GossipSubRouterSpammer) { + distributor. + On("Distribute", mockery.Anything). + Times(expectedNumOfTotalNotif). + Run(f(spammer)). + Return(nil) + } +} + +// setupTest sets up common components of RPC inspector test. +func setupTest(t *testing.T, logger zerolog.Logger, role flow.Role, sporkID flow.Identifier, inspectorConfig *p2pconf.GossipSubRPCValidationInspectorConfigs, mockDistributorOpts ...mockDistributorOption) (*irrecoverable.MockSignalerContext, context.CancelFunc, *corruptlibp2p.GossipSubRouterSpammer, p2p.LibP2PNode, flow.Identity, *mockp2p.GossipSubInspectorNotificationDistributor, *validation.ControlMsgValidationInspector, *mock.IdentityProvider) { + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) + mockDistributorReadyDoneAware(distributor) + for _, mockDistributorOpt := range mockDistributorOpts { + mockDistributorOpt(distributor, spammer) + } + validationInspector, err := validation.NewControlMsgValidationInspector(logger, sporkID, inspectorConfig, distributor, metrics.NewNoopCollector(), metrics.NewNoopCollector(), idProvider, metrics.NewNoopCollector()) + require.NoError(t, err) + corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(role), + internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), + corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), + ) + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + + return signalerCtx, cancel, spammer, victimNode, victimIdentity, distributor, validationInspector, idProvider +} + +// TestGossipSubSpamMitigationIntegration tests that the spam mitigation feature of GossipSub is working as expected. +// The test puts toghether the spam detection (through the GossipSubInspector) and the spam mitigation (through the +// scoring system) and ensures that the mitigation is triggered when the spam detection detects spam. +// The test scenario involves a spammer node that sends a large number of control messages to a victim node. +// The victim node is configured to use the GossipSubInspector to detect spam and the scoring system to mitigate spam. +// The test ensures that the victim node is disconnected from the spammer node on the GossipSub mesh after the spam detection is triggered. +func TestGossipSubSpamMitigationIntegration(t *testing.T) { + t.Parallel() + idProvider := mock.NewIdentityProvider(t) + sporkID := unittest.IdentifierFixture() + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, flow.RoleConsensus, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + victimNode, victimId := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithRole(flow.RoleConsensus), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), + ) + + ids := flow.IdentityList{&victimId, &spammer.SpammerId} + idProvider.On("ByPeerID", mockery.Anything).Return( + func(peerId peer.ID) *flow.Identity { + switch peerId { + case victimNode.Host().ID(): + return &victimId + case spammer.SpammerNode.Host().ID(): + return &spammer.SpammerId + default: + return nil + } + + }, func(peerId peer.ID) bool { + switch peerId { + case victimNode.Host().ID(): + fallthrough + case spammer.SpammerNode.Host().ID(): + return true + default: + return false + } + }) + + spamRpcCount := 10 // total number of individual rpc messages to send + spamCtrlMsgCount := int64(10) // total number of control messages to send on each RPC + + // unknownTopic is an unknown topic to the victim node but shaped like a valid topic (i.e., it has the correct prefix and spork ID). + unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", corruptlibp2p.GossipSubTopicIdFixture(), sporkID)) + + // malformedTopic is a topic that is not shaped like a valid topic (i.e., it does not have the correct prefix and spork ID). + malformedTopic := channels.Topic("!@#$%^&**((") + + // invalidSporkIDTopic is a topic that has a valid prefix but an invalid spork ID (i.e., not the current spork ID). + invalidSporkIDTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, unittest.IdentifierFixture())) + + // duplicateTopic is a valid topic that is used to send duplicate spam messages. + duplicateTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, sporkID)) + + // starting the nodes. + nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + spammer.Start(t) + + // wait for the nodes to discover each other + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + + // as nodes started fresh and no spamming has happened yet, the nodes should be able to exchange messages on the topic. + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkID) + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + // prepares spam graft and prune messages with different strategies. + graftCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), corruptlibp2p.WithGraft(spamRpcCount, unknownTopic.String())) + graftCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), corruptlibp2p.WithGraft(spamRpcCount, malformedTopic.String())) + graftCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), corruptlibp2p.WithGraft(spamRpcCount, invalidSporkIDTopic.String())) + graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), corruptlibp2p.WithGraft(3, duplicateTopic.String())) + + pruneCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), corruptlibp2p.WithPrune(spamRpcCount, unknownTopic.String())) + pruneCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), corruptlibp2p.WithPrune(spamRpcCount, malformedTopic.String())) + pruneCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), corruptlibp2p.WithGraft(spamRpcCount, invalidSporkIDTopic.String())) + pruneCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), corruptlibp2p.WithPrune(3, duplicateTopic.String())) + + // start spamming the victim peer + spammer.SpamControlMessage(t, victimNode, graftCtlMsgsWithUnknownTopic) + spammer.SpamControlMessage(t, victimNode, graftCtlMsgsWithMalformedTopic) + spammer.SpamControlMessage(t, victimNode, graftCtlMsgsInvalidSporkIDTopic) + spammer.SpamControlMessage(t, victimNode, graftCtlMsgsDuplicateTopic) + + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsWithUnknownTopic) + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsWithMalformedTopic) + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsInvalidSporkIDTopic) + spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsDuplicateTopic) + + // wait for three GossipSub heartbeat intervals to ensure that the victim node has penalized the spammer node. + time.Sleep(3 * time.Second) + + // now we expect the detection and mitigation to kick in and the victim node to disconnect from the spammer node. + // so the spammer and victim nodes should not be able to exchange messages on the topic. + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{victimNode}, []p2p.LibP2PNode{spammer.SpammerNode}, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} + +// mockDistributorReadyDoneAware mocks the Ready and Done methods of the distributor to return a channel that is already closed, +// so that the distributor is considered ready and done when the test needs. +func mockDistributorReadyDoneAware(d *mockp2p.GossipSubInspectorNotificationDistributor) { + d.On("Start", mockery.Anything).Return().Maybe() + d.On("Ready").Return(func() <-chan struct{} { + ch := make(chan struct{}) + close(ch) + return ch + }()).Maybe() + d.On("Done").Return(func() <-chan struct{} { + ch := make(chan struct{}) + close(ch) + return ch + }()).Maybe() +} diff --git a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go new file mode 100644 index 00000000000..3342b3ac4dc --- /dev/null +++ b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go @@ -0,0 +1,347 @@ +package scoring + +import ( + "context" + "fmt" + "testing" + "time" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" + corrupt "github.com/yhassanzadeh13/go-libp2p-pubsub" + + "github.com/onflow/flow-go/insecure/corruptlibp2p" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/scoring" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestGossipSubIHaveBrokenPromises_Below_Threshold tests that as long as the spammer stays below the ihave spam thresholds, it is not caught and +// penalized by the victim node. +// The thresholds are: +// Maximum messages that include iHave per heartbeat is: 10 (gossipsub parameter). +// Threshold for broken promises of iHave per heartbeat is: 10 (Flow-specific) parameter. It means that GossipSub samples one iHave id out of the +// entire RPC and if that iHave id is not eventually delivered within 3 seconds (gossipsub parameter), then the promise is considered broken. We set +// this threshold to 10 meaning that the first 10 broken promises are ignored. This is to allow for some network churn. +// Also, per hearbeat (i.e., decay interval), the spammer is allowed to send at most 5000 ihave messages (gossip sub parameter) on aggregate, and +// excess messages are dropped (without being counted as broken promises). +func TestGossipSubIHaveBrokenPromises_Below_Threshold(t *testing.T) { + t.Parallel() + + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + + receivedIWants := unittest.NewProtectedMap[string, struct{}]() + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammerWithRpcInspector(t, sporkId, role, idProvider, func(id peer.ID, rpc *corrupt.RPC) error { + // override rpc inspector of the spammer node to keep track of the iwants it has received. + if rpc.RPC.Control == nil || rpc.RPC.Control.Iwant == nil { + return nil + } + for _, iwant := range rpc.RPC.Control.Iwant { + for _, msgId := range iwant.MessageIDs { + receivedIWants.Add(msgId, struct{}{}) + } + } + return nil + }) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + blockTopicOverrideParams := scoring.DefaultTopicScoreParams() + blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + // we disable invalid message delivery parameters, as the way we implement spammer, when it spams ihave messages, it does not sign them. Hence, without decaying the invalid message deliveries, + // the node would be penalized for invalid message delivery way sooner than it can mount an ihave broken-promises spam attack. + blockTopicOverrideParams.InvalidMessageDeliveriesWeight = 0.0 + blockTopicOverrideParams.InvalidMessageDeliveriesDecay = 0.0 + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ + TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ + blockTopic: blockTopicOverrideParams, + }, + DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + }), + ) + + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + ids := flow.IdentityList{&spammer.SpammerId, &victimIdentity} + nodes := []p2p.LibP2PNode{spammer.SpammerNode, victimNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // checks end-to-end message delivery works on GossipSub + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + // creates 10 RPCs each with 10 iHave messages, each iHave message has 50 message ids, hence overall, we have 5000 iHave message ids. + spamIHaveBrokenPromise(t, spammer, blockTopic.String(), receivedIWants, victimNode) + + // wait till victim counts the spam iHaves as broken promises (one per RPC for a total of 10). + initialBehavioralPenalty := float64(0) // keeps track of the initial behavioral penalty of the spammer node for decay testing. + require.Eventually(t, func() bool { + behavioralPenalty, ok := victimNode.PeerScoreExposer().GetBehaviourPenalty(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + if behavioralPenalty < 9 { // ideally it must be 10 (one per RPC), but we give it a buffer of 1 to account for decays and floating point errors. + return false + } + + initialBehavioralPenalty = behavioralPenalty + return true + // Note: we have to wait at least 3 seconds for an iHave to be considered as broken promise (gossipsub parameters), we set it to 10 + // seconds to be on the safe side. + }, 10*time.Second, 100*time.Millisecond) + + spammerScore, ok := victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + require.True(t, ok, "sanity check failed, we should have a score for the spammer node") + // since spammer is not yet considered to be penalized, its score must be greater than the gossipsub health thresholds. + require.Greaterf(t, spammerScore, scoring.DefaultGossipThreshold, "sanity check failed, the score of the spammer node must be greater than gossip threshold: %f, actual: %f", scoring.DefaultGossipThreshold, spammerScore) + require.Greaterf(t, spammerScore, scoring.DefaultPublishThreshold, "sanity check failed, the score of the spammer node must be greater than publish threshold: %f, actual: %f", scoring.DefaultPublishThreshold, spammerScore) + require.Greaterf(t, spammerScore, scoring.DefaultGraylistThreshold, "sanity check failed, the score of the spammer node must be greater than graylist threshold: %f, actual: %f", scoring.DefaultGraylistThreshold, spammerScore) + + // eventually, after a heartbeat the spammer behavioral counter must be decayed + require.Eventually(t, func() bool { + behavioralPenalty, ok := victimNode.PeerScoreExposer().GetBehaviourPenalty(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + if behavioralPenalty >= initialBehavioralPenalty { // after a heartbeat the spammer behavioral counter must be decayed. + return false + } + + return true + }, 2*time.Second, 100*time.Millisecond, "sanity check failed, the spammer behavioral counter must be decayed after a heartbeat") + + // since spammer stays below the threshold, it should be able to exchange messages with the victim node over pubsub. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} + +// TestGossipSubIHaveBrokenPromises_Above_Threshold tests that a continuous stream of spam iHave broken promises will +// eventually cause the spammer node to be graylisted (i.e., no incoming RPCs from the spammer node will be accepted, and +// no outgoing RPCs to the spammer node will be sent). +// The test performs 3 rounds of attacks: each round with 10 RPCs, each RPC with 10 iHave messages, each iHave message with 50 message ids, hence overall, we have 5000 iHave message ids. +// Note that based on GossipSub parameters 5000 iHave is the most one can send within one decay interval. +// First round of attack makes spammers broken promises still below the threshold of 10 RPCs (broken promises are counted per RPC), hence no degradation of the spammers score. +// Second round of attack makes spammers broken promises above the threshold of 10 RPCs, hence a degradation of the spammers score. +// Third round of attack makes spammers broken promises to around 20 RPCs above the threshold, which causes the graylisting of the spammer node. +func TestGossipSubIHaveBrokenPromises_Above_Threshold(t *testing.T) { + t.Parallel() + + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + + receivedIWants := unittest.NewProtectedMap[string, struct{}]() + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammerWithRpcInspector(t, sporkId, role, idProvider, func(id peer.ID, rpc *corrupt.RPC) error { + // override rpc inspector of the spammer node to keep track of the iwants it has received. + if rpc.RPC.Control == nil || rpc.RPC.Control.Iwant == nil { + return nil + } + for _, iwant := range rpc.RPC.Control.Iwant { + for _, msgId := range iwant.MessageIDs { + receivedIWants.Add(msgId, struct{}{}) + } + } + return nil + }) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + blockTopicOverrideParams := scoring.DefaultTopicScoreParams() + blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + // we disable invalid message delivery parameters, as the way we implement spammer, when it spams ihave messages, it does not sign them. Hence, without decaying the invalid message deliveries, + // the node would be penalized for invalid message delivery way sooner than it can mount an ihave broken-promises spam attack. + blockTopicOverrideParams.InvalidMessageDeliveriesWeight = 0.0 + blockTopicOverrideParams.InvalidMessageDeliveriesDecay = 0.0 + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ + TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ + blockTopic: blockTopicOverrideParams, + }, + DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + }), + ) + + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + ids := flow.IdentityList{&spammer.SpammerId, &victimIdentity} + nodes := []p2p.LibP2PNode{spammer.SpammerNode, victimNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // checks end-to-end message delivery works on GossipSub + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + initScore, ok := victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + require.True(t, ok, "score for spammer node must be present") + + // FIRST ROUND OF ATTACK: spammer sends 10 RPCs to the victim node, each containing 500 iHave messages. + spamIHaveBrokenPromise(t, spammer, blockTopic.String(), receivedIWants, victimNode) + + // wait till victim counts the spam iHaves as broken promises for the second round of attack (one per RPC for a total of 10). + require.Eventually(t, func() bool { + behavioralPenalty, ok := victimNode.PeerScoreExposer().GetBehaviourPenalty(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + // ideally it must be 10 (one per RPC), but we give it a buffer of 1 to account for decays and floating point errors. + // note that we intentionally override the decay speed to be 60-times faster in this test. + if behavioralPenalty < 9 { + return false + } + + return true + // Note: we have to wait at least 3 seconds for an iHave to be considered as broken promise (gossipsub parameters), we set it to 10 + // seconds to be on the safe side. + }, 10*time.Second, 100*time.Millisecond) + + scoreAfterFirstRound, ok := victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + require.True(t, ok, "score for spammer node must be present") + // spammer score after first round must not be decreased severely, we account for 10% drop due to under-performing + // (on sending fresh new messages since that is not part of the test). + require.Greater(t, scoreAfterFirstRound, 0.9*initScore) + + // SECOND ROUND OF ATTACK: spammer sends 10 RPCs to the victim node, each containing 500 iHave messages. + spamIHaveBrokenPromise(t, spammer, blockTopic.String(), receivedIWants, victimNode) + + // wait till victim counts the spam iHaves as broken promises for the second round of attack (one per RPC for a total of 10). + require.Eventually(t, func() bool { + behavioralPenalty, ok := victimNode.PeerScoreExposer().GetBehaviourPenalty(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + + // ideally we should have 20 (10 from the first round, 10 from the second round), but we give it a buffer of 2 to account for decays and floating point errors. + // note that we intentionally override the decay speed to be 60-times faster in this test. + if behavioralPenalty < 18 { + return false + } + + return true + // Note: we have to wait at least 3 seconds for an iHave to be considered as broken promise (gossipsub parameters), we set it to 10 + // seconds to be on the safe side. + }, 10*time.Second, 100*time.Millisecond) + + spammerScore, ok := victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + require.True(t, ok, "sanity check failed, we should have a score for the spammer node") + // with the second round of the attack, the spammer is about 10 broken promises above the threshold (total ~20 broken promises, but the first 10 are not counted). + // we expect the score to be dropped to initScore - 10 * 10 * 0.01 * scoring.MaxAppSpecificReward, however, instead of 10, we consider 8 about the threshold, to account for decays. + require.LessOrEqual(t, spammerScore, initScore-8*8*0.01*scoring.MaxAppSpecificReward, "sanity check failed, the score of the spammer node must be less than the initial score minus 8 * 8 * 0.01 * scoring.MaxAppSpecificReward: %f, actual: %f", initScore-10*10*10-2*scoring.MaxAppSpecificReward, spammerScore) + require.Greaterf(t, spammerScore, scoring.DefaultGossipThreshold, "sanity check failed, the score of the spammer node must be greater than gossip threshold: %f, actual: %f", scoring.DefaultGossipThreshold, spammerScore) + require.Greaterf(t, spammerScore, scoring.DefaultPublishThreshold, "sanity check failed, the score of the spammer node must be greater than publish threshold: %f, actual: %f", scoring.DefaultPublishThreshold, spammerScore) + require.Greaterf(t, spammerScore, scoring.DefaultGraylistThreshold, "sanity check failed, the score of the spammer node must be greater than graylist threshold: %f, actual: %f", scoring.DefaultGraylistThreshold, spammerScore) + + // since the spammer score is above the gossip, graylist and publish thresholds, it should be still able to exchange messages with victim. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + // THIRD ROUND OF ATTACK: spammer sends 10 RPCs to the victim node, each containing 500 iHave messages, we expect spammer to be graylisted. + spamIHaveBrokenPromise(t, spammer, blockTopic.String(), receivedIWants, victimNode) + + // wait till victim counts the spam iHaves as broken promises for the third round of attack (one per RPC for a total of 10). + require.Eventually(t, func() bool { + behavioralPenalty, ok := victimNode.PeerScoreExposer().GetBehaviourPenalty(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + // ideally we should have 30 (10 from the first round, 10 from the second round, 10 from the third round), but we give it a buffer of 3 to account for decays and floating point errors. + // note that we intentionally override the decay speed to be 60-times faster in this test. + if behavioralPenalty < 27 { + return false + } + + return true + // Note: we have to wait at least 3 seconds for an iHave to be considered as broken promise (gossipsub parameters), we set it to 10 + // seconds to be on the safe side. + }, 10*time.Second, 100*time.Millisecond) + + spammerScore, ok = victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + require.True(t, ok, "sanity check failed, we should have a score for the spammer node") + // with the third round of the attack, the spammer is about 20 broken promises above the threshold (total ~30 broken promises), hence its overall score must be below the gossip, publish, and graylist thresholds, meaning that + // victim will not exchange messages with it anymore, and also that it will be graylisted meaning all incoming and outgoing RPCs to and from the spammer will be dropped by the victim. + require.Lessf(t, spammerScore, scoring.DefaultGossipThreshold, "sanity check failed, the score of the spammer node must be less than gossip threshold: %f, actual: %f", scoring.DefaultGossipThreshold, spammerScore) + require.Lessf(t, spammerScore, scoring.DefaultPublishThreshold, "sanity check failed, the score of the spammer node must be less than publish threshold: %f, actual: %f", scoring.DefaultPublishThreshold, spammerScore) + require.Lessf(t, spammerScore, scoring.DefaultGraylistThreshold, "sanity check failed, the score of the spammer node must be less than graylist threshold: %f, actual: %f", scoring.DefaultGraylistThreshold, spammerScore) + + // since the spammer score is below the gossip, graylist and publish thresholds, it should not be able to exchange messages with victim anymore. + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{spammer.SpammerNode}, []p2p.LibP2PNode{victimNode}, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} + +// spamIHaveBrokenPromises is a test utility function that is exclusive for the TestGossipSubIHaveBrokenPromises tests. +// It creates and sends 10 RPCs each with 10 iHave messages, each iHave message has 50 message ids, hence overall, we have 5000 iHave message ids. +// It then sends those iHave spams to the victim node and waits till the victim node receives them. +// Args: +// - t: the test instance. +// - spammer: the spammer node. +// - topic: the topic to spam. +// - receivedIWants: a map to keep track of the iWants received by the victim node (exclusive to TestGossipSubIHaveBrokenPromises). +// - victimNode: the victim node. +func spamIHaveBrokenPromise(t *testing.T, spammer *corruptlibp2p.GossipSubRouterSpammer, topic string, receivedIWants *unittest.ProtectedMap[string, struct{}], victimNode p2p.LibP2PNode) { + spamMsgs := spammer.GenerateCtlMessages(10, corruptlibp2p.WithIHave(10, 50, topic)) + var sentIHaves []string + for _, msg := range spamMsgs { + for _, iHave := range msg.Ihave { + for _, msgId := range iHave.MessageIDs { + require.NotContains(t, sentIHaves, msgId) + sentIHaves = append(sentIHaves, msgId) + } + } + } + require.Len(t, sentIHaves, 5000, "sanity check failed, we should have 5000 iHave message ids, actual: %d", len(sentIHaves)) + + // spams the victim node with 1000 spam iHave messages, since iHave messages are for junk message ids, there will be no + // reply from spammer to victim over the iWants. Hence, the victim must count this towards 10 broken promises. + // This sums up to 10 broken promises (1 per RPC). + spammer.SpamControlMessage(t, victimNode, spamMsgs, p2ptest.PubsubMessageFixture(t, p2ptest.WithTopic(topic))) + + // wait till all the spam iHaves are responded with iWants. + require.Eventually(t, func() bool { + for _, msgId := range sentIHaves { + if _, ok := receivedIWants.Get(msgId); !ok { + return false + } + } + + return true + }, 5*time.Second, 100*time.Millisecond, fmt.Sprintf("sanity check failed, we should have received all the iWants for the spam iHaves, expected: %d, actual: %d", len(sentIHaves), receivedIWants.Size())) +} diff --git a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go new file mode 100644 index 00000000000..d8baf4be735 --- /dev/null +++ b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go @@ -0,0 +1,547 @@ +package scoring + +import ( + "context" + "testing" + "time" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/insecure/corruptlibp2p" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/scoring" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + validator "github.com/onflow/flow-go/network/validator/pubsub" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestGossipSubInvalidMessageDelivery_Integration tests that when a victim peer is spammed with invalid messages from +// a spammer peer, the victim will eventually penalize the spammer and stop receiving messages from them. +// Note: the term integration is used here because it requires integrating all components of the libp2p stack. +func TestGossipSubInvalidMessageDelivery_Integration(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + spamMsgFactory func(spammerId peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message + }{ + { + name: "unknown peer, invalid signature", + spamMsgFactory: func(spammerId peer.ID, _ peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithTopic(topic.String())) + }, + }, + { + name: "unknown peer, missing signature", + spamMsgFactory: func(spammerId peer.ID, _ peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithTopic(topic.String()), p2ptest.WithoutSignature()) + }, + }, + { + name: "known peer, invalid signature", + spamMsgFactory: func(spammerId peer.ID, _ peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithFrom(spammerId), p2ptest.WithTopic(topic.String())) + }, + }, + { + name: "known peer, missing signature", + spamMsgFactory: func(spammerId peer.ID, _ peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithFrom(spammerId), p2ptest.WithTopic(topic.String()), p2ptest.WithoutSignature()) + }, + }, + { + name: "self-origin, invalid signature", // bounce back our own messages + spamMsgFactory: func(_ peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithFrom(victimId), p2ptest.WithTopic(topic.String())) + }, + }, + { + name: "self-origin, no signature", // bounce back our own messages + spamMsgFactory: func(_ peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithFrom(victimId), p2ptest.WithTopic(topic.String()), p2ptest.WithoutSignature()) + }, + }, + { + name: "no sender", + spamMsgFactory: func(_ peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithoutSignerId(), p2ptest.WithTopic(topic.String())) + }, + }, + { + name: "no sender, missing signature", + spamMsgFactory: func(_ peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message { + return p2ptest.PubsubMessageFixture(t, p2ptest.WithoutSignerId(), p2ptest.WithTopic(topic.String()), p2ptest.WithoutSignature()) + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + testGossipSubInvalidMessageDeliveryScoring(t, tc.spamMsgFactory) + }) + } +} + +// testGossipSubInvalidMessageDeliveryScoring tests that when a victim peer is spammed with invalid messages from +// a spammer peer, the victim will eventually penalize the spammer and stop receiving messages from them. +// Args: +// - t: the test instance. +// - spamMsgFactory: a function that creates unique invalid messages to spam the victim with. +func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory func(peer.ID, peer.ID, channels.Topic) *pubsub_pb.Message) { + + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkId, role, idProvider) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), + ) + + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + ids := flow.IdentityList{&spammer.SpammerId, &victimIdentity} + nodes := []p2p.LibP2PNode{spammer.SpammerNode, victimNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // checks end-to-end message delivery works on GossipSub + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + totalSpamMessages := 20 + for i := 0; i <= totalSpamMessages; i++ { + spammer.SpamControlMessage(t, victimNode, + spammer.GenerateCtlMessages(1), + spamMsgFactory(spammer.SpammerNode.Host().ID(), victimNode.Host().ID(), blockTopic)) + } + + // wait for at most 3 seconds for the victim node to penalize the spammer node. + // Each heartbeat is 1 second, so 3 heartbeats should be enough to penalize the spammer node. + // Ideally, we should wait for 1 heartbeat, but the score may not be updated immediately after the heartbeat. + require.Eventually(t, func() bool { + spammerScore, ok := victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + if spammerScore >= scoring.DefaultGossipThreshold { + // ensure the score is low enough so that no gossip is routed by victim node to spammer node. + return false + } + if spammerScore >= scoring.DefaultPublishThreshold { + // ensure the score is low enough so that non of the published messages of the victim node are routed to the spammer node. + return false + } + if spammerScore >= scoring.DefaultGraylistThreshold { + // ensure the score is low enough so that the victim node does not accept RPC messages from the spammer node. + return false + } + + return true + }, 3*time.Second, 100*time.Millisecond) + + topicsSnapshot, ok := victimNode.PeerScoreExposer().GetTopicScores(spammer.SpammerNode.Host().ID()) + require.True(t, ok) + require.NotNil(t, topicsSnapshot, "topic scores must not be nil") + require.NotEmpty(t, topicsSnapshot, "topic scores must not be empty") + blkTopicSnapshot, ok := topicsSnapshot[blockTopic.String()] + require.True(t, ok) + + // ensure that the topic snapshot of the spammer contains a record of at least (60%) of the spam messages sent. The 60% is to account for the messages that were delivered before the score was updated, after the spammer is PRUNED, as well as to account for decay. + require.True(t, blkTopicSnapshot.InvalidMessageDeliveries > 0.6*float64(totalSpamMessages), "invalid message deliveries must be greater than %f. invalid message deliveries: %f", 0.9*float64(totalSpamMessages), blkTopicSnapshot.InvalidMessageDeliveries) + + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{victimNode}, []p2p.LibP2PNode{spammer.SpammerNode}, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} + +// TestGossipSubMeshDeliveryScoring_UnderDelivery_SingleTopic tests that when a peer is under-performing in a topic mesh, its score is (slightly) penalized. +func TestGossipSubMeshDeliveryScoring_UnderDelivery_SingleTopic(t *testing.T) { + t.Parallel() + + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + + idProvider := mock.NewIdentityProvider(t) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + + // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + blockTopicOverrideParams := scoring.DefaultTopicScoreParams() + blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride( + &p2p.PeerScoringConfigOverride{ + TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ + blockTopic: blockTopicOverrideParams, + }, + DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + }), + ) + + underPerformerNode, underPerformerId := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + ) + + idProvider.On("ByPeerID", thisNode.Host().ID()).Return(&thisId, true).Maybe() + idProvider.On("ByPeerID", underPerformerNode.Host().ID()).Return(&underPerformerId, true).Maybe() + ids := flow.IdentityList{&underPerformerId, &thisId} + nodes := []p2p.LibP2PNode{underPerformerNode, thisNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // initially both nodes should be able to publish and receive messages from each other in the topic mesh. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + // Also initially the under-performing node should have a score that is at least equal to the MaxAppSpecificReward. + // The reason is in our scoring system, we reward the staked nodes by MaxAppSpecificReward, and the under-performing node is considered staked + // as it is in the id provider of thisNode. + require.Eventually(t, func() bool { + underPerformingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(underPerformerNode.Host().ID()) + if !ok { + return false + } + if underPerformingNodeScore < scoring.MaxAppSpecificReward { + // ensure the score is high enough so that gossip is routed by victim node to spammer node. + return false + } + + return true + }, 1*time.Second, 100*time.Millisecond) + + // however, after one decay interval, we expect the score of the under-performing node to be penalized by -0.05 * MaxAppSpecificReward as + // it has not been able to deliver messages to this node in the topic mesh since the past decay interval. + require.Eventually(t, func() bool { + underPerformingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(underPerformerNode.Host().ID()) + if !ok { + return false + } + if underPerformingNodeScore > 0.96*scoring.MaxAppSpecificReward { // score must be penalized by -0.05 * MaxAppSpecificReward. + // 0.96 is to account for floating point errors. + return false + } + if underPerformingNodeScore < scoring.DefaultGossipThreshold { // even the node is slightly penalized, it should still be able to gossip with this node. + return false + } + if underPerformingNodeScore < scoring.DefaultPublishThreshold { // even the node is slightly penalized, it should still be able to publish to this node. + return false + } + if underPerformingNodeScore < scoring.DefaultGraylistThreshold { // even the node is slightly penalized, it should still be able to establish rpc connection with this node. + return false + } + + return true + }, 3*time.Second, 100*time.Millisecond) + + // even though the under-performing node is penalized, it should still be able to publish and receive messages from this node in the topic mesh. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} + +// TestGossipSubMeshDeliveryScoring_UnderDelivery_TwoTopics tests that when a peer is under-performing in two topics, it is penalized in both topics. +func TestGossipSubMeshDeliveryScoring_UnderDelivery_TwoTopics(t *testing.T) { + t.Parallel() + + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + + idProvider := mock.NewIdentityProvider(t) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + dkgTopic := channels.TopicFromChannel(channels.DKGCommittee, sporkId) + + // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + blockTopicOverrideParams := scoring.DefaultTopicScoreParams() + blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + dkgTopicOverrideParams := scoring.DefaultTopicScoreParams() + dkgTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride( + &p2p.PeerScoringConfigOverride{ + TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ + blockTopic: blockTopicOverrideParams, + dkgTopic: dkgTopicOverrideParams, + }, + DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + }), + ) + + underPerformerNode, underPerformerId := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + ) + + idProvider.On("ByPeerID", thisNode.Host().ID()).Return(&thisId, true).Maybe() + idProvider.On("ByPeerID", underPerformerNode.Host().ID()).Return(&underPerformerId, true).Maybe() + ids := flow.IdentityList{&underPerformerId, &thisId} + nodes := []p2p.LibP2PNode{underPerformerNode, thisNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // subscribe to the topics. + for _, node := range nodes { + for _, topic := range []channels.Topic{blockTopic, dkgTopic} { + _, err := node.Subscribe(topic, validator.TopicValidator(unittest.Logger(), unittest.AllowAllPeerFilter())) + require.NoError(t, err) + } + } + + // Initially the under-performing node should have a score that is at least equal to the MaxAppSpecificReward. + // The reason is in our scoring system, we reward the staked nodes by MaxAppSpecificReward, and the under-performing node is considered staked + // as it is in the id provider of thisNode. + require.Eventually(t, func() bool { + underPerformingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(underPerformerNode.Host().ID()) + if !ok { + return false + } + if underPerformingNodeScore < scoring.MaxAppSpecificReward { + // ensure the score is high enough so that gossip is routed by victim node to spammer node. + return false + } + + return true + }, 2*time.Second, 100*time.Millisecond) + + // No message delivery happens intentionally, so that the under-performing node is penalized. + + // however, after one decay interval, we expect the score of the under-performing node to be penalized by ~ 2 * -0.05 * MaxAppSpecificReward. + require.Eventually(t, func() bool { + underPerformingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(underPerformerNode.Host().ID()) + if !ok { + return false + } + if underPerformingNodeScore > 0.91*scoring.MaxAppSpecificReward { // score must be penalized by ~ 2 * -0.05 * MaxAppSpecificReward. + // 0.91 is to account for the floating point errors. + return false + } + if underPerformingNodeScore < scoring.DefaultGossipThreshold { // even the node is slightly penalized, it should still be able to gossip with this node. + return false + } + if underPerformingNodeScore < scoring.DefaultPublishThreshold { // even the node is slightly penalized, it should still be able to publish to this node. + return false + } + if underPerformingNodeScore < scoring.DefaultGraylistThreshold { // even the node is slightly penalized, it should still be able to establish rpc connection with this node. + return false + } + + return true + }, 3*time.Second, 100*time.Millisecond) + + // even though the under-performing node is penalized, it should still be able to publish and receive messages from this node in both topic meshes. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, dkgTopic, 1, func() interface{} { + return unittest.DKGMessageFixture() + }) +} + +// TestGossipSubMeshDeliveryScoring_Replay_Will_Not_Counted tests that replayed messages will not be counted towards the mesh message deliveries. +func TestGossipSubMeshDeliveryScoring_Replay_Will_Not_Counted(t *testing.T) { + t.Parallel() + + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + + idProvider := mock.NewIdentityProvider(t) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + + // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + blockTopicOverrideParams := scoring.DefaultTopicScoreParams() + blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride( + &p2p.PeerScoringConfigOverride{ + TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ + blockTopic: blockTopicOverrideParams, + }, + DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + }), + ) + + replayingNode, replayingId := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + ) + + idProvider.On("ByPeerID", thisNode.Host().ID()).Return(&thisId, true).Maybe() + idProvider.On("ByPeerID", replayingNode.Host().ID()).Return(&replayingId, true).Maybe() + ids := flow.IdentityList{&replayingId, &thisId} + nodes := []p2p.LibP2PNode{replayingNode, thisNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // initially both nodes should be able to publish and receive messages from each other in the block topic mesh. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + // Initially the replaying node should have a score that is at least equal to the MaxAppSpecificReward. + // The reason is in our scoring system, we reward the staked nodes by MaxAppSpecificReward, and initially every node is considered staked + // as it is in the id provider of thisNode. + initialReplayingNodeScore := float64(0) + require.Eventually(t, func() bool { + replayingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(replayingNode.Host().ID()) + if !ok { + return false + } + if replayingNodeScore < scoring.MaxAppSpecificReward { + // ensure the score is high enough so that gossip is routed by victim node to spammer node. + return false + } + + initialReplayingNodeScore = replayingNodeScore + return true + }, 2*time.Second, 100*time.Millisecond) + + // replaying node acts honestly and sends 200 block proposals on the topic mesh. This is twice the + // defaultTopicMeshMessageDeliveryThreshold, which prevents the replaying node to be penalized. + proposalList := make([]*messages.BlockProposal, 200) + for i := 0; i < len(proposalList); i++ { + proposalList[i] = unittest.ProposalFixture() + } + i := -1 + p2ptest.EnsurePubsubMessageExchangeFromNode(t, ctx, replayingNode, thisNode, blockTopic, len(proposalList), func() interface{} { + i += 1 + return proposalList[i] + }) + + // as the replaying node is not penalized, we expect its score to be equal to the initial score. + require.Eventually(t, func() bool { + replayingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(replayingNode.Host().ID()) + if !ok { + return false + } + if replayingNodeScore < scoring.MaxAppSpecificReward { + // ensure the score is high enough so that gossip is routed by victim node to spammer node. + return false + } + if replayingNodeScore != initialReplayingNodeScore { + // ensure the score is not penalized. + return false + } + + initialReplayingNodeScore = replayingNodeScore + return true + }, 2*time.Second, 100*time.Millisecond) + + // now the replaying node acts maliciously and just replays the same messages again. + i = -1 + p2ptest.EnsureNoPubsubMessageExchange(t, ctx, []p2p.LibP2PNode{replayingNode}, []p2p.LibP2PNode{thisNode}, blockTopic, len(proposalList), func() interface{} { + i += 1 + return proposalList[i] + }) + + // since the last decay interval, the replaying node has not delivered anything new, so its score should be penalized for under-performing. + require.Eventually(t, func() bool { + replayingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(replayingNode.Host().ID()) + if !ok { + return false + } + + if replayingNodeScore >= initialReplayingNodeScore { + // node must be penalized for just replaying the same messages. + return false + } + + if replayingNodeScore >= scoring.MaxAppSpecificReward { + // node must be penalized for just replaying the same messages. + return false + } + + // following if-statements check that even though the node is penalized, it is not penalized too much, and + // can still participate in the network. We don't desire to disallow list a node for just under-performing. + if replayingNodeScore < scoring.DefaultGossipThreshold { + return false + } + + if replayingNodeScore < scoring.DefaultPublishThreshold { + return false + } + + if replayingNodeScore < scoring.DefaultGraylistThreshold { + return false + } + + initialReplayingNodeScore = replayingNodeScore + return true + }, 2*time.Second, 100*time.Millisecond) + + // even though the replaying node is penalized, it should still be able to publish and receive messages from this node in both topic meshes. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} diff --git a/insecure/integration/test/composability_test.go b/insecure/integration/tests/composability_test.go similarity index 99% rename from insecure/integration/test/composability_test.go rename to insecure/integration/tests/composability_test.go index c7ba04d3b7a..4bac2aeb0c5 100644 --- a/insecure/integration/test/composability_test.go +++ b/insecure/integration/tests/composability_test.go @@ -1,4 +1,4 @@ -package test +package tests import ( "context" diff --git a/insecure/integration/test/mock_orchestrator.go b/insecure/integration/tests/mock_orchestrator.go similarity index 99% rename from insecure/integration/test/mock_orchestrator.go rename to insecure/integration/tests/mock_orchestrator.go index b992e1d6c20..5d11879d6df 100644 --- a/insecure/integration/test/mock_orchestrator.go +++ b/insecure/integration/tests/mock_orchestrator.go @@ -1,4 +1,4 @@ -package test +package tests import ( "github.com/onflow/flow-go/insecure" diff --git a/insecure/rpc_inspector/validation_inspector_test.go b/insecure/rpc_inspector/validation_inspector_test.go deleted file mode 100644 index a1fa702dd58..00000000000 --- a/insecure/rpc_inspector/validation_inspector_test.go +++ /dev/null @@ -1,330 +0,0 @@ -package rpc_inspector - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - pb "github.com/libp2p/go-libp2p-pubsub/pb" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/rs/zerolog" - mockery "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - corrupt "github.com/yhassanzadeh13/go-libp2p-pubsub" - "go.uber.org/atomic" - - "github.com/onflow/flow-go/insecure/corruptlibp2p" - "github.com/onflow/flow-go/insecure/internal" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/network/channels" - "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/inspector/validation" - mockp2p "github.com/onflow/flow-go/network/p2p/mock" - inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" - p2ptest "github.com/onflow/flow-go/network/p2p/test" - "github.com/onflow/flow-go/utils/unittest" -) - -// TestValidationInspector_SafetyThreshold ensures that when RPC control message count is below the configured safety threshold the control message validation inspector -// does not return any errors and validation is skipped. -func TestValidationInspector_SafetyThreshold(t *testing.T) { - t.Parallel() - role := flow.RoleConsensus - sporkID := unittest.IdentifierFixture() - spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role) - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - // if GRAFT/PRUNE message count is lower than safety threshold the RPC validation should pass - safetyThreshold := uint64(10) - // create our RPC validation inspector - inspectorConfig := inspectorbuilder.DefaultRPCValidationConfig() - inspectorConfig.NumberOfWorkers = 1 - inspectorConfig.GraftValidationCfg.SafetyThreshold = safetyThreshold - inspectorConfig.PruneValidationCfg.SafetyThreshold = safetyThreshold - - messageCount := 5 - controlMessageCount := int64(2) - - // expected log message logged when valid number GRAFT control messages spammed under safety threshold - graftExpectedMessageStr := fmt.Sprintf("control message %s inspection passed 5 is below configured safety threshold", p2p.CtrlMsgGraft) - // expected log message logged when valid number PRUNE control messages spammed under safety threshold - pruneExpectedMessageStr := fmt.Sprintf("control message %s inspection passed 5 is below configured safety threshold", p2p.CtrlMsgGraft) - - graftInfoLogsReceived := atomic.NewInt64(0) - pruneInfoLogsReceived := atomic.NewInt64(0) - - // setup logger hook, we expect info log validation is skipped - hook := zerolog.HookFunc(func(e *zerolog.Event, level zerolog.Level, message string) { - if level == zerolog.TraceLevel { - if message == graftExpectedMessageStr { - graftInfoLogsReceived.Inc() - } - - if message == pruneExpectedMessageStr { - pruneInfoLogsReceived.Inc() - } - } - }) - logger := zerolog.New(os.Stdout).Hook(hook) - distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) - defer distributor.AssertNotCalled(t, "DistributeInvalidControlMessageNotification", mockery.Anything) - inspector := validation.NewControlMsgValidationInspector(logger, sporkID, inspectorConfig, distributor) - corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(inspector) - victimNode, _ := p2ptest.NodeFixture( - t, - sporkID, - t.Name(), - p2ptest.WithRole(role), - internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), - corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), - ) - inspector.Start(signalerCtx) - nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} - startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) - spammer.Start(t) - defer stopNodesAndInspector(t, cancel, nodes, inspector) - // prepare to spam - generate control messages - ctlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), - corruptlibp2p.WithGraft(messageCount, channels.PushBlocks.String()), - corruptlibp2p.WithPrune(messageCount, channels.PushBlocks.String())) - - // start spamming the victim peer - spammer.SpamControlMessage(t, victimNode, ctlMsgs) - - // eventually we should receive 2 info logs each for GRAFT inspection and PRUNE inspection - require.Eventually(t, func() bool { - return graftInfoLogsReceived.Load() == controlMessageCount && pruneInfoLogsReceived.Load() == controlMessageCount - }, 2*time.Second, 10*time.Millisecond) -} - -// TestValidationInspector_DiscardThreshold ensures that when RPC control message count is above the configured discard threshold the control message validation inspector -// returns the expected error. -func TestValidationInspector_DiscardThreshold(t *testing.T) { - t.Parallel() - role := flow.RoleConsensus - sporkID := unittest.IdentifierFixture() - spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role) - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - // if GRAFT/PRUNE message count is higher than discard threshold the RPC validation should fail and expected error should be returned - discardThreshold := uint64(10) - // create our RPC validation inspector - inspectorConfig := inspectorbuilder.DefaultRPCValidationConfig() - inspectorConfig.NumberOfWorkers = 1 - inspectorConfig.GraftValidationCfg.DiscardThreshold = discardThreshold - inspectorConfig.PruneValidationCfg.DiscardThreshold = discardThreshold - - messageCount := 50 - controlMessageCount := int64(1) - logger := unittest.Logger() - distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) - count := atomic.NewInt64(0) - done := make(chan struct{}) - distributor.On("DistributeInvalidControlMessageNotification", mockery.Anything). - Twice(). - Run(func(args mockery.Arguments) { - count.Inc() - notification, ok := args[0].(*p2p.InvalidControlMessageNotification) - require.True(t, ok) - require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) - require.True(t, validation.IsErrDiscardThreshold(notification.Err)) - require.Equal(t, uint64(messageCount), notification.Count) - require.True(t, notification.MsgType == p2p.CtrlMsgGraft || notification.MsgType == p2p.CtrlMsgPrune) - if count.Load() == 2 { - close(done) - } - }).Return(nil) - inspector := validation.NewControlMsgValidationInspector(logger, sporkID, inspectorConfig, distributor) - // we use inline inspector here so that we can check the error type when we inspect an RPC and - // track which control message type the error involves - inlineInspector := func(id peer.ID, rpc *corrupt.RPC) error { - pubsubRPC := corruptlibp2p.CorruptRPCToPubSubRPC(rpc) - return inspector.Inspect(id, pubsubRPC) - } - victimNode, _ := p2ptest.NodeFixture( - t, - sporkID, - t.Name(), - p2ptest.WithRole(role), - internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), - corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(inlineInspector)), - ) - - inspector.Start(signalerCtx) - nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} - startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) - spammer.Start(t) - defer stopNodesAndInspector(t, cancel, nodes, inspector) - - // prepare to spam - generate control messages - graftCtlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(messageCount, channels.PushBlocks.String())) - pruneCtlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(messageCount, channels.PushBlocks.String())) - - // start spamming the victim peer - spammer.SpamControlMessage(t, victimNode, graftCtlMsgs) - spammer.SpamControlMessage(t, victimNode, pruneCtlMsgs) - - unittest.RequireCloseBefore(t, done, 2*time.Second, "failed to inspect RPC messages on time") -} - -// TestValidationInspector_RateLimitedPeer ensures that the control message validation inspector rate limits peers per control message type as expected. -func TestValidationInspector_RateLimitedPeer(t *testing.T) { - t.Parallel() - role := flow.RoleConsensus - sporkID := unittest.IdentifierFixture() - spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role) - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - - // create our RPC validation inspector - inspectorConfig := inspectorbuilder.DefaultRPCValidationConfig() - inspectorConfig.NumberOfWorkers = 1 - - // here we set the message count to the amount of flow channels - // so that we can generate a valid ctl msg with all valid topics. - flowChannels := channels.Channels() - messageCount := flowChannels.Len() - controlMessageCount := int64(1) - - distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) - count := atomic.NewInt64(0) - done := make(chan struct{}) - distributor.On("DistributeInvalidControlMessageNotification", mockery.Anything). - Times(4). - Run(func(args mockery.Arguments) { - count.Inc() - notification, ok := args[0].(*p2p.InvalidControlMessageNotification) - require.True(t, ok) - require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) - require.True(t, validation.IsErrRateLimitedControlMsg(notification.Err)) - require.Equal(t, uint64(messageCount), notification.Count) - require.True(t, notification.MsgType == p2p.CtrlMsgGraft || notification.MsgType == p2p.CtrlMsgPrune) - if count.Load() == 4 { - close(done) - } - }).Return(nil) - inspector := validation.NewControlMsgValidationInspector(unittest.Logger(), sporkID, inspectorConfig, distributor) - corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(inspector) - victimNode, _ := p2ptest.NodeFixture( - t, - sporkID, - t.Name(), - p2ptest.WithRole(role), - internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), - corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), - ) - - inspector.Start(signalerCtx) - nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} - startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) - spammer.Start(t) - defer stopNodesAndInspector(t, cancel, nodes, inspector) - - // the first time we spam this message it will be processed completely so we need to ensure - // all topics are valid and no duplicates exists. - validCtlMsgs := spammer.GenerateCtlMessages(int(controlMessageCount), func(message *pb.ControlMessage) { - grafts := make([]*pb.ControlGraft, messageCount) - prunes := make([]*pb.ControlPrune, messageCount) - for i := 0; i < messageCount; i++ { - topic := fmt.Sprintf("%s/%s", flowChannels[i].String(), sporkID) - grafts[i] = &pb.ControlGraft{TopicID: &topic} - prunes[i] = &pb.ControlPrune{TopicID: &topic} - } - message.Graft = grafts - message.Prune = prunes - }) - - // start spamming the victim peer - for i := 0; i < 3; i++ { - spammer.SpamControlMessage(t, victimNode, validCtlMsgs) - } - - unittest.RequireCloseBefore(t, done, 2*time.Second, "failed to inspect RPC messages on time") -} - -// TestValidationInspector_InvalidTopicID ensures that when an RPC control message contains an invalid topic ID the expected error is logged. -func TestValidationInspector_InvalidTopicID(t *testing.T) { - t.Parallel() - role := flow.RoleConsensus - sporkID := unittest.IdentifierFixture() - spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role) - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - // if GRAFT/PRUNE message count is higher than discard threshold the RPC validation should fail and expected error should be returned - // create our RPC validation inspector - inspectorConfig := inspectorbuilder.DefaultRPCValidationConfig() - inspectorConfig.PruneValidationCfg.SafetyThreshold = 0 - inspectorConfig.GraftValidationCfg.SafetyThreshold = 0 - inspectorConfig.NumberOfWorkers = 1 - - // SafetyThreshold < messageCount < DiscardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked - // restricting the message count to 1 allows us to only aggregate a single error when the error is logged in the inspector. - messageCount := inspectorConfig.GraftValidationCfg.SafetyThreshold + 1 - controlMessageCount := int64(1) - unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", corruptlibp2p.GossipSubTopicIdFixture(), sporkID)) - malformedTopic := channels.Topic("!@#$%^&**((") - // a topics spork ID is considered invalid if it does not match the current spork ID - invalidSporkIDTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, unittest.IdentifierFixture())) - duplicateTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, sporkID)) - - distributor := mockp2p.NewGossipSubInspectorNotificationDistributor(t) - count := atomic.NewInt64(0) - done := make(chan struct{}) - distributor.On("DistributeInvalidControlMessageNotification", mockery.Anything). - Times(8). - Run(func(args mockery.Arguments) { - count.Inc() - notification, ok := args[0].(*p2p.InvalidControlMessageNotification) - require.True(t, ok) - require.Equal(t, spammer.SpammerNode.Host().ID(), notification.PeerID) - require.True(t, validation.IsErrInvalidTopic(notification.Err) || validation.IsErrDuplicateTopic(notification.Err)) - require.True(t, messageCount == notification.Count || notification.Count == 3) - require.True(t, notification.MsgType == p2p.CtrlMsgGraft || notification.MsgType == p2p.CtrlMsgPrune) - if count.Load() == 8 { - close(done) - } - }).Return(nil) - inspector := validation.NewControlMsgValidationInspector(unittest.Logger(), sporkID, inspectorConfig, distributor) - corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(inspector) - victimNode, _ := p2ptest.NodeFixture( - t, - sporkID, - t.Name(), - p2ptest.WithRole(role), - internal.WithCorruptGossipSub(corruptlibp2p.CorruptGossipSubFactory(), - corruptlibp2p.CorruptGossipSubConfigFactoryWithInspector(corruptInspectorFunc)), - ) - - inspector.Start(signalerCtx) - nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} - startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) - spammer.Start(t) - defer stopNodesAndInspector(t, cancel, nodes, inspector) - - // prepare to spam - generate control messages - graftCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), unknownTopic.String())) - graftCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), malformedTopic.String())) - graftCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), invalidSporkIDTopic.String())) - graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(3, duplicateTopic.String())) - - pruneCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(int(messageCount), unknownTopic.String())) - pruneCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(int(messageCount), malformedTopic.String())) - pruneCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithGraft(int(messageCount), invalidSporkIDTopic.String())) - pruneCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(controlMessageCount), corruptlibp2p.WithPrune(3, duplicateTopic.String())) - - // start spamming the victim peer - spammer.SpamControlMessage(t, victimNode, graftCtlMsgsWithUnknownTopic) - spammer.SpamControlMessage(t, victimNode, graftCtlMsgsWithMalformedTopic) - spammer.SpamControlMessage(t, victimNode, graftCtlMsgsInvalidSporkIDTopic) - spammer.SpamControlMessage(t, victimNode, graftCtlMsgsDuplicateTopic) - - spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsWithUnknownTopic) - spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsWithMalformedTopic) - spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsInvalidSporkIDTopic) - spammer.SpamControlMessage(t, victimNode, pruneCtlMsgsDuplicateTopic) - - unittest.RequireCloseBefore(t, done, 5*time.Second, "failed to inspect RPC messages on time") -} diff --git a/integration/Makefile b/integration/Makefile index 15cc6fcb557..f44449f7ef4 100644 --- a/integration/Makefile +++ b/integration/Makefile @@ -10,10 +10,10 @@ endif # Run the integration test suite .PHONY: integration-test -integration-test: access-tests ghost-tests mvp-tests execution-tests verification-tests collection-tests epochs-tests network-tests consensus-tests +integration-test: access-tests ghost-tests mvp-tests execution-tests verification-tests upgrades-tests collection-tests epochs-tests network-tests consensus-tests .PHONY: ci-integration-test -ci-integration-test: access-tests ghost-tests mvp-tests epochs-tests consensus-tests execution-tests verification-tests network-tests collection-tests +ci-integration-test: access-tests ghost-tests mvp-tests epochs-tests consensus-tests execution-tests verification-tests upgrades-tests network-tests collection-tests ############################################################################################ # CAUTION: DO NOT MODIFY THE TARGETS BELOW! DOING SO WILL BREAK THE FLAKY TEST MONITOR @@ -57,6 +57,11 @@ execution-tests: verification-tests: go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/verification/... +# upgrades-tests tests need to be run sequentially (-p 1) due to interference between different Docker networks when tests are run in parallel +.PHONY: upgrades-tests +upgrades-tests: + go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/upgrades/... -p 1 + .PHONY: network-tests network-tests: go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/network/... diff --git a/integration/benchmark/cmd/ci/main.go b/integration/benchmark/cmd/ci/main.go index f6dd5f2e26a..adab61e1f4c 100644 --- a/integration/benchmark/cmd/ci/main.go +++ b/integration/benchmark/cmd/ci/main.go @@ -32,7 +32,7 @@ type BenchmarkInfo struct { const ( loadType = "token-transfer" metricport = uint(8080) - accessNodeAddress = "127.0.0.1:3569" + accessNodeAddress = "127.0.0.1:4001" pushgateway = "127.0.0.1:9091" accountMultiplier = 50 feedbackEnabled = true diff --git a/integration/benchmark/cmd/manual/Dockerfile b/integration/benchmark/cmd/manual/Dockerfile index 1ad38985a43..58f2b71d42b 100644 --- a/integration/benchmark/cmd/manual/Dockerfile +++ b/integration/benchmark/cmd/manual/Dockerfile @@ -1,7 +1,7 @@ # syntax = docker/dockerfile:experimental # NOTE: Must be run in the context of the repo's root directory -FROM golang:1.19-buster AS build-setup +FROM golang:1.20-buster AS build-setup RUN apt-get update RUN apt-get -y install cmake zip diff --git a/integration/benchmark/cmd/manual/main.go b/integration/benchmark/cmd/manual/main.go index 9161b823394..9250b2a1521 100644 --- a/integration/benchmark/cmd/manual/main.go +++ b/integration/benchmark/cmd/manual/main.go @@ -39,7 +39,7 @@ func main() { tpsFlag := flag.String("tps", "1", "transactions per second (TPS) to send, accepts a comma separated list of values if used in conjunction with `tps-durations`") tpsDurationsFlag := flag.String("tps-durations", "0", "duration that each load test will run, accepts a comma separted list that will be applied to multiple values of the `tps` flag (defaults to infinite if not provided, meaning only the first tps case will be tested; additional values will be ignored)") chainIDStr := flag.String("chain", string(flowsdk.Emulator), "chain ID") - accessNodes := flag.String("access", net.JoinHostPort("127.0.0.1", "3569"), "access node address") + accessNodes := flag.String("access", net.JoinHostPort("127.0.0.1", "4001"), "access node address") serviceAccountPrivateKeyHex := flag.String("servPrivHex", unittest.ServiceAccountPrivateKeyHex, "service account private key hex") logLvl := flag.String("log-level", "info", "set log level") metricport := flag.Uint("metricport", 8080, "port for /metrics endpoint") diff --git a/integration/benchmark/server/bench.sh b/integration/benchmark/server/bench.sh new file mode 100755 index 00000000000..6ada16119a1 --- /dev/null +++ b/integration/benchmark/server/bench.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -x +set -o pipefail + +# this flow-go sub folder will be where all the TPS tests will be run +# this will keep the TPS automation code separate from the code that's being tested so we won't run into issues +# of having old versions of automation code just because we happen to be testing an older version flow-go +git clone https://github.com/onflow/flow-go.git +cd flow-go/integration/localnet + +git fetch +git fetch --tags + +while read -r branch_hash; do + hash="${branch_hash##*:}" + branch="${branch_hash%%:*}" + + git checkout "$branch" || continue + git reset --hard "$hash" || continue + + git log --oneline | head -1 + git describe + + make -C ../.. crypto_setup_gopath + + # instead of running "make stop" which uses docker-compose for a lot of older versions, + # we explicitly run the command here with "docker compose" + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.nodes.yml down -v --remove-orphans + + make clean-data + make -e COLLECTION=12 VERIFICATION=12 NCLUSTERS=12 LOGLEVEL=INFO bootstrap + + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.nodes.yml build || continue + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.nodes.yml up -d || continue + + # sleep is workaround for slow initialization of some node types, so that benchmark does not quit immediately with "connection refused" + sleep 30; + go run -tags relic ../benchmark/cmd/ci -log-level debug -git-repo-path ../../ -tps-initial 800 -tps-min 1 -tps-max 1200 -duration 30m + + # instead of running "make stop" which uses docker-compose for a lot of older versions, + # we explicitly run the command here with "docker compose" + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.nodes.yml down -v --remove-orphans + + docker system prune -a -f + make clean-data +done ../ diff --git a/integration/go.sum b/integration/go.sum index bde5c26e373..4aac8d7305d 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -22,6 +22,16 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -30,19 +40,20 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.48.0 h1:u+fhS1jJOkPO9vdM84M8HO5VznTfVUicBeoXNKD26ho= -cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.50.0 h1:RscMV6LbnAmhAzD893Lv9nXXy2WCaJmbxYPWDLbGqNQ= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datacatalog v1.12.0 h1:3uaYULZRLByPdbuUvacGeqneudztEM4xqKQsBcxbDnY= +cloud.google.com/go/datacatalog v1.13.0 h1:4H5IJiyUE0X6ShQBqgFFZvGGcrwGVndTwUSLP4c52gw= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= -cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/kms v1.0.0/go.mod h1:nhUehi+w7zht2XrUfvTRNpxrfayBHqP4lu2NSywui/0= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/profiler v0.3.0 h1:R6y/xAeifaUXxd2x6w+jIwKxoKl8Cv5HJvcvASTPWJo= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -55,8 +66,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= -cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -101,6 +112,7 @@ github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBY github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= @@ -124,8 +136,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/apache/arrow/go/v10 v10.0.1 h1:n9dERvixoC/1JjDmBcs9FPaEryoANa2sCgVFo6ez9cI= -github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0 h1:hqauxvFQxww+0mEU/2XHG6LT7eZternCZq+A5Yly2uM= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= @@ -184,15 +196,16 @@ github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAm github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.3.0 h1:h7mv5q31cthBTd7V4kLAZaIThj1e8vPGcSqpPue9KVI= -github.com/bits-and-blooms/bitset v1.3.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= +github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= @@ -220,16 +233,19 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bytecodealliance/wasmtime-go v0.22.0/go.mod h1:q320gUxqyI8yB+ZqRuaJOEnGkAnHh6WtJjMaT2CW4wI= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -258,8 +274,8 @@ github.com/consensys/goff v0.3.10/go.mod h1:xTldOBEHmFiYS0gPXd3NsaEqZWlnmeWcRLWg github.com/consensys/gurvy v0.3.8/go.mod h1:sN75xnsiD593XnhbhvG2PkOy194pZBzqShWF/kwuW/g= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -276,6 +292,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -305,9 +322,9 @@ github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQY github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= @@ -318,8 +335,8 @@ github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdw github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= -github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -360,6 +377,7 @@ github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/ef-ds/deque v1.0.4 h1:iFAZNmveMT9WERAkqLJ+oaABF9AcVQ5AjXem/hroniI= github.com/ef-ds/deque v1.0.4/go.mod h1:gXDnTC3yqvBcHbq2lcExjtAcVrOnJCbMcZXmuj8Z4tg= +github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= @@ -371,9 +389,11 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= github.com/ethereum/go-ethereum v1.10.1 h1:bGQezu+kqqRBczcSAruEoqVzTjtkeDnUGI2I4uroyUE= github.com/ethereum/go-ethereum v1.10.1/go.mod h1:E5e/zvdfUVr91JZ0AwjyuJM3x+no51zZJRz61orLLSk= @@ -391,25 +411,27 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/fxamacker/cbor/v2 v2.4.1-0.20220515183430-ad2eae63303f h1:dxTR4AaxCwuQv9LAVTAC2r1szlS+epeuPT5ClLKT6ZY= -github.com/fxamacker/cbor/v2 v2.4.1-0.20220515183430-ad2eae63303f/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fxamacker/cbor/v2 v2.2.1-0.20210927235116-3d6d5d1de29b/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c h1:5tm/Wbs9d9r+qZaUFXk59CWDD0+77PBqDREffYkyi5c= +github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/circlehash v0.1.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM= github.com/fxamacker/circlehash v0.3.0 h1:XKdvTtIJV9t7DDUtsf0RIpC1OcxZtPbmgIH7ekx28WA= github.com/fxamacker/circlehash v0.3.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gammazero/deque v0.1.0 h1:f9LnNmq66VDeuAlSAapemq/U7hJ2jpIWa4c09q8Dlik= github.com/gammazero/deque v0.1.0/go.mod h1:KQw7vFau1hHuM8xmI9RbgKFbAsQFWmBpqQ2KenFLk6M= github.com/gammazero/workerpool v1.1.2 h1:vuioDQbgrz4HoaCi2q1HLlOXdpbap5AET7xu5/qj87g= github.com/gammazero/workerpool v1.1.2/go.mod h1:UelbXcO0zCIGFcufcirHhq2/xtLXJdQ29qZNlXG9OjQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/glebarez/go-sqlite v1.21.0 h1:b8MHPtBagkSD2gntImZPsG3o3QEXgMDxguW/GLUonHQ= -github.com/glebarez/go-sqlite v1.21.0/go.mod h1:GodsA6yGSa3eKbvpr7dS+JaqazzVfMcjIXvx6KHhW/c= +github.com/glebarez/go-sqlite v1.21.1 h1:7MZyUPh2XTrHS7xNEHQbrhfMZuPSzhkm2A1qgg0y5NY= +github.com/glebarez/go-sqlite v1.21.1/go.mod h1:ISs8MF6yk5cL4n/43rSOmVMGJJjHYr7L2MbZZ5Q4E2E= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= @@ -442,23 +464,31 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= +github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= @@ -480,8 +510,9 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -504,6 +535,7 @@ github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+Licev github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -517,8 +549,10 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -557,6 +591,7 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -568,8 +603,13 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 h1:wORs2YN3R3ona/CXYuTvLM31QlgoNKHvlCNuArCDDCU= -github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -584,11 +624,13 @@ github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -615,8 +657,9 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -641,11 +684,14 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= +github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -656,18 +702,19 @@ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iU github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= -github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= -github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= +github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= +github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= @@ -681,14 +728,17 @@ github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1: github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= +github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUiJ2teMvI= github.com/ipfs/go-bitswap v0.5.0/go.mod h1:WwyyYD33RHCpczgHjpx+xjWYIy8l41K+l5EMy4/ctSM= github.com/ipfs/go-bitswap v0.9.0 h1:/dZi/XhUN/aIk78pI4kaZrilUglJ+7/SCmOHWIpiy8E= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= +github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= +github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-blockservice v0.2.0/go.mod h1:Vzvj2fAnbbyly4+T7D5+p9n3+ZKVHA2bRMMo1QoILtQ= github.com/ipfs/go-blockservice v0.4.0 h1:7MUijAW5SqdsqEW/EhnNFRJXVF8mGU5aGhZ3CQaCWbY= @@ -701,8 +751,8 @@ github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67Fexh github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= -github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= @@ -733,8 +783,8 @@ github.com/ipfs/go-fetcher v1.5.0/go.mod h1:5pDZ0393oRF/fHiLmtFZtpMNBQfHOYNPtryW github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blockstore v0.2.0/go.mod h1:SNeEpz/ICnMYZQYr7KNZTjdn7tEPB/99xpe8xI1RW7o= -github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= -github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= +github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12XwJyvYvMM= +github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -752,8 +802,9 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAz github.com/ipfs/go-ipfs-exchange-offline v0.1.0/go.mod h1:YdJXa+yPF1na+gfYHYejtLwHFpuKv22eatApNiSfanM= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= -github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= +github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= +github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= github.com/ipfs/go-ipfs-provider v0.7.0 h1:5GpHv46eIS8h2mbbKg1ckU5paajDYJtE4GA/SBepOQg= github.com/ipfs/go-ipfs-provider v0.7.0/go.mod h1:mgjsWgDt9j19N1REPxRa31p+eRIQmjNt5McNdQQ5CsA= github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= @@ -762,10 +813,8 @@ github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FF github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-format v0.3.0 h1:Mwm2oRLzIuUwEPewWAWyMuuBQUsn3awfFEYVb8akMOQ= -github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipns v0.2.0 h1:BgmNtQhqOw5XEZ8RAfWEpK4DhqaYiuP6h71MhIp7xXU= -github.com/ipfs/go-ipns v0.2.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= +github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds= +github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= @@ -785,13 +834,14 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= -github.com/ipfs/go-peertaskqueue v0.7.0 h1:VyO6G4sbzX80K58N60cCaHsSsypbUNs1GjO5seGNsQ0= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= +github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= +github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.14.1 h1:n9obcUnuqPK34HlfbiB+o9GhXE/x59uue4z9YTsaoj4= -github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= +github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -838,6 +888,7 @@ github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/go-bindata v3.23.0+incompatible h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8= github.com/kevinburke/go-bindata v3.23.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -852,30 +903,29 @@ github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= -github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= -github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= -github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= -github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= +github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= +github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -885,6 +935,8 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/leanovate/gopter v0.2.8/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v0.0.0-20170810061220-e42267488fe3/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -914,10 +966,10 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= -github.com/libp2p/go-libp2p v0.24.2 h1:iMViPIcLY0D6zr/f+1Yq9EavCZu2i7eDstsr1nEwSAk= -github.com/libp2p/go-libp2p v0.24.2/go.mod h1:WuxtL2V8yGjam03D93ZBC19tvOUiPpewYv1xdFGWu1k= -github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= -github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= +github.com/libp2p/go-libp2p v0.28.1 h1:YurK+ZAI6cKfASLJBVFkpVBdl3wGhFi6fusOt725ii8= +github.com/libp2p/go-libp2p v0.28.1/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= +github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= +github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI= @@ -961,10 +1013,10 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-kad-dht v0.19.0 h1:2HuiInHZTm9ZvQajaqdaPLHr0PCKKigWiflakimttE0= -github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= -github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= -github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= +github.com/libp2p/go-libp2p-kad-dht v0.24.2 h1:zd7myKBKCmtZBhI3I0zm8xBkb28v3gmSEtQfBdAdFwc= +github.com/libp2p/go-libp2p-kad-dht v0.24.2/go.mod h1:BShPzRbK6+fN3hk8a0WGAYKpb8m4k+DtchkqouGTrSg= +github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= +github.com/libp2p/go-libp2p-kbucket v0.6.3/go.mod h1:RCseT7AH6eJWxxk2ol03xtP9pEHetYSPXOaJnOiD8i0= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= @@ -989,8 +1041,8 @@ github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRj github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= -github.com/libp2p/go-libp2p-pubsub v0.8.2 h1:QLGUmkgKmwEVxVDYGsqc5t9CykOMY2Y21cXQHjR462I= -github.com/libp2p/go-libp2p-pubsub v0.8.2/go.mod h1:e4kT+DYjzPUYGZeWk4I+oxCSYTXizzXii5LDRRhjKSw= +github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= +github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc= github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= @@ -1043,13 +1095,13 @@ github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= -github.com/libp2p/go-msgio v0.2.0 h1:W6shmB+FeynDrUVl2dgFQvzfBZcXiyqY4VmpQLu9FqU= -github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= +github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= +github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= -github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= -github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= +github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= +github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= @@ -1061,12 +1113,10 @@ github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= -github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= -github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= -github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= +github.com/libp2p/go-reuseport v0.3.0 h1:iiZslO5byUYZEg9iCwJGf5h+sf1Agmqx2V2FDjPyvUw= +github.com/libp2p/go-reuseport v0.3.0/go.mod h1:laea40AimhtfEqysZ71UpYj4S+R9VpH8PgqLo7L+SwI= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= @@ -1099,40 +1149,35 @@ github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rB github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= +github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= -github.com/lucas-clemente/quic-go v0.31.1 h1:O8Od7hfioqq0PMYHDyBkxU2aA7iZ2W9pjbrWuja2YR4= -github.com/lucas-clemente/quic-go v0.31.1/go.mod h1:0wFbizLgYzqHqtlyxyCaJKlE7bYgE6JQ+54TLd/Dq2g= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c h1:OqVcb1Dkheracn4fgCjxlfhuSnM8jmPbrWkJbRIC4fo= -github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c/go.mod h1:5/Yq7mnb+VdE44ff+FL8LSOPEquOVqm/7Hz40U4VUZo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= -github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= -github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= -github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/marten-seemann/webtransport-go v0.4.3 h1:vkt5o/Ci+luknRteWdYGYH1KcB7ziup+J+1PzZJIvmg= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -1143,20 +1188,25 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= -github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= +github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1167,8 +1217,8 @@ github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= +github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -1185,8 +1235,9 @@ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= @@ -1230,8 +1281,8 @@ github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= -github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= -github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= +github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -1250,11 +1301,10 @@ github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysj github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= -github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= -github.com/multiformats/go-multicodec v0.7.0 h1:rTUjGOwjlhGHbEMbPoSUJowG1spZTVsITRANCjKTUAQ= -github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= +github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1263,16 +1313,15 @@ github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.0.16/go.mod h1:zhfEIgVnB/rPMfxgFw15ZmGoNaKyNUIE4IWHG/kC+Ag= -github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= github.com/multiformats/go-multistream v0.2.0/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= github.com/multiformats/go-multistream v0.2.2/go.mod h1:UIcnm7Zuo8HKG+HkWgfQsGL+/MIEhyTqbODbIUwSXKs= -github.com/multiformats/go-multistream v0.3.3 h1:d5PZpjwRgVlbwfdTDjife7XszfZd8KYWfROYFlGcR8o= -github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= +github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= +github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -1301,28 +1350,39 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onflow/atree v0.5.0 h1:y3lh8hY2fUo8KVE2ALVcz0EiNTq0tXJ6YTXKYVDA+3E= -github.com/onflow/atree v0.5.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= -github.com/onflow/cadence v0.38.1 h1:8YpnE1ixAGB8hF3t+slkHGhjfIBJ95dqUS+sEHrM2kY= -github.com/onflow/cadence v0.38.1/go.mod h1:SpfjNhPsJxGIHbOthE9JD/e8JFaFY73joYLPsov+PY4= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 h1:9QEI+C9k/Cx/TRC3SCAHmNQqV7UlLG0DHQewTl8Lg6w= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1/go.mod h1:xiSs5IkirazpG89H5jH8xVUKNPlCZqUhLH4+vikQVS4= -github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 h1:dhXSFiBkS6Q3XmBctJAfwR4XPkgBT7VNx08F/zTBgkM= -github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1/go.mod h1:cBimYbTvHK77lclJ1JyhvmKAB9KDzCeWm7OW1EeQSr0= -github.com/onflow/flow-emulator v0.46.0 h1:oORapiOgMTlfDNdgFBAkExe9LiSaul9GqVPxOs7h/bg= -github.com/onflow/flow-emulator v0.46.0/go.mod h1:vlv3NUS/HpOpUyHia9vOPCMBLx2jbELTq3Ktb8+4Bmg= -github.com/onflow/flow-ft/lib/go/contracts v0.5.0 h1:Cg4gHGVblxcejfNNG5Mfj98Wf4zbY76O0Y28QB0766A= -github.com/onflow/flow-ft/lib/go/contracts v0.5.0/go.mod h1:1zoTjp1KzNnOPkyqKmWKerUyf0gciw+e6tAEt0Ks3JE= -github.com/onflow/flow-go-sdk v0.40.0 h1:s8uwoyTquN8tjdXpqGmNkXTjf79yUII8JExc5QEl4Xw= -github.com/onflow/flow-go-sdk v0.40.0/go.mod h1:34dxXk9Hp/bQw6Zy6+H44Xo0kQU+aJyQoqdDxq00rJM= -github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= -github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230407005012-727d541fd5f8 h1:O8uM6GVVMhRwBtYaGl93+tDSu6vWqUc47b12fPkZGXk= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230407005012-727d541fd5f8/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= -github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8/go.mod h1:73C8FlT4L/Qe4Cf5iXUNL8b2pvu4zs5dJMMJ5V2TjUI= +github.com/onflow/atree v0.1.0-beta1.0.20211027184039-559ee654ece9/go.mod h1:+6x071HgCF/0v5hQcaE5qqjc2UqN5gCU8h5Mk6uqpOg= +github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= +github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= +github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= +github.com/onflow/cadence v0.39.14 h1:YoR3YFUga49rqzVY1xwI6I2ZDBmvwGh13jENncsleC8= +github.com/onflow/cadence v0.39.14/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d h1:B7PdhdUNkve5MVrekWDuQf84XsGBxNZ/D3x+QQ8XeVs= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d/go.mod h1:xAiV/7TKhw863r6iO3CS5RnQ4F+pBY1TxD272BsILlo= +github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= +github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= +github.com/onflow/flow-emulator v0.53.0 h1:VIMljBL77VnO+CeeJX1N5GVmF245XwZrFGv63dLPQGk= +github.com/onflow/flow-emulator v0.53.0/go.mod h1:o7O+b3fQYs26vJ+4SeMY/T9kA1rT09tFxQccTFyM5b4= +github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= +github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= +github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= +github.com/onflow/flow-go-sdk v0.41.9 h1:cyplhhhc0RnfOAan2t7I/7C9g1hVGDDLUhWj6ZHAkk4= +github.com/onflow/flow-go-sdk v0.41.9/go.mod h1:e9Q5TITCy7g08lkdQJxP8fAKBnBoC5FjALvUKr36j4I= +github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= +github.com/onflow/flow-go/crypto v0.24.9 h1:0EQp+kSZYJepMIiSypfJVe7tzsPcb6UXOdOtsTCDhBs= +github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= +github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= +github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= +github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead h1:2j1Unqs76Z1b95Gu4C3Y28hzNUHBix7wL490e61SMSw= +github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead/go.mod h1:E3ScfQb5XcWJCIAdtIeEnr5i5l2y60GT0BTXeIHseWg= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= github.com/onflow/sdks v0.5.0/go.mod h1:F0dj0EyHC55kknLkeD10js4mo14yTdMotnWMslPirrU= +github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d h1:gAEqYPn3DS83rHIKEpsajnppVD1+zwuYPFyeDVFaQvg= +github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d/go.mod h1:iMC8gkLqu4nkbkAla5HkSBb+FGyQOZiWz3DYm2wSXCk= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -1331,8 +1391,8 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.6.1 h1:1xQPCjcqYw/J5LchOcp4/2q/jzJFjiAOc25chhnDw+Q= -github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1340,7 +1400,7 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= +github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= @@ -1374,10 +1434,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.2 h1:+jQXlF3scKIcSEKkdHzXhCTDLPFi5r1wnK6yPS+49Gw= -github.com/pelletier/go-toml/v2 v2.0.2/go.mod h1:MovirKjgVRESsAvNZlAjtFwV867yGuwRkXbG66OzopI= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= @@ -1398,12 +1456,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= +github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= github.com/plus3it/gorecurcopy v0.0.1 h1:H7AgvM0N/uIo7o1PQRlewEGQ92BNr7DqbPy5lnR3uJI= github.com/plus3it/gorecurcopy v0.0.1/go.mod h1:NvVTm4RX68A1vQbHmHunDO4OtBLVroT6CrsiqAzNyJA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= +github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= @@ -1424,8 +1484,8 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1435,8 +1495,8 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= -github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1455,21 +1515,31 @@ github.com/psiemens/graceland v1.0.0 h1:L580AVV4Q2XLcPpmvxJRH9UpEAYr/eu2jBKmMglh github.com/psiemens/graceland v1.0.0/go.mod h1:1Tof+vt1LbmcZFE0lzgdwMN0QBymAChG3FRgDx8XisU= github.com/psiemens/sconfig v0.1.0 h1:xfWqW+TRpih7mXZIqKYTmpRhlZLQ1kbxV8EjllPv76s= github.com/psiemens/sconfig v0.1.0/go.mod h1:+MLKqdledP/8G3rOBpknbLh0IclCf4WneJUtS26JB2U= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U= +github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= +github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E= +github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= +github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0= +github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA= +github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= +github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 h1:VstopitMQi3hZP0fzvnsLmzXZdQGc4bEcgu24cp+d4M= -github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a h1:s7GrsqeorVkFR1vGmQ6WVL9nup0eyQCC+YVUeSQLH/Q= -github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= @@ -1484,8 +1554,9 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8= github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko= +github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= +github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= @@ -1532,24 +1603,26 @@ github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ys github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag= github.com/slok/go-http-metrics v0.10.0 h1:rh0LaYEKza5eaYRGDXujKrOln57nHBi4TtVhmNEpbgM= github.com/slok/go-http-metrics v0.10.0/go.mod h1:lFqdaS4kWMfUKCSukjC47PdCeTk+hXDUVm8kLHRqJ38= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= -github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.9.0 h1:sFSLUHgxdnN32Qy38hK3QkYBFXZj9DKjVjCUCtD7juY= -github.com/spf13/afero v1.9.0/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= @@ -1557,8 +1630,8 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1570,8 +1643,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= -github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= @@ -1592,17 +1665,19 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= -github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/supranational/blst v0.3.4/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/supranational/blst v0.3.10 h1:CMciDZ/h4pXDDXQASe8ZGTNKUiVNxVVA5hpci2Uuhuk= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= @@ -1623,6 +1698,7 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -1632,10 +1708,10 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA= -github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= -github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= +github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= @@ -1645,8 +1721,6 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= -github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= -github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= @@ -1660,8 +1734,8 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee h1:yFB2xjfswpuRh8FHagdBMKcBMltjr5u/XKzX6fkJO5E= -github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee/go.mod h1:Tylw4k1H86gbJx84i3r7qahN/mBaeMpUBvHY0Igshfw= +github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg.0.20230703223453-544e2fe28a26 h1:C7wI5fYoMlSMEGEVi/PH3Toh9TzpIWlvX9DTLIco52Y= +github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg.0.20230703223453-544e2fe28a26/go.mod h1:bZmV+V29p09ee2aWv/1WCAfHKIwWlwYmNeMspQ2CzJc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1672,8 +1746,10 @@ github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPR github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/blake3 v0.2.0/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= @@ -1696,42 +1772,45 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.8.0 h1:zcvBFizPbpa1q7FehvFiHbQwGzmPILebO0tyqIR5Djg= -go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 h1:ao8CJIShCaIbaMsGxy+jp2YHSudketpDgDRcbirov78= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0 h1:LrHL1A3KqIgAgi6mK7Q0aczmzU414AONAGT5xtnp+uo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0/go.mod h1:w8aZL87GMOvOBa2lU/JlVXE1q4chk/0FX+8ai4513bw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.8.0 h1:00hCSGLIxdYK/Z7r8GkaX0QIlfvgU3tmnLlQvcnix6U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.8.0/go.mod h1:twhIvtDQW2sWP1O2cT1N8nkSBgKCRZv2z6COTTBrf8Q= -go.opentelemetry.io/otel/sdk v1.8.0 h1:xwu69/fNuwbSHWe/0PGS888RmjWY181OmcXDQKu7ZQk= -go.opentelemetry.io/otel/sdk v1.8.0/go.mod h1:uPSfc+yfDH2StDM/Rm35WE8gXSNdvCg023J6HeGNO0c= -go.opentelemetry.io/otel/trace v1.8.0 h1:cSy0DF9eGI5WIfNwZ1q2iUyGj00tGzP24dE1lOlHrfY= -go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.18.0 h1:W5hyXNComRa23tGpKwG+FRAc4rfF6ZUg1JReK+QHS80= -go.opentelemetry.io/proto/otlp v0.18.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE= -go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= -go.uber.org/fx v1.18.2 h1:bUNI6oShr+OVFQeU8cDNbnN7VFsu+SsjHzUF51V/GAU= -go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= +go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= +go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY= +go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -1765,6 +1844,7 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1783,8 +1863,8 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1799,8 +1879,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w= -golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1816,6 +1896,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= @@ -1830,8 +1911,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1887,13 +1968,13 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1905,9 +1986,15 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1921,8 +2008,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1951,6 +2038,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1959,6 +2047,7 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1989,7 +2078,10 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1998,8 +2090,10 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2010,35 +2104,43 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2049,14 +2151,14 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2116,6 +2218,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -2125,11 +2228,13 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2139,7 +2244,9 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNq gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= -gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= +gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= +gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= @@ -2166,6 +2273,17 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -2221,10 +2339,34 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2251,10 +2393,17 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -2287,11 +2436,13 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= -gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -2308,7 +2459,6 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -2328,18 +2478,18 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= -lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0= -modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY= +modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/sqlite v1.20.4 h1:J8+m2trkN+KKoE7jglyHYYYiaq5xmz2HoHJIiBlRzbE= -modernc.org/sqlite v1.20.4/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A= +modernc.org/sqlite v1.21.1 h1:GyDFqNnESLOhwwDRaHGdp2jKLDzpyT/rNLglX3ZkMSU= +modernc.org/sqlite v1.21.1/go.mod h1:XwQ0wZPIh1iKb5mkvCJ3szzbhk+tykC8ZWqTRTgYRwI= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= +pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/integration/localnet/Makefile b/integration/localnet/Makefile index f35cb0643e0..30af385fae3 100644 --- a/integration/localnet/Makefile +++ b/integration/localnet/Makefile @@ -95,27 +95,27 @@ start-cached: start-metrics start-flow-cached # Starts metrics services .PHONY: start-metrics start-metrics: - DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose -f docker-compose.metrics.yml up -d + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.metrics.yml up -d # Starts a version of localnet with just flow nodes and without metrics services. # This prevents port collision and consumption when these services are not needed. # All images are re-built prior to being started. .PHONY: start-flow start-flow: - DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose -f docker-compose.nodes.yml up -d --build + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.nodes.yml up -d --build # Same as start-flow, but most recently built images are used. .PHONY: start-flow-cached start-flow-cached: - DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose -f docker-compose.nodes.yml up -d + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.nodes.yml up -d .PHONY: build-flow build-flow: - DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose -f docker-compose.nodes.yml build + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.nodes.yml build .PHONY: stop stop: - DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker-compose -f docker-compose.metrics.yml -f docker-compose.nodes.yml down -v --remove-orphans + DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f docker-compose.metrics.yml -f docker-compose.nodes.yml down -v --remove-orphans .PHONY: load load: @@ -143,7 +143,7 @@ clean-data: rm -f ./docker-compose.nodes.yml # deletes the stopped environment-clean container(s) - running this command inside another target doesn't delete the containers so it's isolated to run in a separate target -# Note: running this target shows an error on the command line "make: *** [clean-data2] Error 1" but the container is still deletes +# Note: running this target shows an error on the command line "make: *** [clean-data2] Error 1" but the container is still deleted .PHONY: clean-data2 clean-data2: docker rm $(shell docker ps -aq --filter ancestor=environment-clean) diff --git a/integration/localnet/README.md b/integration/localnet/README.md index 079d62ebc34..7dafa747969 100644 --- a/integration/localnet/README.md +++ b/integration/localnet/README.md @@ -217,7 +217,7 @@ An example of the Flow CLI configuration modified for connecting to the localnet ``` { "networks": { - "localnet": "127.0.0.1:3569" + "localnet": "127.0.0.1:4001" } } ``` @@ -238,7 +238,7 @@ An example of the Flow CLI configuration with the service account added: ``` { "networks": { - "localnet": "127.0.0.1:3569" + "localnet": "127.0.0.1:4001" }, "accounts": { "localnet-service-account": { @@ -355,15 +355,15 @@ After the transaction is sealed, the account with `` should hav # admin tool The admin tool is enabled by default in localnet for all node type except access node. -For instance, in order to use admin tool to change log level, first find the local port that maps to `9002` which is the admin tool address, if the local port is `3702`, then run: +For instance, in order to use admin tool to change log level, first find the local port that maps to `9002` which is the admin tool address, if the local port is `6100`, then run: ``` -curl localhost:3702/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-log-level", "data": "debug"}' +curl localhost:6100/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-log-level", "data": "debug"}' ``` To find the local port after launching the localnet, run `docker ps -a`, and find the port mapping. -For instance, the following result of `docker ps -a ` shows `localnet-collection` maps 9002 port to localhost's 3702 port, so we could use 3702 port to connect to admin tool. +For instance, the following result of `docker ps -a ` shows `localnet-collection` maps 9002 port to localhost's 6100 port, so we could use 6100 port to connect to admin tool. ``` -2e0621f7e592 localnet-access "/bin/app --nodeid=9…" 9 seconds ago Up 8 seconds 0.0.0.0:3571->9000/tcp, :::3571->9000/tcp, 0.0.0.0:3572->9001/tcp, :::3572->9001/tcp localnet_access_2_1 -fcd92116f902 localnet-collection "/bin/app --nodeid=0…" 9 seconds ago Up 8 seconds 0.0.0.0:3702->9002/tcp, :::3702->9002/tcp localnet_collection_1_1 -dd841d389e36 localnet-access "/bin/app --nodeid=a…" 10 seconds ago Up 9 seconds 0.0.0.0:3569->9000/tcp, :::3569->9000/tcp, 0.0.0.0:3570->9001/tcp, :::3570->9001/tcp localnet_access_1_1 +2e0621f7e592 localnet-access "/bin/app --nodeid=9…" 9 seconds ago Up 8 seconds 0.0.0.0:4011->9000/tcp, :::4011->9000/tcp, 0.0.0.0:4012->9001/tcp, :::4012->9001/tcp localnet_access_2_1 +fcd92116f902 localnet-collection "/bin/app --nodeid=0…" 9 seconds ago Up 8 seconds 0.0.0.0:6100->9002/tcp, :::6100->9002/tcp localnet_collection_1_1 +dd841d389e36 localnet-access "/bin/app --nodeid=a…" 10 seconds ago Up 9 seconds 0.0.0.0:4001->9000/tcp, :::4001->9000/tcp, 0.0.0.0:4002->9001/tcp, :::4002->9001/tcp localnet_access_1_1 ``` diff --git a/integration/localnet/builder/bootstrap.go b/integration/localnet/builder/bootstrap.go index 201aaaade58..2a42963461f 100644 --- a/integration/localnet/builder/bootstrap.go +++ b/integration/localnet/builder/bootstrap.go @@ -336,7 +336,7 @@ func prepareConsensusService(container testnet.ContainerConfig, i int, n int) Se timeout := 1200*time.Millisecond + consensusDelay service.Command = append(service.Command, - fmt.Sprintf("--block-rate-delay=%s", consensusDelay), + fmt.Sprintf("--cruise-ctl-fallback-proposal-duration=%s", consensusDelay), fmt.Sprintf("--hotstuff-min-timeout=%s", timeout), "--chunk-alpha=1", "--emergency-sealing-active=false", @@ -363,7 +363,6 @@ func prepareCollectionService(container testnet.ContainerConfig, i int, n int) S timeout := 1200*time.Millisecond + collectionDelay service.Command = append(service.Command, - fmt.Sprintf("--block-rate-delay=%s", collectionDelay), fmt.Sprintf("--hotstuff-min-timeout=%s", timeout), fmt.Sprintf("--ingress-addr=%s:%s", container.ContainerName, testnet.GRPCPort), "--insecure-access-api=false", @@ -454,12 +453,14 @@ func prepareObserverService(i int, observerName string, agPublicKey string) Serv fmt.Sprintf("--rpc-addr=%s:%s", observerName, testnet.GRPCPort), fmt.Sprintf("--secure-rpc-addr=%s:%s", observerName, testnet.GRPCSecurePort), fmt.Sprintf("--http-addr=%s:%s", observerName, testnet.GRPCWebPort), + fmt.Sprintf("--rest-addr=%s:%s", observerName, testnet.RESTPort), ) service.AddExposedPorts( testnet.GRPCPort, testnet.GRPCSecurePort, testnet.GRPCWebPort, + testnet.RESTPort, ) // observer services rely on the access gateway @@ -517,7 +518,7 @@ func defaultService(name, role, dataDir, profilerDir string, i int) Service { Dockerfile: "cmd/Dockerfile", Args: map[string]string{ "TARGET": fmt.Sprintf("./cmd/%s", role), - "VERSION": build.Semver(), + "VERSION": build.Version(), "COMMIT": build.Commit(), "GOARCH": runtime.GOARCH, }, diff --git a/integration/localnet/client/Dockerfile b/integration/localnet/client/Dockerfile index ac1fbb8d8e7..4908e287624 100644 --- a/integration/localnet/client/Dockerfile +++ b/integration/localnet/client/Dockerfile @@ -1,13 +1,13 @@ -FROM golang:1.17 +FROM golang:1.20 COPY flow-localnet.json /go WORKDIR /go -RUN curl -L https://github.com/onflow/flow-cli/archive/refs/tags/v0.36.2.tar.gz | tar -xzv -RUN cd flow-cli-0.36.2 && go mod download -RUN cd flow-cli-0.36.2 && make -RUN /go/flow-cli-0.36.2/cmd/flow/flow version -RUN cp /go/flow-cli-0.36.2/cmd/flow/flow /go/flow +RUN curl -L https://github.com/onflow/flow-cli/archive/refs/tags/v1.3.3.tar.gz | tar -xzv +RUN cd flow-cli-1.3.3 && go mod download +RUN cd flow-cli-1.3.3 && make +RUN /go/flow-cli-1.3.3/cmd/flow/flow version +RUN cp /go/flow-cli-1.3.3/cmd/flow/flow /go/flow CMD /go/flow -f /go/flow-localnet.json -n observer blocks get latest diff --git a/integration/testnet/client.go b/integration/testnet/client.go index ab2eb0b751e..51026702085 100644 --- a/integration/testnet/client.go +++ b/integration/testnet/client.go @@ -88,7 +88,7 @@ func NewClient(addr string, chain flow.Chain) (*Client, error) { //if err != nil { // return nil, fmt.Errorf("cannot marshal key json: %w", err) //} - + // //fmt.Printf("New client with private key: \n%s\n", json) //fmt.Printf("and public key: \n%s\n", publicJson) diff --git a/integration/testnet/container.go b/integration/testnet/container.go index 04b26f17092..2ee74894ac1 100644 --- a/integration/testnet/container.go +++ b/integration/testnet/container.go @@ -391,6 +391,7 @@ func (c *Container) OpenState() (*state.State, error) { setups := storage.NewEpochSetups(metrics, db) commits := storage.NewEpochCommits(metrics, db) statuses := storage.NewEpochStatuses(metrics, db) + versionBeacons := storage.NewVersionBeacons(db) return state.OpenState( metrics, @@ -403,6 +404,7 @@ func (c *Container) OpenState() (*state.State, error) { setups, commits, statuses, + versionBeacons, ) } diff --git a/integration/testnet/network.go b/integration/testnet/network.go index 8c797838164..8b9522d7ba6 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" gonet "net" + "net/http" "os" "path/filepath" "sort" @@ -18,13 +19,15 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" dockerclient "github.com/docker/docker/client" - "github.com/onflow/cadence" + io_prometheus_client "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/cadence" + "github.com/onflow/flow-go-sdk/crypto" "github.com/onflow/flow-go/cmd/bootstrap/dkg" "github.com/onflow/flow-go/cmd/bootstrap/run" "github.com/onflow/flow-go/cmd/bootstrap/utils" @@ -317,9 +320,9 @@ func (net *FlowNetwork) ContainerByName(name string) *Container { func (net *FlowNetwork) PrintPorts() { var builder strings.Builder builder.WriteString("endpoints by container name:\n") - for containerName, container := range net.Containers { - builder.WriteString(fmt.Sprintf("\t%s\n", containerName)) - for portName, port := range container.Ports { + for cName, c := range net.Containers { + builder.WriteString(fmt.Sprintf("\t%s\n", cName)) + for portName, port := range c.Ports { switch portName { case MetricsPort: builder.WriteString(fmt.Sprintf("\t\t%s: localhost:%s/metrics\n", portName, port)) @@ -331,6 +334,57 @@ func (net *FlowNetwork) PrintPorts() { fmt.Print(builder.String()) } +// PortsByContainerName returns the specified port for each container in the network. +// Args: +// - portName: name of the port. +// - withGhost: when set to true will include urls's for ghost containers, otherwise ghost containers will be filtered. +// +// Returns: +// - map[string]string: a map of container name to the specified port on the host machine. +func (net *FlowNetwork) PortsByContainerName(portName string, withGhost bool) map[string]string { + portsByContainer := make(map[string]string) + for cName, c := range net.Containers { + if !withGhost && c.Config.Ghost { + continue + } + portsByContainer[cName] = c.Ports[portName] + } + return portsByContainer +} + +// GetMetricFromContainers returns the specified metric for all containers. +// Args: +// +// t: testing pointer +// metricName: name of the metric +// metricsURLs: map of container name to metrics url +// +// Returns: +// +// map[string][]*io_prometheus_client.Metric map of container name to metric result. +func (net *FlowNetwork) GetMetricFromContainers(t *testing.T, metricName string, metricsURLs map[string]string) map[string][]*io_prometheus_client.Metric { + allMetrics := make(map[string][]*io_prometheus_client.Metric, len(metricsURLs)) + for containerName, metricsURL := range metricsURLs { + allMetrics[containerName] = net.GetMetricFromContainer(t, containerName, metricsURL, metricName) + } + return allMetrics +} + +// GetMetricFromContainer makes an HTTP GET request to the metrics url and returns the metric families for each container. +func (net *FlowNetwork) GetMetricFromContainer(t *testing.T, containerName, metricsURL, metricName string) []*io_prometheus_client.Metric { + // download root snapshot from provided URL + res, err := http.Get(metricsURL) + require.NoError(t, err, fmt.Sprintf("failed to get metrics for container %s at url %s: %s", containerName, metricsURL, err)) + defer res.Body.Close() + + var parser expfmt.TextParser + mf, err := parser.TextToMetricFamilies(res.Body) + require.NoError(t, err, fmt.Sprintf("failed to parse metrics for container %s at url %s: %s", containerName, metricsURL, err)) + m, ok := mf[metricName] + require.True(t, ok, "failed to get metric %s for container %s at url %s metric does not exist", metricName, containerName, metricsURL) + return m.GetMetric() +} + type ConsensusFollowerConfig struct { NodeID flow.Identifier NetworkingPrivKey crypto.PrivateKey @@ -714,6 +768,9 @@ func (net *FlowNetwork) addObserver(t *testing.T, conf ObserverConfig) { nodeContainer.exposePort(AdminPort, testingdock.RandomPort(t)) nodeContainer.AddFlag("admin-addr", nodeContainer.ContainerAddr(AdminPort)) + nodeContainer.exposePort(RESTPort, testingdock.RandomPort(t)) + nodeContainer.AddFlag("rest-addr", nodeContainer.ContainerAddr(RESTPort)) + nodeContainer.opts.HealthCheck = testingdock.HealthCheckCustom(nodeContainer.HealthcheckCallback()) suiteContainer := net.suite.Container(containerOpts) @@ -961,7 +1018,6 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl // Sort so that access nodes start up last sort.Sort(&networkConf) - // generate staking and networking keys for each configured node stakedConfs, err := setupKeys(networkConf) if err != nil { @@ -1184,7 +1240,6 @@ func setupKeys(networkConf NetworkConfig) ([]ContainerConfig, error) { // create node container configs and corresponding public identities confs := make([]ContainerConfig, 0, nNodes) for i, conf := range networkConf.Nodes { - // define the node's name _ and address : name := fmt.Sprintf("%s_%d", conf.Role.String(), roleCounter[conf.Role]+1) @@ -1201,13 +1256,14 @@ func setupKeys(networkConf NetworkConfig) ([]ContainerConfig, error) { ) containerConf := ContainerConfig{ - NodeInfo: info, - ContainerName: name, - LogLevel: conf.LogLevel, - Ghost: conf.Ghost, - AdditionalFlags: conf.AdditionalFlags, - Debug: conf.Debug, - Corrupted: conf.Corrupted, + NodeInfo: info, + ContainerName: name, + LogLevel: conf.LogLevel, + Ghost: conf.Ghost, + AdditionalFlags: conf.AdditionalFlags, + Debug: conf.Debug, + Corrupted: conf.Corrupted, + EnableMetricsServer: conf.EnableMetricsServer, } confs = append(confs, containerConf) @@ -1231,7 +1287,7 @@ func runBeaconKG(confs []ContainerConfig) (dkgmod.DKGData, error) { return dkgmod.DKGData{}, err } - dkg, err := dkg.RunFastKG(nConsensusNodes, dkgSeed) + dkg, err := dkg.RandomBeaconKG(nConsensusNodes, dkgSeed) if err != nil { return dkgmod.DKGData{}, err } diff --git a/integration/tests/access/access_circuit_breaker_test.go b/integration/tests/access/access_circuit_breaker_test.go new file mode 100644 index 00000000000..569119c8469 --- /dev/null +++ b/integration/tests/access/access_circuit_breaker_test.go @@ -0,0 +1,184 @@ +package access + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + sdk "github.com/onflow/flow-go-sdk" + sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go-sdk/templates" + "github.com/onflow/flow-go/integration/testnet" + "github.com/onflow/flow-go/integration/tests/lib" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestAccessCircuitBreaker(t *testing.T) { + suite.Run(t, new(AccessCircuitBreakerSuite)) +} + +type AccessCircuitBreakerSuite struct { + suite.Suite + + log zerolog.Logger + + // root context for the current test + ctx context.Context + cancel context.CancelFunc + + net *testnet.FlowNetwork +} + +var requestTimeout = 1500 * time.Millisecond +var cbRestoreTimeout = 6 * time.Second + +func (s *AccessCircuitBreakerSuite) TearDownTest() { + s.log.Info().Msg("================> Start TearDownTest") + s.net.Remove() + s.cancel() + s.log.Info().Msg("================> Finish TearDownTest") +} + +func (s *AccessCircuitBreakerSuite) SetupTest() { + s.log = unittest.LoggerForTest(s.Suite.T(), zerolog.InfoLevel) + s.log.Info().Msg("================> SetupTest") + defer func() { + s.log.Info().Msg("================> Finish SetupTest") + }() + + // need one access node with enabled circuit breaker + nodeConfigs := []testnet.NodeConfig{ + testnet.NewNodeConfig( + flow.RoleAccess, + testnet.WithLogLevel(zerolog.InfoLevel), + testnet.WithAdditionalFlag("--circuit-breaker-enabled=true"), + testnet.WithAdditionalFlag(fmt.Sprintf("--circuit-breaker-restore-timeout=%s", cbRestoreTimeout.String())), + testnet.WithAdditionalFlag("--circuit-breaker-max-requests=1"), + testnet.WithAdditionalFlag("--circuit-breaker-max-failures=1"), + testnet.WithAdditionalFlag(fmt.Sprintf("--collection-client-timeout=%s", requestTimeout.String())), + ), + } + // need one execution node + exeConfig := testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel)) + nodeConfigs = append(nodeConfigs, exeConfig) + + // need one dummy verification node (unused ghost) + verConfig := testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.FatalLevel), testnet.AsGhost()) + nodeConfigs = append(nodeConfigs, verConfig) + + // need one controllable collection node + collConfig := testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--hotstuff-proposal-duration=100ms")) + nodeConfigs = append(nodeConfigs, collConfig) + + // need three consensus nodes (unused ghost) + for n := 0; n < 3; n++ { + conID := unittest.IdentifierFixture() + nodeConfig := testnet.NewNodeConfig(flow.RoleConsensus, + testnet.WithLogLevel(zerolog.FatalLevel), + testnet.WithID(conID), + testnet.AsGhost()) + nodeConfigs = append(nodeConfigs, nodeConfig) + } + + conf := testnet.NewNetworkConfig("access_api_test", nodeConfigs) + s.net = testnet.PrepareFlowNetwork(s.T(), conf, flow.Localnet) + + // start the network + s.T().Logf("starting flow network with docker containers") + s.ctx, s.cancel = context.WithCancel(context.Background()) + + s.net.Start(s.ctx) +} + +// TestCircuitBreaker tests the behavior of the circuit breaker. It verifies the circuit breaker's ability to open, +// prevent further requests, and restore after a timeout. It is done in a few steps: +// 1. Get the collection node and disconnect it from the network. +// 2. Try to send a transaction multiple times to observe the decrease in waiting time for a failed response. +// 3. Connect the collection node to the network and wait for the circuit breaker restore time. +// 4. Successfully send a transaction. +func (s *AccessCircuitBreakerSuite) TestCircuitBreaker() { + // 1. Get the collection node + collectionContainer := s.net.ContainerByName("collection_1") + + // 2. Get the Access Node container and client + accessContainer := s.net.ContainerByName(testnet.PrimaryAN) + + // Check if access node was created with circuit breaker flags + require.True(s.T(), accessContainer.IsFlagSet("circuit-breaker-enabled")) + require.True(s.T(), accessContainer.IsFlagSet("circuit-breaker-restore-timeout")) + require.True(s.T(), accessContainer.IsFlagSet("circuit-breaker-max-requests")) + require.True(s.T(), accessContainer.IsFlagSet("circuit-breaker-max-failures")) + + accessClient, err := accessContainer.TestnetClient() + require.NoError(s.T(), err, "failed to get access node client") + require.NotNil(s.T(), accessClient, "failed to get access node client") + + latestBlockID, err := accessClient.GetLatestBlockID(s.ctx) + require.NoError(s.T(), err) + + // Create a new account to deploy Counter to + accountPrivateKey := lib.RandomPrivateKey() + + accountKey := sdk.NewAccountKey(). + FromPrivateKey(accountPrivateKey). + SetHashAlgo(sdkcrypto.SHA3_256). + SetWeight(sdk.AccountKeyWeightThreshold) + + serviceAddress := sdk.Address(accessClient.Chain.ServiceAddress()) + + // Generate the account creation transaction + createAccountTx, err := templates.CreateAccount( + []*sdk.AccountKey{accountKey}, + []templates.Contract{ + { + Name: lib.CounterContract.Name, + Source: lib.CounterContract.ToCadence(), + }, + }, serviceAddress) + require.NoError(s.T(), err) + + createAccountTx. + SetReferenceBlockID(sdk.Identifier(latestBlockID)). + SetProposalKey(serviceAddress, 0, accessClient.GetSeqNumber()). + SetPayer(serviceAddress). + SetGasLimit(9999) + + // Sign the transaction + signedTx, err := accessClient.SignTransaction(createAccountTx) + require.NoError(s.T(), err) + + // 3. Disconnect the collection node from the network to activate the Circuit Breaker + err = collectionContainer.Disconnect() + require.NoError(s.T(), err, "failed to pause connection node") + + // 4. Send a couple of transactions to test if the circuit breaker opens correctly + // Try to send the transaction for the first time. It should wait at least the timeout time and return Unavailable error + err = accessClient.SendTransaction(s.ctx, signedTx) + assert.Equal(s.T(), codes.Unavailable, status.Code(err)) + + // Try to send the transaction for the second time. It should wait less than a second because the circuit breaker + // is configured to break after the first failure + err = accessClient.SendTransaction(s.ctx, signedTx) + //Here we catch the codes.Unknown error, as this is the one that comes from the Circuit Breaker when the state is Open. + assert.Equal(s.T(), codes.Unknown, status.Code(err)) + + // Reconnect the collection node + err = collectionContainer.Connect() + require.NoError(s.T(), err, "failed to start collection node") + + // Wait for the circuit breaker to restore + time.Sleep(cbRestoreTimeout) + + // Try to send the transaction for the third time. The transaction should be sent successfully + err = accessClient.SendTransaction(s.ctx, signedTx) + require.NoError(s.T(), err, "transaction should be sent") +} diff --git a/integration/tests/access/access_test.go b/integration/tests/access/access_test.go index e7d34cc6424..82d268d9a65 100644 --- a/integration/tests/access/access_test.go +++ b/integration/tests/access/access_test.go @@ -6,6 +6,10 @@ import ( "testing" "time" + "github.com/onflow/flow-go/consensus/hotstuff/committees" + "github.com/onflow/flow-go/consensus/hotstuff/signature" + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -112,3 +116,84 @@ func (s *AccessSuite) TestAPIsAvailable() { assert.NoError(t, err, "failed to ping access node") }) } + +// TestSignerIndicesDecoding tests that access node uses signer indices' decoder to correctly parse encoded data in blocks. +// This test receives blocks from consensus follower and then requests same blocks from access API and checks if returned data +// matches. +func (s *AccessSuite) TestSignerIndicesDecoding() { + + container := s.net.ContainerByName(testnet.PrimaryAN) + + ctx, cancel := context.WithCancel(s.ctx) + defer cancel() + + // create access API + grpcAddress := container.Addr(testnet.GRPCPort) + conn, err := grpc.DialContext(ctx, grpcAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(s.T(), err, "failed to connect to access node") + defer conn.Close() + + client := accessproto.NewAccessAPIClient(conn) + + // query latest finalized block + latestFinalizedBlock, err := makeApiRequest(client.GetLatestBlockHeader, ctx, &accessproto.GetLatestBlockHeaderRequest{ + IsSealed: false, + }) + require.NoError(s.T(), err) + + blockByID, err := makeApiRequest(client.GetBlockHeaderByID, ctx, &accessproto.GetBlockHeaderByIDRequest{Id: latestFinalizedBlock.Block.Id}) + require.NoError(s.T(), err) + + require.Equal(s.T(), latestFinalizedBlock, blockByID, "expect to receive same block by ID") + + blockByHeight, err := makeApiRequest(client.GetBlockHeaderByHeight, ctx, + &accessproto.GetBlockHeaderByHeightRequest{Height: latestFinalizedBlock.Block.Height}) + require.NoError(s.T(), err) + + require.Equal(s.T(), blockByID, blockByHeight, "expect to receive same block by height") + + // stop container, so we can access it's state and perform assertions + err = s.net.StopContainerByName(ctx, testnet.PrimaryAN) + require.NoError(s.T(), err) + + err = container.WaitForContainerStopped(5 * time.Second) + require.NoError(s.T(), err) + + // open state to build a block singer decoder + state, err := container.OpenState() + require.NoError(s.T(), err) + + // create committee so we can create decoder to assert validity of data + committee, err := committees.NewConsensusCommittee(state, container.Config.NodeID) + require.NoError(s.T(), err) + blockSignerDecoder := signature.NewBlockSignerDecoder(committee) + + expectedFinalizedBlock, err := state.AtBlockID(flow.HashToID(latestFinalizedBlock.Block.Id)).Head() + require.NoError(s.T(), err) + + // since all blocks should be equal we will execute just check on one of them + require.Equal(s.T(), latestFinalizedBlock.Block.ParentVoterIndices, expectedFinalizedBlock.ParentVoterIndices) + + // check if the response contains valid encoded signer IDs. + msg := latestFinalizedBlock.Block + block, err := convert.MessageToBlockHeader(msg) + require.NoError(s.T(), err) + decodedIdentities, err := blockSignerDecoder.DecodeSignerIDs(block) + require.NoError(s.T(), err) + // transform to assert + var transformed [][]byte + for _, identity := range decodedIdentities { + identity := identity + transformed = append(transformed, identity[:]) + } + assert.ElementsMatch(s.T(), transformed, msg.ParentVoterIds, "response must contain correctly encoded signer IDs") +} + +// makeApiRequest is a helper function that encapsulates context creation for grpc client call, used to avoid repeated creation +// of new context for each call. +func makeApiRequest[Func func(context.Context, *Req, ...grpc.CallOption) (*Resp, error), Req any, Resp any](apiCall Func, ctx context.Context, req *Req) (*Resp, error) { + clientCtx, cancel := context.WithTimeout(ctx, 1*time.Second) + resp, err := apiCall(clientCtx, req) + cancel() + return resp, err +} diff --git a/integration/tests/access/consensus_follower_test.go b/integration/tests/access/consensus_follower_test.go index 7bde1a794d8..b29a76c67af 100644 --- a/integration/tests/access/consensus_follower_test.go +++ b/integration/tests/access/consensus_follower_test.go @@ -48,33 +48,33 @@ func (s *ConsensusFollowerSuite) TearDownTest() { s.log.Info().Msgf("================> Finish TearDownTest") } -func (suite *ConsensusFollowerSuite) SetupTest() { - suite.log = unittest.LoggerForTest(suite.Suite.T(), zerolog.InfoLevel) - suite.log.Info().Msg("================> SetupTest") - suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.buildNetworkConfig() +func (s *ConsensusFollowerSuite) SetupTest() { + s.log = unittest.LoggerForTest(s.Suite.T(), zerolog.InfoLevel) + s.log.Info().Msg("================> SetupTest") + s.ctx, s.cancel = context.WithCancel(context.Background()) + s.buildNetworkConfig() // start the network - suite.net.Start(suite.ctx) + s.net.Start(s.ctx) } // TestReceiveBlocks tests the following // 1. The consensus follower follows the chain and persists blocks in storage. // 2. The consensus follower can catch up if it is started after the chain has started producing blocks. -func (suite *ConsensusFollowerSuite) TestReceiveBlocks() { - ctx, cancel := context.WithCancel(suite.ctx) +func (s *ConsensusFollowerSuite) TestReceiveBlocks() { + ctx, cancel := context.WithCancel(s.ctx) defer cancel() receivedBlocks := make(map[flow.Identifier]struct{}, blockCount) - suite.Run("consensus follower follows the chain", func() { + s.Run("consensus follower follows the chain", func() { // kick off the first follower - suite.followerMgr1.startFollower(ctx) + s.followerMgr1.startFollower(ctx) var err error receiveBlocks := func() { for i := 0; i < blockCount; i++ { - blockID := <-suite.followerMgr1.blockIDChan + blockID := <-s.followerMgr1.blockIDChan receivedBlocks[blockID] = struct{}{} - _, err = suite.followerMgr1.getBlock(blockID) + _, err = s.followerMgr1.getBlock(blockID) if err != nil { return } @@ -82,18 +82,18 @@ func (suite *ConsensusFollowerSuite) TestReceiveBlocks() { } // wait for finalized blocks - unittest.AssertReturnsBefore(suite.T(), receiveBlocks, 2*time.Minute) // waiting 2 minute for 5 blocks + unittest.AssertReturnsBefore(s.T(), receiveBlocks, 2*time.Minute) // waiting 2 minute for 5 blocks // all blocks were found in the storage - require.NoError(suite.T(), err, "finalized block not found in storage") + require.NoError(s.T(), err, "finalized block not found in storage") // assert that blockCount number of blocks were received - require.Len(suite.T(), receivedBlocks, blockCount) + require.Len(s.T(), receivedBlocks, blockCount) }) - suite.Run("consensus follower sync up with the chain", func() { + s.Run("consensus follower sync up with the chain", func() { // kick off the second follower - suite.followerMgr2.startFollower(ctx) + s.followerMgr2.startFollower(ctx) // the second follower is now atleast blockCount blocks behind and should sync up and get all the missed blocks receiveBlocks := func() { @@ -101,7 +101,7 @@ func (suite *ConsensusFollowerSuite) TestReceiveBlocks() { select { case <-ctx.Done(): return - case blockID := <-suite.followerMgr2.blockIDChan: + case blockID := <-s.followerMgr2.blockIDChan: delete(receivedBlocks, blockID) if len(receivedBlocks) == 0 { return @@ -110,17 +110,18 @@ func (suite *ConsensusFollowerSuite) TestReceiveBlocks() { } } // wait for finalized blocks - unittest.AssertReturnsBefore(suite.T(), receiveBlocks, 2*time.Minute) // waiting 2 minute for the missing 5 blocks + unittest.AssertReturnsBefore(s.T(), receiveBlocks, 2*time.Minute) // waiting 2 minute for the missing 5 blocks }) } -func (suite *ConsensusFollowerSuite) buildNetworkConfig() { +func (s *ConsensusFollowerSuite) buildNetworkConfig() { // staked access node - suite.stakedID = unittest.IdentifierFixture() + unittest.IdentityFixture() + s.stakedID = unittest.IdentifierFixture() stakedConfig := testnet.NewNodeConfig( flow.RoleAccess, - testnet.WithID(suite.stakedID), + testnet.WithID(s.stakedID), testnet.WithAdditionalFlag("--supports-observer=true"), testnet.WithLogLevel(zerolog.WarnLevel), ) @@ -131,7 +132,7 @@ func (suite *ConsensusFollowerSuite) buildNetworkConfig() { } consensusConfigs := []func(config *testnet.NodeConfig){ - testnet.WithAdditionalFlag("--block-rate-delay=100ms"), + testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=100ms"), testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", 1)), testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", 1)), testnet.WithLogLevel(zerolog.FatalLevel), @@ -151,26 +152,26 @@ func (suite *ConsensusFollowerSuite) buildNetworkConfig() { } unstakedKey1, err := UnstakedNetworkingKey() - require.NoError(suite.T(), err) + require.NoError(s.T(), err) unstakedKey2, err := UnstakedNetworkingKey() - require.NoError(suite.T(), err) + require.NoError(s.T(), err) followerConfigs := []testnet.ConsensusFollowerConfig{ - testnet.NewConsensusFollowerConfig(suite.T(), unstakedKey1, suite.stakedID, consensus_follower.WithLogLevel("warn")), - testnet.NewConsensusFollowerConfig(suite.T(), unstakedKey2, suite.stakedID, consensus_follower.WithLogLevel("warn")), + testnet.NewConsensusFollowerConfig(s.T(), unstakedKey1, s.stakedID, consensus_follower.WithLogLevel("warn")), + testnet.NewConsensusFollowerConfig(s.T(), unstakedKey2, s.stakedID, consensus_follower.WithLogLevel("warn")), } // consensus followers conf := testnet.NewNetworkConfig("consensus follower test", net, testnet.WithConsensusFollowers(followerConfigs...)) - suite.net = testnet.PrepareFlowNetwork(suite.T(), conf, flow.Localnet) + s.net = testnet.PrepareFlowNetwork(s.T(), conf, flow.Localnet) - follower1 := suite.net.ConsensusFollowerByID(followerConfigs[0].NodeID) - suite.followerMgr1, err = newFollowerManager(suite.T(), follower1) - require.NoError(suite.T(), err) + follower1 := s.net.ConsensusFollowerByID(followerConfigs[0].NodeID) + s.followerMgr1, err = newFollowerManager(s.T(), follower1) + require.NoError(s.T(), err) - follower2 := suite.net.ConsensusFollowerByID(followerConfigs[1].NodeID) - suite.followerMgr2, err = newFollowerManager(suite.T(), follower2) - require.NoError(suite.T(), err) + follower2 := s.net.ConsensusFollowerByID(followerConfigs[1].NodeID) + s.followerMgr2, err = newFollowerManager(s.T(), follower2) + require.NoError(s.T(), err) } // TODO: Move this to unittest and resolve the circular dependency issue diff --git a/integration/tests/access/execution_state_sync_test.go b/integration/tests/access/execution_state_sync_test.go index b75b45704f9..078441010d0 100644 --- a/integration/tests/access/execution_state_sync_test.go +++ b/integration/tests/access/execution_state_sync_test.go @@ -24,7 +24,6 @@ import ( ) func TestExecutionStateSync(t *testing.T) { - unittest.SkipUnless(t, unittest.TEST_FLAKY, "flaky as it constantly runs into badger errors or blob not found errors") suite.Run(t, new(ExecutionStateSyncSuite)) } @@ -92,7 +91,7 @@ func (s *ExecutionStateSyncSuite) buildNetworkConfig() { testnet.AsGhost()) consensusConfigs := []func(config *testnet.NodeConfig){ - testnet.WithAdditionalFlag("--block-rate-delay=100ms"), + testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=100ms"), testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", 1)), testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", 1)), testnet.WithLogLevel(zerolog.FatalLevel), @@ -158,7 +157,7 @@ func (s *ExecutionStateSyncSuite) TestHappyPath() { s.T().Logf("getting execution data for height %d, block %s, execution_data %s", header.Height, header.ID(), result.ExecutionDataID) - ed, err := eds.GetExecutionData(s.ctx, result.ExecutionDataID) + ed, err := eds.Get(s.ctx, result.ExecutionDataID) if assert.NoError(s.T(), err, "could not get execution data for height %v", i) { s.T().Logf("got execution data for height %d", i) assert.Equal(s.T(), header.ID(), ed.BlockID) diff --git a/integration/tests/access/observer_test.go b/integration/tests/access/observer_test.go index 29b96da49e6..25bfeab2f3a 100644 --- a/integration/tests/access/observer_test.go +++ b/integration/tests/access/observer_test.go @@ -1,22 +1,29 @@ package access import ( + "bytes" "context" + "encoding/json" + "fmt" + "net/http" "testing" - "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/integration/testnet" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) func TestObserver(t *testing.T) { @@ -25,9 +32,10 @@ func TestObserver(t *testing.T) { type ObserverSuite struct { suite.Suite - net *testnet.FlowNetwork - teardown func() - local map[string]struct{} + net *testnet.FlowNetwork + teardown func() + localRpc map[string]struct{} + localRest map[string]struct{} cancel context.CancelFunc } @@ -44,7 +52,7 @@ func (s *ObserverSuite) TearDownTest() { } func (s *ObserverSuite) SetupTest() { - s.local = map[string]struct{}{ + s.localRpc = map[string]struct{}{ "Ping": {}, "GetLatestBlockHeader": {}, "GetBlockHeaderByID": {}, @@ -56,18 +64,26 @@ func (s *ObserverSuite) SetupTest() { "GetNetworkParameters": {}, } + s.localRest = map[string]struct{}{ + "getBlocksByIDs": {}, + "getBlocksByHeight": {}, + "getBlockPayloadByID": {}, + "getNetworkParameters": {}, + "getNodeVersionInfo": {}, + } + nodeConfigs := []testnet.NodeConfig{ // access node with unstaked nodes supported testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.InfoLevel), testnet.WithAdditionalFlag("--supports-observer=true")), - // need one dummy execution node (unused ghost) - testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel), testnet.AsGhost()), + // need one dummy execution node + testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel)), // need one dummy verification node (unused ghost) testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.FatalLevel), testnet.AsGhost()), - // need one controllable collection node (unused ghost) - testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.AsGhost()), + // need one controllable collection node + testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel)), // need three consensus nodes (unused ghost) testnet.NewNodeConfig(flow.RoleConsensus, testnet.WithLogLevel(zerolog.FatalLevel), testnet.AsGhost()), @@ -90,11 +106,11 @@ func (s *ObserverSuite) SetupTest() { s.net.Start(ctx) } -// TestObserver runs the following tests: +// TestObserverRPC runs the following tests: // 1. CompareRPCs: verifies that the observer client returns the same errors as the access client for rpcs proxied to the upstream AN // 2. HandledByUpstream: stops the upstream AN and verifies that the observer client returns errors for all rpcs handled by the upstream // 3. HandledByObserver: stops the upstream AN and verifies that the observer client handles all other queries -func (s *ObserverSuite) TestObserver() { +func (s *ObserverSuite) TestObserverRPC() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -111,7 +127,7 @@ func (s *ObserverSuite) TestObserver() { // verify that both clients return the same errors for proxied rpcs for _, rpc := range s.getRPCs() { // skip rpcs handled locally by observer - if _, local := s.local[rpc.name]; local { + if _, local := s.localRpc[rpc.name]; local { continue } t.Run(rpc.name, func(t *testing.T) { @@ -129,7 +145,7 @@ func (s *ObserverSuite) TestObserver() { t.Run("HandledByUpstream", func(t *testing.T) { // verify that we receive Unavailable errors from all rpcs handled upstream for _, rpc := range s.getRPCs() { - if _, local := s.local[rpc.name]; local { + if _, local := s.localRpc[rpc.name]; local { continue } t.Run(rpc.name, func(t *testing.T) { @@ -142,7 +158,7 @@ func (s *ObserverSuite) TestObserver() { t.Run("HandledByObserver", func(t *testing.T) { // verify that we receive NotFound or no error from all rpcs handled locally for _, rpc := range s.getRPCs() { - if _, local := s.local[rpc.name]; !local { + if _, local := s.localRpc[rpc.name]; !local { continue } t.Run(rpc.name, func(t *testing.T) { @@ -154,7 +170,90 @@ func (s *ObserverSuite) TestObserver() { }) } }) +} + +// TestObserverRest runs the following tests: +// 1. CompareEndpoints: verifies that the observer client returns the same errors as the access client for rests proxied to the upstream AN +// 2. HandledByUpstream: stops the upstream AN and verifies that the observer client returns errors for all rests handled by the upstream +// 3. HandledByObserver: stops the upstream AN and verifies that the observer client handles all other queries +func (s *ObserverSuite) TestObserverRest() { + t := s.T() + + accessAddr := s.net.ContainerByName(testnet.PrimaryAN).Addr(testnet.RESTPort) + observerAddr := s.net.ContainerByName("observer_1").Addr(testnet.RESTPort) + + httpClient := http.DefaultClient + makeHttpCall := func(method string, url string, body interface{}) (*http.Response, error) { + switch method { + case http.MethodGet: + return httpClient.Get(url) + case http.MethodPost: + jsonBody, _ := json.Marshal(body) + return httpClient.Post(url, "application/json", bytes.NewBuffer(jsonBody)) + } + panic("not supported") + } + makeObserverCall := func(method string, path string, body interface{}) (*http.Response, error) { + return makeHttpCall(method, "http://"+observerAddr+"/v1"+path, body) + } + makeAccessCall := func(method string, path string, body interface{}) (*http.Response, error) { + return makeHttpCall(method, "http://"+accessAddr+"/v1"+path, body) + } + + t.Run("CompareEndpoints", func(t *testing.T) { + // verify that both clients return the same errors for proxied rests + for _, endpoint := range s.getRestEndpoints() { + // skip rest handled locally by observer + if _, local := s.localRest[endpoint.name]; local { + continue + } + t.Run(endpoint.name, func(t *testing.T) { + accessResp, accessErr := makeAccessCall(endpoint.method, endpoint.path, endpoint.body) + observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path, endpoint.body) + assert.NoError(t, accessErr) + assert.NoError(t, observerErr) + assert.Equal(t, accessResp.Status, observerResp.Status) + assert.Equal(t, accessResp.StatusCode, observerResp.StatusCode) + assert.Contains(t, [...]int{ + http.StatusNotFound, + http.StatusOK, + }, observerResp.StatusCode) + }) + } + }) + + // stop the upstream access container + err := s.net.StopContainerByName(context.Background(), testnet.PrimaryAN) + require.NoError(t, err) + + t.Run("HandledByUpstream", func(t *testing.T) { + // verify that we receive StatusInternalServerError, StatusServiceUnavailable errors from all rests handled upstream + for _, endpoint := range s.getRestEndpoints() { + if _, local := s.localRest[endpoint.name]; local { + continue + } + t.Run(endpoint.name, func(t *testing.T) { + observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path, endpoint.body) + require.NoError(t, observerErr) + assert.Contains(t, [...]int{ + http.StatusServiceUnavailable}, observerResp.StatusCode) + }) + } + }) + t.Run("HandledByObserver", func(t *testing.T) { + // verify that we receive NotFound or no error from all rests handled locally + for _, endpoint := range s.getRestEndpoints() { + if _, local := s.localRest[endpoint.name]; !local { + continue + } + t.Run(endpoint.name, func(t *testing.T) { + observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path, endpoint.body) + require.NoError(t, observerErr) + assert.Contains(t, [...]int{http.StatusNotFound, http.StatusOK}, observerResp.StatusCode) + }) + } + }) } func (s *ObserverSuite) getAccessClient() (accessproto.AccessAPIClient, error) { @@ -287,3 +386,126 @@ func (s *ObserverSuite) getRPCs() []RPCTest { }}, } } + +type RestEndpointTest struct { + name string + method string + path string + body interface{} +} + +func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { + transactionId := unittest.IdentifierFixture().String() + account := flow.Localnet.Chain().ServiceAddress().String() + block := unittest.BlockFixture() + executionResult := unittest.ExecutionResultFixture() + collection := unittest.CollectionFixture(2) + eventType := "A.0123456789abcdef.flow.event" + + return []RestEndpointTest{ + { + name: "getTransactionByID", + method: http.MethodGet, + path: "/transactions/" + transactionId, + }, + { + name: "createTransaction", + method: http.MethodPost, + path: "/transactions", + body: createTx(s.net), + }, + { + name: "getTransactionResultByID", + method: http.MethodGet, + path: fmt.Sprintf("/transaction_results/%s?block_id=%s&collection_id=%s", transactionId, block.ID().String(), collection.ID().String()), + }, + { + name: "getBlocksByIDs", + method: http.MethodGet, + path: "/blocks/" + block.ID().String(), + }, + { + name: "getBlocksByHeight", + method: http.MethodGet, + path: "/blocks?height=1", + }, + { + name: "getBlockPayloadByID", + method: http.MethodGet, + path: "/blocks/" + block.ID().String() + "/payload", + }, + { + name: "getExecutionResultByID", + method: http.MethodGet, + path: "/execution_results/" + executionResult.ID().String(), + }, + { + name: "getExecutionResultByBlockID", + method: http.MethodGet, + path: "/execution_results?block_id=" + block.ID().String(), + }, + { + name: "getCollectionByID", + method: http.MethodGet, + path: "/collections/" + collection.ID().String(), + }, + { + name: "executeScript", + method: http.MethodPost, + path: "/scripts", + body: createScript(), + }, + { + name: "getAccount", + method: http.MethodGet, + path: "/accounts/" + account + "?block_height=1", + }, + { + name: "getEvents", + method: http.MethodGet, + path: fmt.Sprintf("/events?type=%s&start_height=%d&end_height=%d", eventType, 0, 3), + }, + { + name: "getNetworkParameters", + method: http.MethodGet, + path: "/network/parameters", + }, + { + name: "getNodeVersionInfo", + method: http.MethodGet, + path: "/node_version_info", + }, + } +} + +func createTx(net *testnet.FlowNetwork) interface{} { + flowAddr := flow.Localnet.Chain().ServiceAddress() + payloadSignature := unittest.TransactionSignatureFixture() + envelopeSignature := unittest.TransactionSignatureFixture() + + payloadSignature.Address = flowAddr + + envelopeSignature.Address = flowAddr + envelopeSignature.KeyIndex = 2 + + tx := flow.NewTransactionBody(). + AddAuthorizer(flowAddr). + SetPayer(flowAddr). + SetScript(unittest.NoopTxScript()). + SetReferenceBlockID(net.Root().ID()). + SetProposalKey(flowAddr, 1, 0) + tx.PayloadSignatures = []flow.TransactionSignature{payloadSignature} + tx.EnvelopeSignatures = []flow.TransactionSignature{envelopeSignature} + + return unittest.CreateSendTxHttpPayload(*tx) +} + +func createScript() interface{} { + validCode := []byte(`pub fun main(foo: String): String { return foo }`) + validArgs := []byte(`{ "type": "String", "value": "hello world" }`) + body := map[string]interface{}{ + "script": util.ToBase64(validCode), + "arguments": []string{util.ToBase64(validArgs)}, + } + return body +} diff --git a/integration/tests/bft/base_suite.go b/integration/tests/bft/base_suite.go index a1942f05b7d..2ce0d7f6fdc 100644 --- a/integration/tests/bft/base_suite.go +++ b/integration/tests/bft/base_suite.go @@ -28,7 +28,6 @@ type BaseSuite struct { GhostID flow.Identifier // represents id of ghost node NodeConfigs testnet.NodeConfigs // used to keep configuration of nodes in testnet OrchestratorNetwork *orchestrator.Network - BlockRateFlag string } // Ghost returns a client to interact with the Ghost node on testnet. @@ -48,7 +47,6 @@ func (b *BaseSuite) AccessClient() *testnet.Client { // SetupSuite sets up node configs to run a bare minimum Flow network to function correctly. func (b *BaseSuite) SetupSuite() { b.Log = unittest.LoggerForTest(b.Suite.T(), zerolog.InfoLevel) - b.BlockRateFlag = "--block-rate-delay=1ms" // setup access nodes b.NodeConfigs = append(b.NodeConfigs, @@ -63,7 +61,7 @@ func (b *BaseSuite) SetupSuite() { testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--required-verification-seal-approvals=1"), testnet.WithAdditionalFlag("--required-construction-seal-approvals=1"), - testnet.WithAdditionalFlag(b.BlockRateFlag), + testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=1ms"), ) b.NodeConfigs = append(b.NodeConfigs, nodeConfig) } @@ -82,8 +80,8 @@ func (b *BaseSuite) SetupSuite() { // setup collection nodes b.NodeConfigs = append(b.NodeConfigs, - testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag(b.BlockRateFlag)), - testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag(b.BlockRateFlag)), + testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms")), + testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms")), ) // Ghost Node @@ -102,7 +100,10 @@ func (b *BaseSuite) SetupSuite() { func (b *BaseSuite) TearDownSuite() { b.Net.Remove() b.Cancel() - unittest.RequireCloseBefore(b.T(), b.OrchestratorNetwork.Done(), 1*time.Second, "could not stop orchestrator network on time") + // check if orchestrator network is set on the base suite, not all tests use the corrupted network. + if b.OrchestratorNetwork != nil { + unittest.RequireCloseBefore(b.T(), b.OrchestratorNetwork.Done(), 1*time.Second, "could not stop orchestrator network on time") + } } // StartCorruptedNetwork starts the corrupted network with the configured node configs, this func should be used after test suite is setup. diff --git a/integration/tests/bft/gossipsub/rpc_inspector/false_positive_test.go b/integration/tests/bft/gossipsub/rpc_inspector/false_positive_test.go new file mode 100644 index 00000000000..c948b9f99e1 --- /dev/null +++ b/integration/tests/bft/gossipsub/rpc_inspector/false_positive_test.go @@ -0,0 +1,41 @@ +package rpc_inspector + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +const numOfTestAccounts = 1000 + +type GossipsubRPCInspectorFalsePositiveNotificationsTestSuite struct { + Suite +} + +func TestGossipSubRpcInspectorFalsePositiveNotifications(t *testing.T) { + suite.Run(t, new(GossipsubRPCInspectorFalsePositiveNotificationsTestSuite)) +} + +// TestGossipsubRPCInspectorFalsePositiveNotifications this test ensures that any underlying changes or updates to any of the underlying libp2p libraries +// does not result in any of the gossip sub rpc control message inspector validation rules being broken. Anytime a validation rule is broken an invalid +// control message notification is disseminated. Using this fact, this tests sets up a full flow network and submits some transactions to generate network +// activity. After some time we ensure that no invalid control message notifications are disseminated. +func (s *GossipsubRPCInspectorFalsePositiveNotificationsTestSuite) TestGossipsubRPCInspectorFalsePositiveNotifications() { + loaderLoopDuration := 5 * time.Second + loaderLoopInterval := 500 * time.Millisecond + ctx, cancel := context.WithTimeout(s.Ctx, loaderLoopDuration) + defer cancel() + // the network has started submit some transactions to create flow accounts. + // We wait for each of these transactions to be sealed ensuring we generate + // some artificial network activity. + go s.loaderLoop(ctx, numOfTestAccounts, loaderLoopInterval) + // wait 20 state commitment changes, this ensures we simulated load on network as expected. + s.waitForStateCommitments(s.Ctx, 3, time.Minute, 500*time.Millisecond) + + // ensure no node in the network has disseminated an invalid control message notification + metricName := s.inspectorNotificationQSizeMetricName() + metricsByContainer := s.Net.GetMetricFromContainers(s.T(), metricName, s.metricsUrls()) + s.ensureNoNotificationsDisseminated(metricsByContainer) +} diff --git a/integration/tests/bft/gossipsub/rpc_inspector/suite.go b/integration/tests/bft/gossipsub/rpc_inspector/suite.go new file mode 100644 index 00000000000..17f37a472d8 --- /dev/null +++ b/integration/tests/bft/gossipsub/rpc_inspector/suite.go @@ -0,0 +1,126 @@ +package rpc_inspector + +import ( + "context" + "fmt" + "time" + + io_prometheus_client "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/integration/testnet" + "github.com/onflow/flow-go/integration/tests/bft" + "github.com/onflow/flow-go/integration/utils" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" +) + +// Suite represents a test suite that sets up a full flow network. +type Suite struct { + bft.BaseSuite + client *testnet.Client +} + +// SetupSuite generates, initializes, and starts the Flow network. +func (s *Suite) SetupSuite() { + s.BaseSuite.SetupSuite() + + // enable metrics server for all nodes + for i := range s.NodeConfigs { + s.NodeConfigs[i].EnableMetricsServer = true + } + + name := "bft_control_message_validation_false_positive_test" + // short epoch lens ensure faster state commitments + stakingAuctionLen := uint64(10) + dkgPhaseLen := uint64(50) + epochLen := uint64(300) + epochCommitSafetyThreshold := uint64(50) + netConfig := testnet.NewNetworkConfigWithEpochConfig(name, s.NodeConfigs, stakingAuctionLen, dkgPhaseLen, epochLen, epochCommitSafetyThreshold) + s.Net = testnet.PrepareFlowNetwork(s.T(), netConfig, flow.BftTestnet) + + s.Ctx, s.Cancel = context.WithCancel(context.Background()) + s.Net.Start(s.Ctx) + + // starts tracking blocks by the ghost node + s.Track(s.T(), s.Ctx, s.Ghost()) + + client, err := s.Net.ContainerByName(testnet.PrimaryAN).TestnetClient() + require.NoError(s.T(), err) + s.client = client +} + +// submitSmokeTestTransaction will submit a create account transaction to smoke test network +// This ensures a single transaction can be sealed by the network. +func (s *Suite) submitSmokeTestTransaction(ctx context.Context) { + _, err := utils.CreateFlowAccount(ctx, s.client) + require.NoError(s.T(), err) +} + +// ensureNoNotificationsDisseminated ensures the metrics result for the rpc inspector notification queue cache size metric for each container is 0 +// indicating no notifications have been disseminated. +func (s *Suite) ensureNoNotificationsDisseminated(metricEndpoints map[string][]*io_prometheus_client.Metric) { + for containerName, metric := range metricEndpoints { + val := metric[0].GetGauge().GetValue() + require.Zerof(s.T(), val, fmt.Sprintf("expected inspector notification queue cache size for container %s to be 0 got %v", containerName, val)) + } +} + +// inspectorNotifQSizeMetricName returns the metric name for the rpc inspector notification queue cache size. +func (s *Suite) inspectorNotificationQSizeMetricName() string { + return fmt.Sprintf("network_hero_cache_%s_successful_write_count_total", metrics.ResourceNetworkingRpcInspectorNotificationQueue) +} + +// metricsUrls returns a list of metrics urls for each node configured on the test suite. +func (s *Suite) metricsUrls() map[string]string { + urls := make(map[string]string, 0) + for containerName, port := range s.Net.PortsByContainerName(testnet.MetricsPort, false) { + urls[containerName] = fmt.Sprintf("http://0.0.0.0:%s/metrics", port) + } + return urls +} + +// loaderLoop submits load to the network in the form of account creation on the provided interval simulating some network traffic. +func (s *Suite) loaderLoop(ctx context.Context, numOfTestAccounts int, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + go func() { + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + for i := 0; i < numOfTestAccounts; i++ { + s.submitSmokeTestTransaction(s.Ctx) + } + } + } + }() +} + +// waitStateCommitments waits for n number of state commitment changes. +func (s *Suite) waitForStateCommitments(ctx context.Context, n int, waitFor, tick time.Duration) { + prevStateComm := s.getCurrentFinalExecutionStateCommitment(ctx) + numOfStateCommChanges := 0 + require.Eventually(s.T(), func() bool { + currStateComm := s.getCurrentFinalExecutionStateCommitment(ctx) + if prevStateComm != currStateComm { + numOfStateCommChanges++ + prevStateComm = currStateComm + } + return numOfStateCommChanges >= n + }, waitFor, tick) +} + +// getCurrentFinalizedHeight returns the current finalized height. +func (s *Suite) getCurrentFinalExecutionStateCommitment(ctx context.Context) string { + snapshot, err := s.client.GetLatestProtocolSnapshot(ctx) + require.NoError(s.T(), err) + executionResult, _, err := snapshot.SealedResult() + require.NoError(s.T(), err) + sc, err := executionResult.FinalStateCommitment() + require.NoError(s.T(), err) + bz, err := sc.MarshalJSON() + require.NoError(s.T(), err) + return string(bz) +} diff --git a/integration/tests/collection/proposal_test.go b/integration/tests/collection/proposal_test.go index 778e0af1800..68ec0e67e3f 100644 --- a/integration/tests/collection/proposal_test.go +++ b/integration/tests/collection/proposal_test.go @@ -13,11 +13,9 @@ import ( "github.com/onflow/flow-go/integration/convert" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/unittest" ) func TestMultiCluster(t *testing.T) { - unittest.SkipUnless(t, unittest.TEST_FLAKY, "flaky as it often hits port already allocated, since too many containers are created") suite.Run(t, new(MultiClusterSuite)) } diff --git a/integration/tests/collection/suite.go b/integration/tests/collection/suite.go index 4349282b456..608f8cdf4fb 100644 --- a/integration/tests/collection/suite.go +++ b/integration/tests/collection/suite.go @@ -82,7 +82,7 @@ func (suite *CollectorSuite) SetupTest(name string, nNodes, nClusters uint) { } colNodes := testnet.NewNodeConfigSet(nNodes, flow.RoleCollection, testnet.WithLogLevel(zerolog.InfoLevel), - testnet.WithAdditionalFlag("--block-rate-delay=1ms"), + testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms"), ) suite.nClusters = nClusters @@ -320,8 +320,7 @@ func (suite *CollectorSuite) AwaitTransactionsIncluded(txIDs ...flow.Identifier) suite.T().Fatalf("missing transactions: %v", missing) } -// Collector returns the collector node with the given index in the -// given cluster. +// Collector returns the collector node with the given index in the given cluster. func (suite *CollectorSuite) Collector(clusterIdx, nodeIdx uint) *testnet.Container { clusters := suite.Clusters() @@ -335,8 +334,7 @@ func (suite *CollectorSuite) Collector(clusterIdx, nodeIdx uint) *testnet.Contai return suite.net.ContainerByID(node.ID()) } -// ClusterStateFor returns a cluster state instance for the collector node -// with the given ID. +// ClusterStateFor returns a cluster state instance for the collector node with the given ID. func (suite *CollectorSuite) ClusterStateFor(id flow.Identifier) *clusterstateimpl.State { myCluster, _, ok := suite.Clusters().ByNodeID(id) @@ -351,9 +349,9 @@ func (suite *CollectorSuite) ClusterStateFor(id flow.Identifier) *clusterstateim require.Nil(suite.T(), err, "could not get node db") rootQC := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(rootBlock.ID())) - clusterStateRoot, err := clusterstateimpl.NewStateRoot(rootBlock, rootQC) + clusterStateRoot, err := clusterstateimpl.NewStateRoot(rootBlock, rootQC, setup.Counter) suite.NoError(err) - clusterState, err := clusterstateimpl.OpenState(db, nil, nil, nil, clusterStateRoot.ClusterID()) + clusterState, err := clusterstateimpl.OpenState(db, nil, nil, nil, clusterStateRoot.ClusterID(), clusterStateRoot.EpochCounter()) require.NoError(suite.T(), err, "could not get cluster state") return clusterState diff --git a/integration/tests/consensus/inclusion_test.go b/integration/tests/consensus/inclusion_test.go index e36ef7dae8e..85ef3fd8046 100644 --- a/integration/tests/consensus/inclusion_test.go +++ b/integration/tests/consensus/inclusion_test.go @@ -43,8 +43,6 @@ func (is *InclusionSuite) SetupTest() { is.log = unittest.LoggerForTest(is.Suite.T(), zerolog.InfoLevel) is.log.Info().Msgf("================> SetupTest") - // seed random generator - // to collect node confiis... var nodeConfigs []testnet.NodeConfig diff --git a/integration/tests/epochs/suite.go b/integration/tests/epochs/suite.go index dc9a1d99d76..9da32c2ebf2 100644 --- a/integration/tests/epochs/suite.go +++ b/integration/tests/epochs/suite.go @@ -72,11 +72,11 @@ func (s *Suite) SetupTest() { }() collectionConfigs := []func(*testnet.NodeConfig){ - testnet.WithAdditionalFlag("--block-rate-delay=100ms"), + testnet.WithAdditionalFlag("--hotstuff-proposal-duration=100ms"), testnet.WithLogLevel(zerolog.WarnLevel)} consensusConfigs := []func(config *testnet.NodeConfig){ - testnet.WithAdditionalFlag("--block-rate-delay=100ms"), + testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=100ms"), testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", s.RequiredSealApprovals)), testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", s.RequiredSealApprovals)), testnet.WithLogLevel(zerolog.WarnLevel)} @@ -184,7 +184,6 @@ func (s *Suite) StakeNode(ctx context.Context, env templates.Environment, role f _, stakeAmount, err := s.client.TokenAmountByRole(role) require.NoError(s.T(), err) - containerName := s.getTestContainerName(role) latestBlockID, err := s.client.GetLatestBlockID(ctx) @@ -268,22 +267,6 @@ func (s *Suite) generateAccountKeys(role flow.Role) ( return } -// createAccount creates a new flow account, can be used to test staking -func (s *Suite) createAccount(ctx context.Context, - accountKey *sdk.AccountKey, - payerAccount *sdk.Account, - payer sdk.Address, -) (sdk.Address, error) { - latestBlockID, err := s.client.GetLatestBlockID(ctx) - require.NoError(s.T(), err) - - addr, err := s.client.CreateAccount(ctx, accountKey, payerAccount, payer, sdk.Identifier(latestBlockID)) - require.NoError(s.T(), err) - - payerAccount.Keys[0].SequenceNumber++ - return addr, nil -} - // removeNodeFromProtocol removes the given node from the protocol. // NOTE: assumes staking occurs in first epoch (counter 0) func (s *Suite) removeNodeFromProtocol(ctx context.Context, env templates.Environment, nodeID flow.Identifier) { @@ -364,7 +347,7 @@ func (s *Suite) SubmitSetApprovedListTx(ctx context.Context, env templates.Envir // ExecuteReadApprovedNodesScript executes the return proposal table script and returns a list of approved nodes func (s *Suite) ExecuteReadApprovedNodesScript(ctx context.Context, env templates.Environment) cadence.Value { - v, err := s.client.ExecuteScriptBytes(ctx, templates.GenerateReturnProposedTableScript(env), []cadence.Value{}) + v, err := s.client.ExecuteScriptBytes(ctx, templates.GenerateGetApprovedNodesScript(env), []cadence.Value{}) require.NoError(s.T(), err) return v @@ -380,8 +363,15 @@ func (s *Suite) getTestContainerName(role flow.Role) string { // and checks that the info.NodeID is in both list func (s *Suite) assertNodeApprovedAndProposed(ctx context.Context, env templates.Environment, info *StakedNodeOperationInfo) { // ensure node ID in approved list - approvedNodes := s.ExecuteReadApprovedNodesScript(ctx, env) - require.Containsf(s.T(), approvedNodes.(cadence.Array).Values, cadence.String(info.NodeID.String()), "expected new node to be in approved nodes list: %x", info.NodeID) + //approvedNodes := s.ExecuteReadApprovedNodesScript(ctx, env) + //require.Containsf(s.T(), approvedNodes.(cadence.Array).Values, cadence.String(info.NodeID.String()), "expected new node to be in approved nodes list: %x", info.NodeID) + + // Access Nodes go through a separate selection process, so they do not immediately + // appear on the proposed table -- skip checking for them here. + if info.Role == flow.RoleAccess { + s.T().Logf("skipping checking proposed table for joining Access Node") + return + } // check if node is in proposed table proposedTable := s.ExecuteGetProposedTableScript(ctx, env, info.NodeID) @@ -546,18 +536,7 @@ func (s *Suite) getLatestFinalizedHeader(ctx context.Context) *flow.Header { // submitSmokeTestTransaction will submit a create account transaction to smoke test network // This ensures a single transaction can be sealed by the network. func (s *Suite) submitSmokeTestTransaction(ctx context.Context) { - fullAccountKey := sdk.NewAccountKey(). - SetPublicKey(unittest.PrivateKeyFixture(crypto.ECDSAP256, crypto.KeyGenSeedMinLen).PublicKey()). - SetHashAlgo(sdkcrypto.SHA2_256). - SetWeight(sdk.AccountKeyWeightThreshold) - - // createAccount will submit a create account transaction and wait for it to be sealed - _, err := s.createAccount( - ctx, - fullAccountKey, - s.client.Account(), - s.client.SDKServiceAddress(), - ) + _, err := utils.CreateFlowAccount(ctx, s.client) require.NoError(s.T(), err) } diff --git a/integration/tests/execution/suite.go b/integration/tests/execution/suite.go index 09666c24aa2..d12b8ef5902 100644 --- a/integration/tests/execution/suite.go +++ b/integration/tests/execution/suite.go @@ -105,8 +105,6 @@ func (s *Suite) SetupTest() { s.log = unittest.LoggerForTest(s.Suite.T(), zerolog.InfoLevel) s.log.Info().Msg("================> SetupTest") - blockRateFlag := "--block-rate-delay=1ms" - s.nodeConfigs = append(s.nodeConfigs, testnet.NewNodeConfig(flow.RoleAccess)) // generate the four consensus identities @@ -114,7 +112,7 @@ func (s *Suite) SetupTest() { for _, nodeID := range s.nodeIDs { nodeConfig := testnet.NewNodeConfig(flow.RoleConsensus, testnet.WithID(nodeID), testnet.WithLogLevel(zerolog.FatalLevel), - testnet.WithAdditionalFlag(blockRateFlag), + testnet.WithAdditionalFlag("cruise-ctl-fallback-proposal-duration=1ms"), ) s.nodeConfigs = append(s.nodeConfigs, nodeConfig) } @@ -128,11 +126,11 @@ func (s *Suite) SetupTest() { // need two collection node coll1Config := testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), - testnet.WithAdditionalFlag(blockRateFlag), + testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms"), ) coll2Config := testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), - testnet.WithAdditionalFlag(blockRateFlag), + testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms"), ) s.nodeConfigs = append(s.nodeConfigs, coll1Config, coll2Config) diff --git a/integration/tests/mvp/mvp_test.go b/integration/tests/mvp/mvp_test.go index 89cfbfaa176..cb6d6fb6c4f 100644 --- a/integration/tests/mvp/mvp_test.go +++ b/integration/tests/mvp/mvp_test.go @@ -114,12 +114,12 @@ func TestMVP_Bootstrap(t *testing.T) { func buildMVPNetConfig() testnet.NetworkConfig { collectionConfigs := []func(*testnet.NodeConfig){ - testnet.WithAdditionalFlag("--block-rate-delay=100ms"), + testnet.WithAdditionalFlag("--hotstuff-proposal-duration=100ms"), testnet.WithLogLevel(zerolog.FatalLevel), } consensusConfigs := []func(config *testnet.NodeConfig){ - testnet.WithAdditionalFlag("--block-rate-delay=100ms"), + testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=100ms"), testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", 1)), testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", 1)), testnet.WithLogLevel(zerolog.FatalLevel), @@ -177,7 +177,7 @@ func runMVPTest(t *testing.T, ctx context.Context, net *testnet.FlowNetwork) { SetGasLimit(9999) childCtx, cancel := context.WithTimeout(ctx, defaultTimeout) - err = serviceAccountClient.SignAndSendTransaction(ctx, createAccountTx) + err = serviceAccountClient.SignAndSendTransaction(childCtx, createAccountTx) require.NoError(t, err) cancel() diff --git a/integration/tests/execution/stop_at_height_test.go b/integration/tests/upgrades/stop_at_height_test.go similarity index 59% rename from integration/tests/execution/stop_at_height_test.go rename to integration/tests/upgrades/stop_at_height_test.go index 0faf12a1237..2ea8ba1a253 100644 --- a/integration/tests/execution/stop_at_height_test.go +++ b/integration/tests/upgrades/stop_at_height_test.go @@ -1,12 +1,16 @@ -package execution +package upgrades import ( "context" + "fmt" "testing" "time" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + + adminClient "github.com/onflow/flow-go/integration/client" + "github.com/onflow/flow-go/integration/testnet" ) func TestStopAtHeight(t *testing.T) { @@ -17,22 +21,24 @@ type TestStopAtHeightSuite struct { Suite } -type AdminCommandListCommands []string - type StopAtHeightRequest struct { Height uint64 `json:"height"` Crash bool `json:"crash"` } func (s *TestStopAtHeightSuite) TestStopAtHeight() { + enContainer := s.net.ContainerByID(s.exe1ID) + serverAddr := fmt.Sprintf("localhost:%s", enContainer.Port(testnet.AdminPort)) + admin := adminClient.NewAdminClient(serverAddr) + // make sure stop at height admin command is available - commandsList := AdminCommandListCommands{} - err := s.SendExecutionAdminCommand(context.Background(), "list-commands", struct{}{}, &commandsList) + resp, err := admin.RunCommand(context.Background(), "list-commands", struct{}{}) require.NoError(s.T(), err) - - require.Contains(s.T(), commandsList, "stop-at-height") + commandsList, ok := resp.Output.([]interface{}) + s.True(ok) + s.Contains(commandsList, "stop-at-height") // wait for some blocks being finalized s.BlockState.WaitForHighestFinalizedProgress(s.T(), 2) @@ -47,18 +53,27 @@ func (s *TestStopAtHeightSuite) TestStopAtHeight() { Crash: true, } - var commandResponse string - err = s.SendExecutionAdminCommand(context.Background(), "stop-at-height", stopAtHeightRequest, &commandResponse) - require.NoError(s.T(), err) - - require.Equal(s.T(), "ok", commandResponse) + resp, err = admin.RunCommand( + context.Background(), + "stop-at-height", + stopAtHeightRequest, + ) + s.NoError(err) + commandResponse, ok := resp.Output.(string) + s.True(ok) + s.Equal("ok", commandResponse) shouldExecute := s.BlockState.WaitForBlocksByHeight(s.T(), stopHeight-1) shouldNotExecute := s.BlockState.WaitForBlocksByHeight(s.T(), stopHeight) s.ReceiptState.WaitForReceiptFrom(s.T(), shouldExecute[0].Header.ID(), s.exe1ID) - s.ReceiptState.WaitForNoReceiptFrom(s.T(), 5*time.Second, shouldNotExecute[0].Header.ID(), s.exe1ID) + s.ReceiptState.WaitForNoReceiptFrom( + s.T(), + 5*time.Second, + shouldNotExecute[0].Header.ID(), + s.exe1ID, + ) err = enContainer.WaitForContainerStopped(10 * time.Second) - require.NoError(s.T(), err) + s.NoError(err) } diff --git a/integration/tests/upgrades/suite.go b/integration/tests/upgrades/suite.go new file mode 100644 index 00000000000..dbc40e810aa --- /dev/null +++ b/integration/tests/upgrades/suite.go @@ -0,0 +1,125 @@ +package upgrades + +import ( + "context" + "fmt" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/engine/ghost/client" + "github.com/onflow/flow-go/integration/testnet" + "github.com/onflow/flow-go/integration/tests/lib" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +type Suite struct { + suite.Suite + log zerolog.Logger + lib.TestnetStateTracker + cancel context.CancelFunc + net *testnet.FlowNetwork + ghostID flow.Identifier + exe1ID flow.Identifier +} + +func (s *Suite) Ghost() *client.GhostClient { + client, err := s.net.ContainerByID(s.ghostID).GhostClient() + require.NoError(s.T(), err, "could not get ghost client") + return client +} + +func (s *Suite) AccessClient() *testnet.Client { + client, err := s.net.ContainerByName(testnet.PrimaryAN).TestnetClient() + s.NoError(err, "could not get access client") + return client +} + +func (s *Suite) SetupTest() { + s.log = unittest.LoggerForTest(s.Suite.T(), zerolog.InfoLevel) + s.log.Info().Msg("================> SetupTest") + defer func() { + s.log.Info().Msg("================> Finish SetupTest") + }() + + collectionConfigs := []func(*testnet.NodeConfig){ + testnet.WithAdditionalFlag("--hotstuff-proposal-duration=10ms"), + testnet.WithLogLevel(zerolog.WarnLevel), + } + + consensusConfigs := []func(config *testnet.NodeConfig){ + testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=10ms"), + testnet.WithAdditionalFlag( + fmt.Sprintf( + "--required-verification-seal-approvals=%d", + 1, + ), + ), + testnet.WithAdditionalFlag( + fmt.Sprintf( + "--required-construction-seal-approvals=%d", + 1, + ), + ), + testnet.WithLogLevel(zerolog.WarnLevel), + } + + // a ghost node masquerading as an access node + s.ghostID = unittest.IdentifierFixture() + ghostNode := testnet.NewNodeConfig( + flow.RoleAccess, + testnet.WithLogLevel(zerolog.FatalLevel), + testnet.WithID(s.ghostID), + testnet.AsGhost(), + ) + + s.exe1ID = unittest.IdentifierFixture() + confs := []testnet.NodeConfig{ + testnet.NewNodeConfig(flow.RoleCollection, collectionConfigs...), + testnet.NewNodeConfig( + flow.RoleExecution, + testnet.WithLogLevel(zerolog.WarnLevel), + testnet.WithID(s.exe1ID), + testnet.WithAdditionalFlag("--extensive-logging=true"), + ), + testnet.NewNodeConfig( + flow.RoleExecution, + testnet.WithLogLevel(zerolog.WarnLevel), + ), + testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...), + testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...), + testnet.NewNodeConfig( + flow.RoleVerification, + testnet.WithLogLevel(zerolog.WarnLevel), + ), + testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.WarnLevel)), + ghostNode, + } + + netConfig := testnet.NewNetworkConfig( + "upgrade_tests", + confs, + // set long staking phase to avoid QC/DKG transactions during test run + testnet.WithViewsInStakingAuction(10_000), + testnet.WithViewsInEpoch(100_000), + ) + // initialize the network + s.net = testnet.PrepareFlowNetwork(s.T(), netConfig, flow.Localnet) + + // start the network + ctx, cancel := context.WithCancel(context.Background()) + s.cancel = cancel + s.net.Start(ctx) + + // start tracking blocks + s.Track(s.T(), ctx, s.Ghost()) +} + +func (s *Suite) TearDownTest() { + s.log.Info().Msg("================> Start TearDownTest") + s.net.Remove() + s.cancel() + s.log.Info().Msg("================> Finish TearDownTest") +} diff --git a/integration/tests/upgrades/version_beacon_service_event_test.go b/integration/tests/upgrades/version_beacon_service_event_test.go new file mode 100644 index 00000000000..71142af8e66 --- /dev/null +++ b/integration/tests/upgrades/version_beacon_service_event_test.go @@ -0,0 +1,298 @@ +package upgrades + +import ( + "context" + "math" + "testing" + "time" + + "github.com/coreos/go-semver/semver" + "github.com/onflow/cadence" + "github.com/onflow/flow-core-contracts/lib/go/templates" + "github.com/stretchr/testify/require" + + sdk "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" + + "github.com/stretchr/testify/suite" +) + +type TestServiceEventVersionControl struct { + Suite +} + +func (s *TestServiceEventVersionControl) TestEmittingVersionBeaconServiceEvent() { + unittest.SkipUnless(s.T(), unittest.TEST_FLAKY, + "flaky in CI but works 100% of the time locally") + + // freezePeriodForTheseTests controls the version beacon freeze period. The longer the + // freeze period the more blocks we need to wait for the version beacon to take effect, + // making the test slower. But if the freeze period is too short + // we might execute to many blocks, before the version beacon takes effect. + // + // - If the test is flaky try increasing this value. + // - If the test is too slow try decreasing this value. + freezePeriodForTheseTests := uint64(100) + + ctx := context.Background() + + serviceAddress := sdk.Address(s.net.Root().Header.ChainID.Chain().ServiceAddress()) + env := templates.Environment{ + NodeVersionBeaconAddress: serviceAddress.String(), + } + + freezePeriod := s.getFreezePeriod(ctx, env) + s.Run("set freeze period script should work", func() { + // we also want to do this for the next test to conclude faster + newFreezePeriod := freezePeriodForTheseTests + + s.Require().NotEqual( + newFreezePeriod, + freezePeriod, + "the test is pointless, "+ + "please change the freeze period in the test") + + setFreezePeriodScript := templates.GenerateChangeVersionFreezePeriodScript(env) + latestBlockID, err := s.AccessClient().GetLatestBlockID(ctx) + require.NoError(s.T(), err) + + tx := sdk.NewTransaction(). + SetScript(setFreezePeriodScript). + SetReferenceBlockID(sdk.Identifier(latestBlockID)). + SetProposalKey(serviceAddress, + 0, s.AccessClient().GetSeqNumber()). // todo track sequence number + AddAuthorizer(serviceAddress). + SetPayer(serviceAddress) + + err = tx.AddArgument(cadence.NewUInt64(newFreezePeriod)) + s.Require().NoError(err) + + err = s.AccessClient().SignAndSendTransaction(ctx, tx) + s.Require().NoError(err) + + result, err := s.AccessClient().WaitForSealed(ctx, tx.ID()) + require.NoError(s.T(), err) + + s.Require().NoError(result.Error) + + freezePeriod = s.getFreezePeriod(ctx, env) + s.Require().Equal(newFreezePeriod, freezePeriod) + }) + + s.Run("should fail adding version boundary inside the freeze period", func() { + latestFinalized, err := s.AccessClient().GetLatestFinalizedBlockHeader(ctx) + require.NoError(s.T(), err) + + height := latestFinalized.Height + freezePeriod - 5 + major := uint8(0) + minor := uint8(0) + patch := uint8(1) + + txResult := s.sendSetVersionBoundaryTransaction( + ctx, + env, + versionBoundary{ + Major: major, + Minor: minor, + Patch: patch, + PreRelease: "", + BlockHeight: height, + }) + s.Require().Error(txResult.Error) + + sealed := s.ReceiptState.WaitForReceiptFromAny( + s.T(), + flow.Identifier(txResult.BlockID)) + s.Require().Len(sealed.ExecutionResult.ServiceEvents, 0) + }) + + s.Run("should add version boundary after the freeze period", func() { + latestFinalized, err := s.AccessClient().GetLatestFinalizedBlockHeader(ctx) + require.NoError(s.T(), err) + + // make sure target height is correct + // the height at which the version change will take effect should be after + // the current height + the freeze period + height := latestFinalized.Height + freezePeriod + 100 + + // version 0.0.1 + // low version to not interfere with other tests + major := uint8(0) + minor := uint8(0) + patch := uint8(1) + + txResult := s.sendSetVersionBoundaryTransaction( + ctx, + env, + versionBoundary{ + Major: major, + Minor: minor, + Patch: patch, + PreRelease: "", + BlockHeight: height, + }) + s.Require().NoError(txResult.Error) + + sealed := s.ReceiptState.WaitForReceiptFromAny( + s.T(), + flow.Identifier(txResult.BlockID)) + + s.Require().Len(sealed.ExecutionResult.ServiceEvents, 1) + s.Require().IsType( + &flow.VersionBeacon{}, + sealed.ExecutionResult.ServiceEvents[0].Event) + + versionTable := sealed.ExecutionResult.ServiceEvents[0].Event.(*flow.VersionBeacon) + // this should be the second ever emitted + // the first was emitted at bootstrap + s.Require().Equal(uint64(1), versionTable.Sequence) + s.Require().Len(versionTable.VersionBoundaries, 2) + + // zeroth boundary should be present, as it is the one we should be on + s.Require().Equal(uint64(0), versionTable.VersionBoundaries[0].BlockHeight) + + version, err := semver.NewVersion(versionTable.VersionBoundaries[0].Version) + s.Require().NoError(err) + s.Require().Equal(uint8(0), uint8(version.Major)) + s.Require().Equal(uint8(0), uint8(version.Minor)) + s.Require().Equal(uint8(0), uint8(version.Patch)) + + s.Require().Equal(height, versionTable.VersionBoundaries[1].BlockHeight) + + version, err = semver.NewVersion(versionTable.VersionBoundaries[1].Version) + s.Require().NoError(err) + s.Require().Equal(major, uint8(version.Major)) + s.Require().Equal(minor, uint8(version.Minor)) + s.Require().Equal(patch, uint8(version.Patch)) + }) + + s.Run("stop with version beacon", func() { + latestFinalized, err := s.AccessClient().GetLatestFinalizedBlockHeader(ctx) + require.NoError(s.T(), err) + + // make sure target height is correct + // the height at which the version change will take effect should be after + // the current height + the freeze period + height := latestFinalized.Height + freezePeriod + 100 + + // max version to be sure that the node version is lower so we force a stop + major := uint8(math.MaxUint8) + minor := uint8(math.MaxUint8) + patch := uint8(math.MaxUint8) + + txResult := s.sendSetVersionBoundaryTransaction( + ctx, + env, + versionBoundary{ + Major: major, + Minor: minor, + Patch: patch, + PreRelease: "", + BlockHeight: height, + }) + s.Require().NoError(txResult.Error) + + sealed := s.ReceiptState.WaitForReceiptFromAny( + s.T(), + flow.Identifier(txResult.BlockID)) + + s.Require().Len(sealed.ExecutionResult.ServiceEvents, 1) + s.Require().IsType( + &flow.VersionBeacon{}, + sealed.ExecutionResult.ServiceEvents[0].Event) + + versionTable := sealed.ExecutionResult.ServiceEvents[0].Event.(*flow.VersionBeacon) + + s.Require().Equal(height, versionTable.VersionBoundaries[len(versionTable.VersionBoundaries)-1].BlockHeight) + version, err := semver.NewVersion(versionTable.VersionBoundaries[len(versionTable.VersionBoundaries)-1].Version) + s.Require().NoError(err) + s.Require().Equal(major, uint8(version.Major)) + s.Require().Equal(minor, uint8(version.Minor)) + s.Require().Equal(patch, uint8(version.Patch)) + + shouldExecute := s.BlockState.WaitForBlocksByHeight(s.T(), height-1) + shouldNotExecute := s.BlockState.WaitForBlocksByHeight(s.T(), height) + + s.ReceiptState.WaitForReceiptFrom(s.T(), shouldExecute[0].Header.ID(), s.exe1ID) + s.ReceiptState.WaitForNoReceiptFrom( + s.T(), + 5*time.Second, + shouldNotExecute[0].Header.ID(), + s.exe1ID, + ) + + enContainer := s.net.ContainerByID(s.exe1ID) + err = enContainer.WaitForContainerStopped(30 * time.Second) + s.NoError(err) + }) +} + +func (s *TestServiceEventVersionControl) getFreezePeriod( + ctx context.Context, + env templates.Environment, +) uint64 { + + freezePeriodScript := templates.GenerateGetVersionBoundaryFreezePeriodScript(env) + + freezePeriodRaw, err := s.AccessClient(). + ExecuteScriptBytes(ctx, freezePeriodScript, nil) + s.Require().NoError(err) + + cadenceBuffer, is := freezePeriodRaw.(cadence.UInt64) + + s.Require().True(is, "version freezePeriod script returned unknown type") + + return cadenceBuffer.ToGoValue().(uint64) +} + +type versionBoundary struct { + BlockHeight uint64 + Major uint8 + Minor uint8 + Patch uint8 + PreRelease string +} + +func (s *TestServiceEventVersionControl) sendSetVersionBoundaryTransaction( + ctx context.Context, + env templates.Environment, + boundary versionBoundary, +) *sdk.TransactionResult { + serviceAddress := s.net.Root().Header.ChainID.Chain().ServiceAddress() + + versionTableChangeScript := templates.GenerateSetVersionBoundaryScript(env) + + latestBlockId, err := s.AccessClient().GetLatestBlockID(ctx) + s.Require().NoError(err) + seq := s.AccessClient().GetSeqNumber() + + tx := sdk.NewTransaction(). + SetScript(versionTableChangeScript). + SetReferenceBlockID(sdk.Identifier(latestBlockId)). + SetProposalKey(sdk.Address(serviceAddress), 0, seq). + SetPayer(sdk.Address(serviceAddress)). + AddAuthorizer(sdk.Address(serviceAddress)) + + err = tx.AddArgument(cadence.NewUInt8(boundary.Major)) + s.Require().NoError(err) + err = tx.AddArgument(cadence.NewUInt8(boundary.Minor)) + s.Require().NoError(err) + err = tx.AddArgument(cadence.NewUInt8(boundary.Patch)) + s.Require().NoError(err) + err = tx.AddArgument(cadence.String(boundary.PreRelease)) + s.Require().NoError(err) + err = tx.AddArgument(cadence.NewUInt64(boundary.BlockHeight)) + s.Require().NoError(err) + + err = s.AccessClient().SignAndSendTransaction(ctx, tx) + s.Require().NoError(err) + + txResult, err := s.AccessClient().WaitForSealed(ctx, tx.ID()) + s.Require().NoError(err) + return txResult +} + +func TestVersionControlServiceEvent(t *testing.T) { + suite.Run(t, new(TestServiceEventVersionControl)) +} diff --git a/integration/tests/verification/suite.go b/integration/tests/verification/suite.go index 0bef62132f4..484e1dde72e 100644 --- a/integration/tests/verification/suite.go +++ b/integration/tests/verification/suite.go @@ -65,8 +65,6 @@ func (s *Suite) SetupSuite() { s.log = unittest.LoggerForTest(s.Suite.T(), zerolog.InfoLevel) s.log.Info().Msg("================> SetupTest") - blockRateFlag := "--block-rate-delay=1ms" - s.nodeConfigs = append(s.nodeConfigs, testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.FatalLevel))) // generate the four consensus identities @@ -77,7 +75,7 @@ func (s *Suite) SetupSuite() { testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--required-verification-seal-approvals=1"), testnet.WithAdditionalFlag("--required-construction-seal-approvals=1"), - testnet.WithAdditionalFlag(blockRateFlag), + testnet.WithAdditionalFlag("cruise-ctl-fallback-proposal-duration=1ms"), ) s.nodeConfigs = append(s.nodeConfigs, nodeConfig) } @@ -111,11 +109,11 @@ func (s *Suite) SetupSuite() { // generates two collection node coll1Config := testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), - testnet.WithAdditionalFlag(blockRateFlag), + testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms"), ) coll2Config := testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), - testnet.WithAdditionalFlag(blockRateFlag), + testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms"), ) s.nodeConfigs = append(s.nodeConfigs, coll1Config, coll2Config) diff --git a/integration/utils/emulator_client.go b/integration/utils/emulator_client.go index 18ddee8cfde..8d42e1388fd 100644 --- a/integration/utils/emulator_client.go +++ b/integration/utils/emulator_client.go @@ -6,31 +6,37 @@ import ( "github.com/onflow/cadence" jsoncdc "github.com/onflow/cadence/encoding/json" - emulator "github.com/onflow/flow-emulator" + "github.com/onflow/flow-emulator/adapters" + emulator "github.com/onflow/flow-emulator/emulator" + "github.com/rs/zerolog" sdk "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go-sdk/templates" "github.com/onflow/flow-go/model/flow" ) // EmulatorClient is a wrapper around the emulator to implement the same interface // used by the SDK client. Used for testing against the emulator. type EmulatorClient struct { - blockchain *emulator.Blockchain + adapter *adapters.SDKAdapter } -func NewEmulatorClient(blockchain *emulator.Blockchain) *EmulatorClient { +func NewEmulatorClient(blockchain emulator.Emulator) *EmulatorClient { + logger := zerolog.Nop() + + adapter := adapters.NewSDKAdapter(&logger, blockchain) client := &EmulatorClient{ - blockchain: blockchain, + adapter: adapter, } return client } func (c *EmulatorClient) GetAccount(ctx context.Context, address sdk.Address) (*sdk.Account, error) { - return c.blockchain.GetAccount(address) + return c.adapter.GetAccount(ctx, address) } func (c *EmulatorClient) GetAccountAtLatestBlock(ctx context.Context, address sdk.Address) (*sdk.Account, error) { - return c.blockchain.GetAccount(address) + return c.adapter.GetAccount(ctx, address) } func (c *EmulatorClient) SendTransaction(ctx context.Context, tx sdk.Transaction) error { @@ -39,24 +45,19 @@ func (c *EmulatorClient) SendTransaction(ctx context.Context, tx sdk.Transaction } func (c *EmulatorClient) GetLatestBlock(ctx context.Context, isSealed bool) (*sdk.Block, error) { - block, err := c.blockchain.GetLatestBlock() + block, _, err := c.adapter.GetLatestBlock(ctx, true) if err != nil { return nil, err } - blockID := block.ID() - - var id sdk.Identifier - copy(id[:], blockID[:]) - sdkBlock := &sdk.Block{ - BlockHeader: sdk.BlockHeader{ID: id}, + BlockHeader: sdk.BlockHeader{ID: block.ID}, } return sdkBlock, nil } func (c *EmulatorClient) GetTransactionResult(ctx context.Context, txID sdk.Identifier) (*sdk.TransactionResult, error) { - return c.blockchain.GetTransactionResult(txID) + return c.adapter.GetTransactionResult(ctx, txID) } func (c *EmulatorClient) ExecuteScriptAtLatestBlock(ctx context.Context, script []byte, args []cadence.Value) (cadence.Value, error) { @@ -70,12 +71,17 @@ func (c *EmulatorClient) ExecuteScriptAtLatestBlock(ctx context.Context, script arguments = append(arguments, val) } - scriptResult, err := c.blockchain.ExecuteScript(script, arguments) + scriptResult, err := c.adapter.ExecuteScriptAtLatestBlock(ctx, script, arguments) + if err != nil { + return nil, err + } + + value, err := jsoncdc.Decode(nil, scriptResult) if err != nil { return nil, err } - return scriptResult.Value, nil + return value, nil } func (c *EmulatorClient) ExecuteScriptAtBlockID(ctx context.Context, blockID sdk.Identifier, script []byte, args []cadence.Value) (cadence.Value, error) { @@ -90,31 +96,38 @@ func (c *EmulatorClient) ExecuteScriptAtBlockID(ctx context.Context, blockID sdk } // get block by ID - block, err := c.blockchain.GetBlockByID(blockID) + block, _, err := c.adapter.GetBlockByID(ctx, blockID) if err != nil { return nil, err } - scriptResult, err := c.blockchain.ExecuteScriptAtBlock(script, arguments, block.Header.Height) + scriptResult, err := c.adapter.ExecuteScriptAtBlockHeight(ctx, block.BlockHeader.Height, script, arguments) + if err != nil { - return nil, err + return nil, fmt.Errorf("error in script: %w", err) } - if scriptResult.Error != nil { - return nil, fmt.Errorf("error in script: %w", scriptResult.Error) + value, err := jsoncdc.Decode(nil, scriptResult) + if err != nil { + return nil, err } - return scriptResult.Value, nil + return value, nil +} + +func (c *EmulatorClient) CreateAccount(keys []*sdk.AccountKey, contracts []templates.Contract) (sdk.Address, error) { + return c.adapter.CreateAccount(context.Background(), keys, contracts) + } func (c *EmulatorClient) Submit(tx *sdk.Transaction) (*flow.Block, error) { // submit the signed transaction - err := c.blockchain.AddTransaction(*tx) + err := c.adapter.SendTransaction(context.Background(), *tx) if err != nil { return nil, err } - block, _, err := c.blockchain.ExecuteAndCommitBlock() + block, _, err := c.adapter.Emulator().ExecuteAndCommitBlock() if err != nil { return nil, err } diff --git a/integration/utils/templates/remove-node.cdc b/integration/utils/templates/remove-node.cdc index 88679d076ec..3cc185b87fe 100644 --- a/integration/utils/templates/remove-node.cdc +++ b/integration/utils/templates/remove-node.cdc @@ -14,12 +14,8 @@ transaction(id: String) { } execute { + // this method also removes them from the approve-list self.adminRef.removeAndRefundNodeRecord(id) - let nodeIDs = FlowIDTableStaking.getApprovedList() - nodeIDs[id] = nil - - // set the approved list to the new allow-list - self.adminRef.setApprovedList(nodeIDs) } } diff --git a/integration/utils/transactions.go b/integration/utils/transactions.go index 26e1eb2012a..2ae98c87eb5 100644 --- a/integration/utils/transactions.go +++ b/integration/utils/transactions.go @@ -1,14 +1,20 @@ package utils import ( + "context" _ "embed" + "fmt" "github.com/onflow/cadence" "github.com/onflow/flow-core-contracts/lib/go/templates" sdk "github.com/onflow/flow-go-sdk" + sdkcrypto "github.com/onflow/flow-go-sdk/crypto" sdktemplates "github.com/onflow/flow-go-sdk/templates" + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/integration/testnet" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" ) //go:embed templates/create-and-setup-node.cdc @@ -172,3 +178,26 @@ func MakeAdminRemoveNodeTx( return tx, nil } + +// submitSmokeTestTransaction will submit a create account transaction to smoke test network +// This ensures a single transaction can be sealed by the network. +func CreateFlowAccount(ctx context.Context, client *testnet.Client) (sdk.Address, error) { + fullAccountKey := sdk.NewAccountKey(). + SetPublicKey(unittest.PrivateKeyFixture(crypto.ECDSAP256, crypto.KeyGenSeedMinLen).PublicKey()). + SetHashAlgo(sdkcrypto.SHA2_256). + SetWeight(sdk.AccountKeyWeightThreshold) + + latestBlockID, err := client.GetLatestBlockID(ctx) + if err != nil { + return sdk.EmptyAddress, fmt.Errorf("failed to get latest block id: %w", err) + } + + // createAccount will submit a create account transaction and wait for it to be sealed + addr, err := client.CreateAccount(ctx, fullAccountKey, client.Account(), client.SDKServiceAddress(), sdk.Identifier(latestBlockID)) + if err != nil { + return sdk.EmptyAddress, fmt.Errorf("failed to create account: %w", err) + } + + client.Account().Keys[0].SequenceNumber++ + return addr, nil +} diff --git a/ledger/common/testutils/testutils.go b/ledger/common/testutils/testutils.go index ab30000c47c..e0e100ee46c 100644 --- a/ledger/common/testutils/testutils.go +++ b/ledger/common/testutils/testutils.go @@ -206,7 +206,7 @@ func RandomValues(n int, minByteSize, maxByteSize int) []l.Value { byteSize = minByteSize + rand.Intn(maxByteSize-minByteSize) } value := make([]byte, byteSize) - _, err := rand.Read(value) + _, err := crand.Read(value) if err != nil { panic("random generation failed") } diff --git a/ledger/complete/compactor_test.go b/ledger/complete/compactor_test.go index 7617c7bb9b2..e06eff54ec1 100644 --- a/ledger/complete/compactor_test.go +++ b/ledger/complete/compactor_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - prometheusWAL "github.com/m4ksio/wal/wal" + prometheusWAL "github.com/onflow/wal/wal" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" diff --git a/ledger/complete/ledger.go b/ledger/complete/ledger.go index 1a2b6fd1e35..cc132815830 100644 --- a/ledger/complete/ledger.go +++ b/ledger/complete/ledger.go @@ -1,10 +1,8 @@ package complete import ( - "encoding/json" "fmt" "io" - "os" "time" "github.com/rs/zerolog" @@ -328,16 +326,11 @@ func (l *Ledger) Checkpointer() (*realWAL.Checkpointer, error) { return checkpointer, nil } -// ExportCheckpointAt exports a checkpoint at specific state commitment after applying migrations and returns the new state (after migration) and any errors -func (l *Ledger) ExportCheckpointAt( +func (l *Ledger) MigrateAt( state ledger.State, migrations []ledger.Migration, - preCheckpointReporters []ledger.Reporter, - postCheckpointReporters []ledger.Reporter, targetPathFinderVersion uint8, - outputDir, outputFile string, -) (ledger.State, error) { - +) (*trie.MTrie, error) { l.logger.Info().Msgf( "Ledger is loaded, checkpoint export has started for state %s, and %d migrations have been planed", state.String(), @@ -351,14 +344,14 @@ func (l *Ledger) ExportCheckpointAt( l.logger.Info(). Str("hash", rh.String()). Msgf("Most recently touched root hash.") - return ledger.State(hash.DummyHash), + return nil, fmt.Errorf("cannot get trie at the given state commitment: %w", err) } // clean up tries to release memory err = l.keepOnlyOneTrie(state) if err != nil { - return ledger.State(hash.DummyHash), + return nil, fmt.Errorf("failed to clean up tries to reduce memory usage: %w", err) } @@ -370,9 +363,6 @@ func (l *Ledger) ExportCheckpointAt( if noMigration { // when there is no migration, reuse the trie without rebuilding it newTrie = t - // when there is no migration, we don't generate the payloads here until later running the - // postCheckpointReporters, because the ExportReporter is currently the only - // preCheckpointReporters, which doesn't use the payloads. } else { // get all payloads payloads = t.AllPayloads() @@ -387,7 +377,7 @@ func (l *Ledger) ExportCheckpointAt( elapsed := time.Since(start) if err != nil { - return ledger.State(hash.DummyHash), fmt.Errorf("error applying migration (%d): %w", i, err) + return nil, fmt.Errorf("error applying migration (%d): %w", i, err) } newPayloadSize := len(payloads) @@ -409,7 +399,7 @@ func (l *Ledger) ExportCheckpointAt( // get paths paths, err := pathfinder.PathsFromPayloads(payloads, targetPathFinderVersion) if err != nil { - return ledger.State(hash.DummyHash), fmt.Errorf("cannot export checkpoint, can't construct paths: %w", err) + return nil, fmt.Errorf("cannot export checkpoint, can't construct paths: %w", err) } l.logger.Info().Msgf("constructing a new trie with migrated payloads (count: %d)...", len(payloads)) @@ -420,7 +410,7 @@ func (l *Ledger) ExportCheckpointAt( applyPruning := false newTrie, _, err = trie.NewTrieWithUpdatedRegisters(emptyTrie, paths, payloads, applyPruning) if err != nil { - return ledger.State(hash.DummyHash), fmt.Errorf("constructing updated trie failed: %w", err) + return nil, fmt.Errorf("constructing updated trie failed: %w", err) } } @@ -428,60 +418,7 @@ func (l *Ledger) ExportCheckpointAt( l.logger.Info().Msgf("successfully built new trie. NEW ROOT STATECOMMIEMENT: %v", statecommitment.String()) - l.logger.Info().Msgf("running pre-checkpoint reporters") - // run post migration reporters - for i, reporter := range preCheckpointReporters { - l.logger.Info().Msgf("running a pre-checkpoint generation reporter: %s, (%v/%v)", reporter.Name(), i, len(preCheckpointReporters)) - err := runReport(reporter, payloads, statecommitment, l.logger) - if err != nil { - return ledger.State(hash.DummyHash), err - } - } - - l.logger.Info().Msgf("finished running pre-checkpoint reporters") - - l.logger.Info().Msg("creating a checkpoint for the new trie, storing the checkpoint to the file") - - err = os.MkdirAll(outputDir, os.ModePerm) - if err != nil { - return ledger.State(hash.DummyHash), fmt.Errorf("could not create output dir %s: %w", outputDir, err) - } - - err = realWAL.StoreCheckpointV6Concurrently([]*trie.MTrie{newTrie}, outputDir, outputFile, &l.logger) - - // Writing the checkpoint takes time to write and copy. - // Without relying on an exit code or stdout, we need to know when the copy is complete. - writeStatusFileErr := writeStatusFile("checkpoint_status.json", err) - if writeStatusFileErr != nil { - return ledger.State(hash.DummyHash), fmt.Errorf("failed to write checkpoint status file: %w", writeStatusFileErr) - } - - if err != nil { - return ledger.State(hash.DummyHash), fmt.Errorf("failed to store the checkpoint: %w", err) - } - - l.logger.Info().Msgf("checkpoint file successfully stored at: %v %v", outputDir, outputFile) - - l.logger.Info().Msgf("start running post-checkpoint reporters") - - if noMigration { - // when there is no mgiration, we generate the payloads now before - // running the postCheckpointReporters - payloads = newTrie.AllPayloads() - } - - // running post checkpoint reporters - for i, reporter := range postCheckpointReporters { - l.logger.Info().Msgf("running a post-checkpoint generation reporter: %s, (%v/%v)", reporter.Name(), i, len(postCheckpointReporters)) - err := runReport(reporter, payloads, statecommitment, l.logger) - if err != nil { - return ledger.State(hash.DummyHash), err - } - } - - l.logger.Info().Msgf("ran all post-checkpoint reporters") - - return statecommitment, nil + return newTrie, nil } // MostRecentTouchedState returns a state which is most recently touched. @@ -512,29 +449,3 @@ func (l *Ledger) keepOnlyOneTrie(state ledger.State) error { defer l.wal.UnpauseRecord() return l.forest.PurgeCacheExcept(ledger.RootHash(state)) } - -func runReport(r ledger.Reporter, p []ledger.Payload, commit ledger.State, l zerolog.Logger) error { - l.Info(). - Str("name", r.Name()). - Msg("starting reporter") - - start := time.Now() - err := r.Report(p, commit) - elapsed := time.Since(start) - - l.Info(). - Str("timeTaken", elapsed.String()). - Str("name", r.Name()). - Msg("reporter done") - if err != nil { - return fmt.Errorf("error running reporter (%s): %w", r.Name(), err) - } - return nil -} - -func writeStatusFile(fileName string, e error) error { - checkpointStatus := map[string]bool{"succeeded": e == nil} - checkpointStatusJson, _ := json.MarshalIndent(checkpointStatus, "", " ") - err := os.WriteFile(fileName, checkpointStatusJson, 0644) - return err -} diff --git a/ledger/complete/ledger_test.go b/ledger/complete/ledger_test.go index a723d2a58f1..11fdb26ec9c 100644 --- a/ledger/complete/ledger_test.go +++ b/ledger/complete/ledger_test.go @@ -703,180 +703,6 @@ func TestLedgerFunctionality(t *testing.T) { } } -func Test_ExportCheckpointAt(t *testing.T) { - t.Run("noop migration", func(t *testing.T) { - // the exported state has two key/value pairs - // (/1/1/22/2, "A") and (/1/3/22/4, "B") - // this tests the migration at the specific state - // without any special migration so we expect both - // register to show up in the new trie and with the same values - unittest.RunWithTempDir(t, func(dbDir string) { - unittest.RunWithTempDir(t, func(dir2 string) { - - const ( - capacity = 100 - checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. - checkpointsToKeep = 1 - ) - - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize) - require.NoError(t, err) - led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) - require.NoError(t, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep, atomic.NewBool(false)) - require.NoError(t, err) - <-compactor.Ready() - - state := led.InitialState() - u := testutils.UpdateFixture() - u.SetState(state) - - state, _, err = led.Set(u) - require.NoError(t, err) - - newState, err := led.ExportCheckpointAt(state, []ledger.Migration{noOpMigration}, []ledger.Reporter{}, []ledger.Reporter{}, complete.DefaultPathFinderVersion, dir2, "root.checkpoint") - require.NoError(t, err) - assert.Equal(t, newState, state) - - diskWal2, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir2, capacity, pathfinder.PathByteSize, wal.SegmentSize) - require.NoError(t, err) - led2, err := complete.NewLedger(diskWal2, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) - require.NoError(t, err) - compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep, atomic.NewBool(false)) - require.NoError(t, err) - <-compactor2.Ready() - - q, err := ledger.NewQuery(state, u.Keys()) - require.NoError(t, err) - - retValues, err := led2.Get(q) - require.NoError(t, err) - - for i, v := range u.Values() { - assert.Equal(t, v, retValues[i]) - } - - <-led.Done() - <-compactor.Done() - <-led2.Done() - <-compactor2.Done() - }) - }) - }) - t.Run("migration by value", func(t *testing.T) { - // the exported state has two key/value pairs - // ("/1/1/22/2", "A") and ("/1/3/22/4", "B") - // during the migration we change all keys with value "A" to "C" - // so in this case the resulting exported trie is ("/1/1/22/2", "C"), ("/1/3/22/4", "B") - unittest.RunWithTempDir(t, func(dbDir string) { - unittest.RunWithTempDir(t, func(dir2 string) { - - const ( - capacity = 100 - checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. - checkpointsToKeep = 1 - ) - - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize) - require.NoError(t, err) - led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) - require.NoError(t, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep, atomic.NewBool(false)) - require.NoError(t, err) - <-compactor.Ready() - - state := led.InitialState() - u := testutils.UpdateFixture() - u.SetState(state) - - state, _, err = led.Set(u) - require.NoError(t, err) - - newState, err := led.ExportCheckpointAt(state, []ledger.Migration{migrationByValue}, []ledger.Reporter{}, []ledger.Reporter{}, complete.DefaultPathFinderVersion, dir2, "root.checkpoint") - require.NoError(t, err) - - diskWal2, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir2, capacity, pathfinder.PathByteSize, wal.SegmentSize) - require.NoError(t, err) - led2, err := complete.NewLedger(diskWal2, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) - require.NoError(t, err) - compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep, atomic.NewBool(false)) - require.NoError(t, err) - <-compactor2.Ready() - - q, err := ledger.NewQuery(newState, u.Keys()) - require.NoError(t, err) - - retValues, err := led2.Get(q) - require.NoError(t, err) - - assert.Equal(t, retValues[0], ledger.Value([]byte{'C'})) - assert.Equal(t, retValues[1], ledger.Value([]byte{'B'})) - - <-led.Done() - <-compactor.Done() - <-led2.Done() - <-compactor2.Done() - }) - }) - }) - t.Run("migration by key", func(t *testing.T) { - // the exported state has two key/value pairs - // ("/1/1/22/2", "A") and ("/1/3/22/4", "B") - // during the migration we change the value to "D" for key "zero" - // so in this case the resulting exported trie is ("/1/1/22/2", "D"), ("/1/3/22/4", "B") - unittest.RunWithTempDir(t, func(dbDir string) { - unittest.RunWithTempDir(t, func(dir2 string) { - - const ( - capacity = 100 - checkpointDistance = math.MaxInt // A large number to prevent checkpoint creation. - checkpointsToKeep = 1 - ) - - diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dbDir, capacity, pathfinder.PathByteSize, wal.SegmentSize) - require.NoError(t, err) - led, err := complete.NewLedger(diskWal, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) - require.NoError(t, err) - compactor, err := complete.NewCompactor(led, diskWal, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep, atomic.NewBool(false)) - require.NoError(t, err) - <-compactor.Ready() - - state := led.InitialState() - u := testutils.UpdateFixture() - u.SetState(state) - - state, _, err = led.Set(u) - require.NoError(t, err) - - newState, err := led.ExportCheckpointAt(state, []ledger.Migration{migrationByKey}, []ledger.Reporter{}, []ledger.Reporter{}, complete.DefaultPathFinderVersion, dir2, "root.checkpoint") - require.NoError(t, err) - - diskWal2, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir2, capacity, pathfinder.PathByteSize, wal.SegmentSize) - require.NoError(t, err) - led2, err := complete.NewLedger(diskWal2, capacity, &metrics.NoopCollector{}, zerolog.Logger{}, complete.DefaultPathFinderVersion) - require.NoError(t, err) - compactor2, err := complete.NewCompactor(led2, diskWal2, zerolog.Nop(), capacity, checkpointDistance, checkpointsToKeep, atomic.NewBool(false)) - require.NoError(t, err) - <-compactor2.Ready() - - q, err := ledger.NewQuery(newState, u.Keys()) - require.NoError(t, err) - - retValues, err := led2.Get(q) - require.NoError(t, err) - - assert.Equal(t, retValues[0], ledger.Value([]byte{'D'})) - assert.Equal(t, retValues[1], ledger.Value([]byte{'B'})) - - <-led.Done() - <-compactor.Done() - <-led2.Done() - <-compactor2.Done() - }) - }) - }) -} - func TestWALUpdateFailuresBubbleUp(t *testing.T) { unittest.RunWithTempDir(t, func(dir string) { diff --git a/ledger/complete/wal/checkpoint_v6_leaf_reader.go b/ledger/complete/wal/checkpoint_v6_leaf_reader.go index 8c19fe62e84..77dbc0716b5 100644 --- a/ledger/complete/wal/checkpoint_v6_leaf_reader.go +++ b/ledger/complete/wal/checkpoint_v6_leaf_reader.go @@ -18,11 +18,6 @@ type LeafNode struct { Payload *ledger.Payload } -type LeafNodeResult struct { - LeafNode *LeafNode - Err error -} - func nodeToLeaf(leaf *node.Node) *LeafNode { return &LeafNode{ Hash: leaf.Hash(), @@ -31,14 +26,20 @@ func nodeToLeaf(leaf *node.Node) *LeafNode { } } -func OpenAndReadLeafNodesFromCheckpointV6(dir string, fileName string, logger *zerolog.Logger) ( - allLeafNodesCh <-chan LeafNodeResult, errToReturn error) { +// OpenAndReadLeafNodesFromCheckpointV6 takes a channel for pushing the leaf nodes that are read from +// the given checkpoint file specified by dir and fileName. +// It returns when finish reading the checkpoint file and the input channel can be closed. +func OpenAndReadLeafNodesFromCheckpointV6(allLeafNodesCh chan<- *LeafNode, dir string, fileName string, logger *zerolog.Logger) (errToReturn error) { + // we are the only sender of the channel, closing it after done + defer func() { + close(allLeafNodesCh) + }() filepath := filePathCheckpointHeader(dir, fileName) f, err := os.Open(filepath) if err != nil { - return nil, fmt.Errorf("could not open file %v: %w", filepath, err) + return fmt.Errorf("could not open file %v: %w", filepath, err) } defer func(file *os.File) { errToReturn = closeAndMergeError(file, errToReturn) @@ -46,33 +47,29 @@ func OpenAndReadLeafNodesFromCheckpointV6(dir string, fileName string, logger *z subtrieChecksums, _, err := readCheckpointHeader(filepath, logger) if err != nil { - return nil, fmt.Errorf("could not read header: %w", err) + return fmt.Errorf("could not read header: %w", err) } // ensure all checkpoint part file exists, might return os.ErrNotExist error // if a file is missing err = allPartFileExist(dir, fileName, len(subtrieChecksums)) if err != nil { - return nil, fmt.Errorf("fail to check all checkpoint part file exist: %w", err) + return fmt.Errorf("fail to check all checkpoint part file exist: %w", err) } - bufSize := 1000 - leafNodesCh := make(chan LeafNodeResult, bufSize) - allLeafNodesCh = leafNodesCh - defer func() { - close(leafNodesCh) - }() - // push leaf nodes to allLeafNodesCh for i, checksum := range subtrieChecksums { - readCheckpointSubTrieLeafNodes(leafNodesCh, dir, fileName, i, checksum, logger) + err := readCheckpointSubTrieLeafNodes(allLeafNodesCh, dir, fileName, i, checksum, logger) + if err != nil { + return fmt.Errorf("fail to read checkpoint leaf nodes from %v-th subtrie file: %w", i, err) + } } - return allLeafNodesCh, nil + return nil } -func readCheckpointSubTrieLeafNodes(leafNodesCh chan<- LeafNodeResult, dir string, fileName string, index int, checksum uint32, logger *zerolog.Logger) { - err := processCheckpointSubTrie(dir, fileName, index, checksum, logger, +func readCheckpointSubTrieLeafNodes(leafNodesCh chan<- *LeafNode, dir string, fileName string, index int, checksum uint32, logger *zerolog.Logger) error { + return processCheckpointSubTrie(dir, fileName, index, checksum, logger, func(reader *Crc32Reader, nodesCount uint64) error { scratch := make([]byte, 1024*4) // must not be less than 1024 @@ -89,21 +86,11 @@ func readCheckpointSubTrieLeafNodes(leafNodesCh chan<- LeafNodeResult, dir strin return fmt.Errorf("cannot read node %d: %w", i, err) } if node.IsLeaf() { - leafNodesCh <- LeafNodeResult{ - LeafNode: nodeToLeaf(node), - Err: nil, - } + leafNodesCh <- nodeToLeaf(node) } logging(i) } return nil }) - - if err != nil { - leafNodesCh <- LeafNodeResult{ - LeafNode: nil, - Err: err, - } - } } diff --git a/ledger/complete/wal/checkpoint_v6_test.go b/ledger/complete/wal/checkpoint_v6_test.go index 0aeb38cec35..fb98777e0ec 100644 --- a/ledger/complete/wal/checkpoint_v6_test.go +++ b/ledger/complete/wal/checkpoint_v6_test.go @@ -140,7 +140,7 @@ func createMultipleRandomTriesMini(t *testing.T) []*trie.MTrie { var err error // add tries with no shared paths for i := 0; i < 5; i++ { - paths, payloads := randNPathPayloads(10) + paths, payloads := randNPathPayloads(20) activeTrie, _, err = trie.NewTrieWithUpdatedRegisters(activeTrie, paths, payloads, false) require.NoError(t, err, "update registers") tries = append(tries, activeTrie) @@ -318,9 +318,14 @@ func TestWriteAndReadCheckpointV6LeafEmptyTrie(t *testing.T) { fileName := "checkpoint-empty-trie" logger := unittest.Logger() require.NoErrorf(t, StoreCheckpointV6Concurrently(tries, dir, fileName, &logger), "fail to store checkpoint") - resultChan, err := OpenAndReadLeafNodesFromCheckpointV6(dir, fileName, &logger) - require.NoErrorf(t, err, "fail to read checkpoint %v/%v", dir, fileName) - for range resultChan { + + bufSize := 10 + leafNodesCh := make(chan *LeafNode, bufSize) + go func() { + err := OpenAndReadLeafNodesFromCheckpointV6(leafNodesCh, dir, fileName, &logger) + require.NoErrorf(t, err, "fail to read checkpoint %v/%v", dir, fileName) + }() + for range leafNodesCh { require.Fail(t, "should not return any nodes") } }) @@ -332,14 +337,17 @@ func TestWriteAndReadCheckpointV6LeafSimpleTrie(t *testing.T) { fileName := "checkpoint" logger := unittest.Logger() require.NoErrorf(t, StoreCheckpointV6Concurrently(tries, dir, fileName, &logger), "fail to store checkpoint") - resultChan, err := OpenAndReadLeafNodesFromCheckpointV6(dir, fileName, &logger) - require.NoErrorf(t, err, "fail to read checkpoint %v/%v", dir, fileName) + bufSize := 1 + leafNodesCh := make(chan *LeafNode, bufSize) + go func() { + err := OpenAndReadLeafNodesFromCheckpointV6(leafNodesCh, dir, fileName, &logger) + require.NoErrorf(t, err, "fail to read checkpoint %v/%v", dir, fileName) + }() resultPayloads := make([]ledger.Payload, 0) - for readResult := range resultChan { - require.NoError(t, readResult.Err, "no errors in read results") + for leafNode := range leafNodesCh { // avoid dummy payload from empty trie - if readResult.LeafNode.Payload != nil { - resultPayloads = append(resultPayloads, *readResult.LeafNode.Payload) + if leafNode.Payload != nil { + resultPayloads = append(resultPayloads, *leafNode.Payload) } } require.EqualValues(t, tries[1].AllPayloads(), resultPayloads) @@ -352,12 +360,15 @@ func TestWriteAndReadCheckpointV6LeafMultipleTries(t *testing.T) { tries := createMultipleRandomTriesMini(t) logger := unittest.Logger() require.NoErrorf(t, StoreCheckpointV6Concurrently(tries, dir, fileName, &logger), "fail to store checkpoint") - resultChan, err := OpenAndReadLeafNodesFromCheckpointV6(dir, fileName, &logger) - require.NoErrorf(t, err, "fail to read checkpoint %v/%v", dir, fileName) + bufSize := 5 + leafNodesCh := make(chan *LeafNode, bufSize) + go func() { + err := OpenAndReadLeafNodesFromCheckpointV6(leafNodesCh, dir, fileName, &logger) + require.NoErrorf(t, err, "fail to read checkpoint %v/%v", dir, fileName) + }() resultPayloads := make([]ledger.Payload, 0) - for readResult := range resultChan { - require.NoError(t, readResult.Err, "no errors in read results") - resultPayloads = append(resultPayloads, *readResult.LeafNode.Payload) + for leafNode := range leafNodesCh { + resultPayloads = append(resultPayloads, *leafNode.Payload) } require.NotEmpty(t, resultPayloads) }) @@ -528,7 +539,9 @@ func TestAllPartFileExistLeafReader(t *testing.T) { err = os.Remove(fileToDelete) require.NoError(t, err, "fail to remove part file") - _, err = OpenAndReadLeafNodesFromCheckpointV6(dir, fileName, &logger) + bufSize := 10 + leafNodesCh := make(chan *LeafNode, bufSize) + err = OpenAndReadLeafNodesFromCheckpointV6(leafNodesCh, dir, fileName, &logger) require.ErrorIs(t, err, os.ErrNotExist, "wrong error type returned") } }) diff --git a/ledger/complete/wal/wal.go b/ledger/complete/wal/wal.go index 6a9b38d1b3f..de0ed6e2489 100644 --- a/ledger/complete/wal/wal.go +++ b/ledger/complete/wal/wal.go @@ -4,7 +4,7 @@ import ( "fmt" "sort" - prometheusWAL "github.com/m4ksio/wal/wal" + prometheusWAL "github.com/onflow/wal/wal" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" @@ -29,7 +29,7 @@ type DiskWAL struct { func NewDiskWAL(logger zerolog.Logger, reg prometheus.Registerer, metrics module.WALMetrics, dir string, forestCapacity int, pathByteSize int, segmentSize int) (*DiskWAL, error) { w, err := prometheusWAL.NewSize(logger, reg, dir, segmentSize, false) if err != nil { - return nil, err + return nil, fmt.Errorf("could not create disk wal from dir %v, segmentSize %v: %w", dir, segmentSize, err) } return &DiskWAL{ wal: w, diff --git a/model/cluster/payload.go b/model/cluster/payload.go index b8dc209b32c..959eb20575c 100644 --- a/model/cluster/payload.go +++ b/model/cluster/payload.go @@ -18,7 +18,9 @@ type Payload struct { // the proposer may choose any reference block, so long as it is finalized // and within the epoch the cluster is associated with. If a cluster was // assigned for epoch E, then all of its reference blocks must have a view - // in the range [E.FirstView, E.FinalView]. + // in the range [E.FirstView, E.FinalView]. However, if epoch fallback is + // triggered in epoch E, then any reference block with view ≥ E.FirstView + // may be used. // // This determines when the collection expires, using the same expiry rules // as transactions. It is also used as the reference point for committee diff --git a/model/convert/service_event.go b/model/convert/service_event.go index 3f6b9a41370..81c1d9cb9a4 100644 --- a/model/convert/service_event.go +++ b/model/convert/service_event.go @@ -4,8 +4,10 @@ import ( "encoding/hex" "fmt" + "github.com/coreos/go-semver/semver" + "github.com/onflow/cadence" - "github.com/onflow/cadence/encoding/json" + "github.com/onflow/cadence/encoding/ccf" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/fvm/systemcontracts" @@ -30,6 +32,8 @@ func ServiceEvent(chainID flow.ChainID, event flow.Event) (*flow.ServiceEvent, e return convertServiceEventEpochSetup(event) case events.EpochCommit.EventType(): return convertServiceEventEpochCommit(event) + case events.VersionBeacon.EventType(): + return convertServiceEventVersionBeacon(event) default: return nil, fmt.Errorf("invalid event type: %s", event.Type) } @@ -39,84 +43,189 @@ func ServiceEvent(chainID flow.ChainID, event flow.Event) (*flow.ServiceEvent, e // flow.Event type to a ServiceEvent type for an EpochSetup event func convertServiceEventEpochSetup(event flow.Event) (*flow.ServiceEvent, error) { - // decode bytes using jsoncdc - payload, err := json.Decode(nil, event.Payload) + // decode bytes using ccf + payload, err := ccf.Decode(nil, event.Payload) if err != nil { return nil, fmt.Errorf("could not unmarshal event payload: %w", err) } - // parse cadence types to required fields - setup := new(flow.EpochSetup) - // NOTE: variable names prefixed with cdc represent cadence types cdcEvent, ok := payload.(cadence.Event) if !ok { return nil, invalidCadenceTypeError("payload", payload, cadence.Event{}) } - if len(cdcEvent.Fields) < 9 { - return nil, fmt.Errorf("insufficient fields in EpochSetup event (%d < 9)", len(cdcEvent.Fields)) + const expectedFieldCount = 9 + if len(cdcEvent.Fields) < expectedFieldCount { + return nil, fmt.Errorf( + "insufficient fields in EpochSetup event (%d < %d)", + len(cdcEvent.Fields), + expectedFieldCount, + ) } - // extract simple fields - counter, ok := cdcEvent.Fields[0].(cadence.UInt64) - if !ok { - return nil, invalidCadenceTypeError("counter", cdcEvent.Fields[0], cadence.UInt64(0)) + if cdcEvent.Type() == nil { + return nil, fmt.Errorf("EpochSetup event doesn't have type") } - setup.Counter = uint64(counter) - firstView, ok := cdcEvent.Fields[2].(cadence.UInt64) - if !ok { - return nil, invalidCadenceTypeError("firstView", cdcEvent.Fields[2], cadence.UInt64(0)) + + // parse EpochSetup event + + var counter cadence.UInt64 + var firstView cadence.UInt64 + var finalView cadence.UInt64 + var randomSrcHex cadence.String + var dkgPhase1FinalView cadence.UInt64 + var dkgPhase2FinalView cadence.UInt64 + var dkgPhase3FinalView cadence.UInt64 + var cdcClusters cadence.Array + var cdcParticipants cadence.Array + var foundFieldCount int + + evt := cdcEvent.Type().(*cadence.EventType) + + for i, f := range evt.Fields { + switch f.Identifier { + case "counter": + foundFieldCount++ + counter, ok = cdcEvent.Fields[i].(cadence.UInt64) + if !ok { + return nil, invalidCadenceTypeError( + "counter", + cdcEvent.Fields[i], + cadence.UInt64(0), + ) + } + + case "nodeInfo": + foundFieldCount++ + cdcParticipants, ok = cdcEvent.Fields[i].(cadence.Array) + if !ok { + return nil, invalidCadenceTypeError( + "participants", + cdcEvent.Fields[i], + cadence.Array{}, + ) + } + + case "firstView": + foundFieldCount++ + firstView, ok = cdcEvent.Fields[i].(cadence.UInt64) + if !ok { + return nil, invalidCadenceTypeError( + "firstView", + cdcEvent.Fields[i], + cadence.UInt64(0), + ) + } + + case "finalView": + foundFieldCount++ + finalView, ok = cdcEvent.Fields[i].(cadence.UInt64) + if !ok { + return nil, invalidCadenceTypeError( + "finalView", + cdcEvent.Fields[i], + cadence.UInt64(0), + ) + } + + case "collectorClusters": + foundFieldCount++ + cdcClusters, ok = cdcEvent.Fields[i].(cadence.Array) + if !ok { + return nil, invalidCadenceTypeError( + "clusters", + cdcEvent.Fields[i], + cadence.Array{}, + ) + } + + case "randomSource": + foundFieldCount++ + randomSrcHex, ok = cdcEvent.Fields[i].(cadence.String) + if !ok { + return nil, invalidCadenceTypeError( + "randomSource", + cdcEvent.Fields[i], + cadence.String(""), + ) + } + + case "DKGPhase1FinalView": + foundFieldCount++ + dkgPhase1FinalView, ok = cdcEvent.Fields[i].(cadence.UInt64) + if !ok { + return nil, invalidCadenceTypeError( + "dkgPhase1FinalView", + cdcEvent.Fields[i], + cadence.UInt64(0), + ) + } + + case "DKGPhase2FinalView": + foundFieldCount++ + dkgPhase2FinalView, ok = cdcEvent.Fields[i].(cadence.UInt64) + if !ok { + return nil, invalidCadenceTypeError( + "dkgPhase2FinalView", + cdcEvent.Fields[i], + cadence.UInt64(0), + ) + } + + case "DKGPhase3FinalView": + foundFieldCount++ + dkgPhase3FinalView, ok = cdcEvent.Fields[i].(cadence.UInt64) + if !ok { + return nil, invalidCadenceTypeError( + "dkgPhase3FinalView", + cdcEvent.Fields[i], + cadence.UInt64(0), + ) + } + } } - setup.FirstView = uint64(firstView) - finalView, ok := cdcEvent.Fields[3].(cadence.UInt64) - if !ok { - return nil, invalidCadenceTypeError("finalView", cdcEvent.Fields[3], cadence.UInt64(0)) + + if foundFieldCount != expectedFieldCount { + return nil, fmt.Errorf( + "EpochSetup event required fields not found (%d != %d)", + foundFieldCount, + expectedFieldCount, + ) } - setup.FinalView = uint64(finalView) - randomSrcHex, ok := cdcEvent.Fields[5].(cadence.String) - if !ok { - return nil, invalidCadenceTypeError("randomSource", cdcEvent.Fields[5], cadence.String("")) + + setup := &flow.EpochSetup{ + Counter: uint64(counter), + FirstView: uint64(firstView), + FinalView: uint64(finalView), + DKGPhase1FinalView: uint64(dkgPhase1FinalView), + DKGPhase2FinalView: uint64(dkgPhase2FinalView), + DKGPhase3FinalView: uint64(dkgPhase3FinalView), } + // Cadence's unsafeRandom().toString() produces a string of variable length. // Here we pad it with enough 0s to meet the required length. - paddedRandomSrcHex := fmt.Sprintf("%0*s", 2*flow.EpochSetupRandomSourceLength, string(randomSrcHex)) + paddedRandomSrcHex := fmt.Sprintf( + "%0*s", + 2*flow.EpochSetupRandomSourceLength, + string(randomSrcHex), + ) setup.RandomSource, err = hex.DecodeString(paddedRandomSrcHex) if err != nil { - return nil, fmt.Errorf("could not decode random source hex (%v): %w", paddedRandomSrcHex, err) + return nil, fmt.Errorf( + "could not decode random source hex (%v): %w", + paddedRandomSrcHex, + err, + ) } - dkgPhase1FinalView, ok := cdcEvent.Fields[6].(cadence.UInt64) - if !ok { - return nil, invalidCadenceTypeError("dkgPhase1FinalView", cdcEvent.Fields[6], cadence.UInt64(0)) - } - setup.DKGPhase1FinalView = uint64(dkgPhase1FinalView) - dkgPhase2FinalView, ok := cdcEvent.Fields[7].(cadence.UInt64) - if !ok { - return nil, invalidCadenceTypeError("dkgPhase2FinalView", cdcEvent.Fields[7], cadence.UInt64(0)) - } - setup.DKGPhase2FinalView = uint64(dkgPhase2FinalView) - dkgPhase3FinalView, ok := cdcEvent.Fields[8].(cadence.UInt64) - if !ok { - return nil, invalidCadenceTypeError("dkgPhase3FinalView", cdcEvent.Fields[8], cadence.UInt64(0)) - } - setup.DKGPhase3FinalView = uint64(dkgPhase3FinalView) - // parse cluster assignments - cdcClusters, ok := cdcEvent.Fields[4].(cadence.Array) - if !ok { - return nil, invalidCadenceTypeError("clusters", cdcEvent.Fields[4], cadence.Array{}) - } setup.Assignments, err = convertClusterAssignments(cdcClusters.Values) if err != nil { return nil, fmt.Errorf("could not convert cluster assignments: %w", err) } // parse epoch participants - cdcParticipants, ok := cdcEvent.Fields[1].(cadence.Array) - if !ok { - return nil, invalidCadenceTypeError("participants", cdcEvent.Fields[1], cadence.Array{}) - } setup.Participants, err = convertParticipants(cdcParticipants.Values) if err != nil { return nil, fmt.Errorf("could not convert participants: %w", err) @@ -135,19 +244,89 @@ func convertServiceEventEpochSetup(event flow.Event) (*flow.ServiceEvent, error) // flow.Event type to a ServiceEvent type for an EpochCommit event func convertServiceEventEpochCommit(event flow.Event) (*flow.ServiceEvent, error) { - // decode bytes using jsoncdc - payload, err := json.Decode(nil, event.Payload) + // decode bytes using ccf + payload, err := ccf.Decode(nil, event.Payload) if err != nil { return nil, fmt.Errorf("could not unmarshal event payload: %w", err) } - // parse cadence types to Go types - commit := new(flow.EpochCommit) - commit.Counter = uint64(payload.(cadence.Event).Fields[0].(cadence.UInt64)) + cdcEvent, ok := payload.(cadence.Event) + if !ok { + return nil, invalidCadenceTypeError("payload", payload, cadence.Event{}) + } + + const expectedFieldCount = 3 + if len(cdcEvent.Fields) < expectedFieldCount { + return nil, fmt.Errorf( + "insufficient fields in EpochCommit event (%d < %d)", + len(cdcEvent.Fields), + expectedFieldCount, + ) + } + + if cdcEvent.Type() == nil { + return nil, fmt.Errorf("EpochCommit event doesn't have type") + } + + // Extract EpochCommit event fields + var counter cadence.UInt64 + var cdcClusterQCVotes cadence.Array + var cdcDKGKeys cadence.Array + var foundFieldCount int + + evt := cdcEvent.Type().(*cadence.EventType) + + for i, f := range evt.Fields { + switch f.Identifier { + case "counter": + foundFieldCount++ + counter, ok = cdcEvent.Fields[i].(cadence.UInt64) + if !ok { + return nil, invalidCadenceTypeError( + "counter", + cdcEvent.Fields[i], + cadence.UInt64(0), + ) + } + + case "clusterQCs": + foundFieldCount++ + cdcClusterQCVotes, ok = cdcEvent.Fields[i].(cadence.Array) + if !ok { + return nil, invalidCadenceTypeError( + "clusterQCs", + cdcEvent.Fields[i], + cadence.Array{}, + ) + } + + case "dkgPubKeys": + foundFieldCount++ + cdcDKGKeys, ok = cdcEvent.Fields[i].(cadence.Array) + if !ok { + return nil, invalidCadenceTypeError( + "dkgPubKeys", + cdcEvent.Fields[i], + cadence.Array{}, + ) + } + } + } + + if foundFieldCount != expectedFieldCount { + return nil, fmt.Errorf( + "EpochCommit event required fields not found (%d != %d)", + foundFieldCount, + expectedFieldCount, + ) + } + + commit := &flow.EpochCommit{ + Counter: uint64(counter), + } // parse cluster qc votes - cdcClusterQCVotes := payload.(cadence.Event).Fields[1].(cadence.Array).Values - commit.ClusterQCs, err = convertClusterQCVotes(cdcClusterQCVotes) + commit.ClusterQCs, err = convertClusterQCVotes(cdcClusterQCVotes.Values) if err != nil { return nil, fmt.Errorf("could not convert cluster qc votes: %w", err) } @@ -155,12 +334,10 @@ func convertServiceEventEpochCommit(event flow.Event) (*flow.ServiceEvent, error // parse DKG group key and participants // Note: this is read in the same order as `DKGClient.SubmitResult` ie. with the group public key first followed by individual keys // https://github.com/onflow/flow-go/blob/feature/dkg/module/dkg/client.go#L182-L183 - cdcDKGKeys := payload.(cadence.Event).Fields[2].(cadence.Array).Values - dkgGroupKey, dkgParticipantKeys, err := convertDKGKeys(cdcDKGKeys) + dkgGroupKey, dkgParticipantKeys, err := convertDKGKeys(cdcDKGKeys.Values) if err != nil { return nil, fmt.Errorf("could not convert DKG keys: %w", err) } - commit.DKGGroupKey = dkgGroupKey commit.DKGParticipantKeys = dkgParticipantKeys @@ -190,18 +367,67 @@ func convertClusterAssignments(cdcClusters []cadence.Value) (flow.AssignmentList return nil, invalidCadenceTypeError("cluster", cdcCluster, cadence.Struct{}) } - expectedFields := 2 - if len(cdcCluster.Fields) < expectedFields { - return nil, fmt.Errorf("insufficient fields (%d < %d)", len(cdcCluster.Fields), expectedFields) + const expectedFieldCount = 2 + if len(cdcCluster.Fields) < expectedFieldCount { + return nil, fmt.Errorf( + "insufficient fields (%d < %d)", + len(cdcCluster.Fields), + expectedFieldCount, + ) } - // ensure cluster index is valid - clusterIndex, ok := cdcCluster.Fields[0].(cadence.UInt16) - if !ok { - return nil, invalidCadenceTypeError("clusterIndex", cdcCluster.Fields[0], cadence.UInt16(0)) + if cdcCluster.Type() == nil { + return nil, fmt.Errorf("cluster struct doesn't have type") } + + // Extract cluster fields + var clusterIndex cadence.UInt16 + var weightsByNodeID cadence.Dictionary + var foundFieldCount int + + cdcClusterType := cdcCluster.Type().(*cadence.StructType) + + for i, f := range cdcClusterType.Fields { + switch f.Identifier { + case "index": + foundFieldCount++ + clusterIndex, ok = cdcCluster.Fields[i].(cadence.UInt16) + if !ok { + return nil, invalidCadenceTypeError( + "index", + cdcCluster.Fields[i], + cadence.UInt16(0), + ) + } + + case "nodeWeights": + foundFieldCount++ + weightsByNodeID, ok = cdcCluster.Fields[i].(cadence.Dictionary) + if !ok { + return nil, invalidCadenceTypeError( + "nodeWeights", + cdcCluster.Fields[i], + cadence.Dictionary{}, + ) + } + } + } + + if foundFieldCount != expectedFieldCount { + return nil, fmt.Errorf( + "cluster struct required fields not found (%d != %d)", + foundFieldCount, + expectedFieldCount, + ) + } + + // ensure cluster index is valid if int(clusterIndex) >= len(cdcClusters) { - return nil, fmt.Errorf("invalid cdcCluster index (%d) outside range [0,%d]", clusterIndex, len(cdcClusters)-1) + return nil, fmt.Errorf( + "invalid cdcCluster index (%d) outside range [0,%d]", + clusterIndex, + len(cdcClusters)-1, + ) } _, dup := indices[uint(clusterIndex)] if dup { @@ -209,20 +435,22 @@ func convertClusterAssignments(cdcClusters []cadence.Value) (flow.AssignmentList } // read weights to retrieve node IDs of cdcCluster members - weightsByNodeID, ok := cdcCluster.Fields[1].(cadence.Dictionary) - if !ok { - return nil, invalidCadenceTypeError("clusterWeights", cdcCluster.Fields[1], cadence.Dictionary{}) - } - for _, pair := range weightsByNodeID.Pairs { nodeIDString, ok := pair.Key.(cadence.String) if !ok { - return nil, invalidCadenceTypeError("clusterWeights.nodeID", pair.Key, cadence.String("")) + return nil, invalidCadenceTypeError( + "clusterWeights.nodeID", + pair.Key, + cadence.String(""), + ) } nodeID, err := flow.HexStringToIdentifier(string(nodeIDString)) if err != nil { - return nil, fmt.Errorf("could not convert hex string to identifer: %w", err) + return nil, fmt.Errorf( + "could not convert hex string to identifer: %w", + err, + ) } identifierLists[clusterIndex] = append(identifierLists[clusterIndex], nodeID) @@ -246,72 +474,160 @@ func convertParticipants(cdcParticipants []cadence.Value) (flow.IdentityList, er cdcNodeInfoStruct, ok := value.(cadence.Struct) if !ok { - return nil, invalidCadenceTypeError("cdcNodeInfoFields", value, cadence.Struct{}) + return nil, invalidCadenceTypeError( + "cdcNodeInfoFields", + value, + cadence.Struct{}, + ) } - cdcNodeInfoFields := cdcNodeInfoStruct.Fields - expectedFields := 14 - if len(cdcNodeInfoFields) < expectedFields { - return nil, fmt.Errorf("insufficient fields (%d < %d)", len(cdcNodeInfoFields), expectedFields) + const expectedFieldCount = 14 + if len(cdcNodeInfoStruct.Fields) < expectedFieldCount { + return nil, fmt.Errorf( + "insufficient fields (%d < %d)", + len(cdcNodeInfoStruct.Fields), + expectedFieldCount, + ) } - // create and assign fields to identity from cadence Struct - identity := new(flow.Identity) - role, ok := cdcNodeInfoFields[1].(cadence.UInt8) - if !ok { - return nil, invalidCadenceTypeError("nodeInfo.role", cdcNodeInfoFields[1], cadence.UInt8(0)) + if cdcNodeInfoStruct.Type() == nil { + return nil, fmt.Errorf("nodeInfo struct doesn't have type") } - identity.Role = flow.Role(role) - if !identity.Role.Valid() { - return nil, fmt.Errorf("invalid role %d", role) + + cdcNodeInfoStructType := cdcNodeInfoStruct.Type().(*cadence.StructType) + + const requiredFieldCount = 6 + var foundFieldCount int + + var nodeIDHex cadence.String + var role cadence.UInt8 + var address cadence.String + var networkKeyHex cadence.String + var stakingKeyHex cadence.String + var initialWeight cadence.UInt64 + + for i, f := range cdcNodeInfoStructType.Fields { + switch f.Identifier { + case "id": + foundFieldCount++ + nodeIDHex, ok = cdcNodeInfoStruct.Fields[i].(cadence.String) + if !ok { + return nil, invalidCadenceTypeError( + "nodeInfo.id", + cdcNodeInfoStruct.Fields[i], + cadence.String(""), + ) + } + + case "role": + foundFieldCount++ + role, ok = cdcNodeInfoStruct.Fields[i].(cadence.UInt8) + if !ok { + return nil, invalidCadenceTypeError( + "nodeInfo.role", + cdcNodeInfoStruct.Fields[i], + cadence.UInt8(0), + ) + } + + case "networkingAddress": + foundFieldCount++ + address, ok = cdcNodeInfoStruct.Fields[i].(cadence.String) + if !ok { + return nil, invalidCadenceTypeError( + "nodeInfo.networkingAddress", + cdcNodeInfoStruct.Fields[i], + cadence.String(""), + ) + } + + case "networkingKey": + foundFieldCount++ + networkKeyHex, ok = cdcNodeInfoStruct.Fields[i].(cadence.String) + if !ok { + return nil, invalidCadenceTypeError( + "nodeInfo.networkingKey", + cdcNodeInfoStruct.Fields[i], + cadence.String(""), + ) + } + + case "stakingKey": + foundFieldCount++ + stakingKeyHex, ok = cdcNodeInfoStruct.Fields[i].(cadence.String) + if !ok { + return nil, invalidCadenceTypeError( + "nodeInfo.stakingKey", + cdcNodeInfoStruct.Fields[i], + cadence.String(""), + ) + } + + case "initialWeight": + foundFieldCount++ + initialWeight, ok = cdcNodeInfoStruct.Fields[i].(cadence.UInt64) + if !ok { + return nil, invalidCadenceTypeError( + "nodeInfo.initialWeight", + cdcNodeInfoStruct.Fields[i], + cadence.UInt64(0), + ) + } + } } - address, ok := cdcNodeInfoFields[2].(cadence.String) - if !ok { - return nil, invalidCadenceTypeError("nodeInfo.address", cdcNodeInfoFields[2], cadence.String("")) + if foundFieldCount != requiredFieldCount { + return nil, fmt.Errorf( + "NodeInfo struct required fields not found (%d != %d)", + foundFieldCount, + requiredFieldCount, + ) } - identity.Address = string(address) - initialWeight, ok := cdcNodeInfoFields[13].(cadence.UInt64) - if !ok { - return nil, invalidCadenceTypeError("nodeInfo.initialWeight", cdcNodeInfoFields[13], cadence.UInt64(0)) + if !flow.Role(role).Valid() { + return nil, fmt.Errorf("invalid role %d", role) } - identity.Weight = uint64(initialWeight) - // convert nodeID string into identifier - nodeIDHex, ok := cdcNodeInfoFields[0].(cadence.String) - if !ok { - return nil, invalidCadenceTypeError("nodeInfo.id", cdcNodeInfoFields[0], cadence.String("")) + identity := &flow.Identity{ + Address: string(address), + Weight: uint64(initialWeight), + Role: flow.Role(role), } + + // convert nodeID string into identifier identity.NodeID, err = flow.HexStringToIdentifier(string(nodeIDHex)) if err != nil { return nil, fmt.Errorf("could not convert hex string to identifer: %w", err) } // parse to PublicKey the networking key hex string - networkKeyHex, ok := cdcNodeInfoFields[3].(cadence.String) - if !ok { - return nil, invalidCadenceTypeError("nodeInfo.networkKey", cdcNodeInfoFields[3], cadence.String("")) - } networkKeyBytes, err := hex.DecodeString(string(networkKeyHex)) if err != nil { - return nil, fmt.Errorf("could not decode network public key into bytes: %w", err) + return nil, fmt.Errorf( + "could not decode network public key into bytes: %w", + err, + ) } - identity.NetworkPubKey, err = crypto.DecodePublicKey(crypto.ECDSAP256, networkKeyBytes) + identity.NetworkPubKey, err = crypto.DecodePublicKey( + crypto.ECDSAP256, + networkKeyBytes, + ) if err != nil { return nil, fmt.Errorf("could not decode network public key: %w", err) } // parse to PublicKey the staking key hex string - stakingKeyHex, ok := cdcNodeInfoFields[4].(cadence.String) - if !ok { - return nil, invalidCadenceTypeError("nodeInfo.stakingKey", cdcNodeInfoFields[4], cadence.String("")) - } stakingKeyBytes, err := hex.DecodeString(string(stakingKeyHex)) if err != nil { - return nil, fmt.Errorf("could not decode staking public key into bytes: %w", err) + return nil, fmt.Errorf( + "could not decode staking public key into bytes: %w", + err, + ) } - identity.StakingPubKey, err = crypto.DecodePublicKey(crypto.BLSBLS12381, stakingKeyBytes) + identity.StakingPubKey, err = crypto.DecodePublicKey( + crypto.BLSBLS12381, + stakingKeyBytes, + ) if err != nil { return nil, fmt.Errorf("could not decode staking public key: %w", err) } @@ -326,7 +642,10 @@ func convertParticipants(cdcParticipants []cadence.Value) (flow.IdentityList, er // convertClusterQCVotes converts raw cluster QC votes from the EpochCommit event // to a representation suitable for inclusion in the protocol state. Votes are // aggregated as part of this conversion. -func convertClusterQCVotes(cdcClusterQCs []cadence.Value) ([]flow.ClusterQCVoteData, error) { +func convertClusterQCVotes(cdcClusterQCs []cadence.Value) ( + []flow.ClusterQCVoteData, + error, +) { // avoid duplicate indices indices := make(map[uint]struct{}) @@ -339,37 +658,101 @@ func convertClusterQCVotes(cdcClusterQCs []cadence.Value) ([]flow.ClusterQCVoteD for _, cdcClusterQC := range cdcClusterQCs { cdcClusterQCStruct, ok := cdcClusterQC.(cadence.Struct) if !ok { - return nil, invalidCadenceTypeError("clusterQC", cdcClusterQC, cadence.Struct{}) + return nil, invalidCadenceTypeError( + "clusterQC", + cdcClusterQC, + cadence.Struct{}, + ) } - cdcClusterQCFields := cdcClusterQCStruct.Fields - expectedFields := 4 - if len(cdcClusterQCFields) < expectedFields { - return nil, fmt.Errorf("insufficient fields (%d < %d)", len(cdcClusterQCFields), expectedFields) + const expectedFieldCount = 4 + if len(cdcClusterQCStruct.Fields) < expectedFieldCount { + return nil, fmt.Errorf( + "insufficient fields (%d < %d)", + len(cdcClusterQCStruct.Fields), + expectedFieldCount, + ) } - index, ok := cdcClusterQCFields[0].(cadence.UInt16) - if !ok { - return nil, invalidCadenceTypeError("clusterQC.index", cdcClusterQCFields[0], cadence.UInt16(0)) + if cdcClusterQCStruct.Type() == nil { + return nil, fmt.Errorf("clusterQC struct doesn't have type") } + + cdcClusterQCStructType := cdcClusterQCStruct.Type().(*cadence.StructType) + + const requiredFieldCount = 3 + var foundFieldCount int + + var index cadence.UInt16 + var cdcVoterIDs cadence.Array + var cdcRawVotes cadence.Array + + for i, f := range cdcClusterQCStructType.Fields { + switch f.Identifier { + case "index": + foundFieldCount++ + index, ok = cdcClusterQCStruct.Fields[i].(cadence.UInt16) + if !ok { + return nil, invalidCadenceTypeError( + "ClusterQC.index", + cdcClusterQCStruct.Fields[i], + cadence.UInt16(0), + ) + } + + case "voteSignatures": + foundFieldCount++ + cdcRawVotes, ok = cdcClusterQCStruct.Fields[i].(cadence.Array) + if !ok { + return nil, invalidCadenceTypeError( + "clusterQC.voteSignatures", + cdcClusterQCStruct.Fields[i], + cadence.Array{}, + ) + } + + case "voterIDs": + foundFieldCount++ + cdcVoterIDs, ok = cdcClusterQCStruct.Fields[i].(cadence.Array) + if !ok { + return nil, invalidCadenceTypeError( + "clusterQC.voterIDs", + cdcClusterQCStruct.Fields[i], + cadence.Array{}, + ) + } + } + } + + if foundFieldCount != requiredFieldCount { + return nil, fmt.Errorf( + "clusterQC struct required fields not found (%d != %d)", + foundFieldCount, + requiredFieldCount, + ) + } + if int(index) >= len(cdcClusterQCs) { - return nil, fmt.Errorf("invalid index (%d) not in range [0,%d]", index, len(cdcClusterQCs)) + return nil, fmt.Errorf( + "invalid index (%d) not in range [0,%d]", + index, + len(cdcClusterQCs), + ) } _, dup := indices[uint(index)] if dup { return nil, fmt.Errorf("duplicate cluster QC index (%d)", index) } - cdcVoterIDs, ok := cdcClusterQCFields[3].(cadence.Array) - if !ok { - return nil, invalidCadenceTypeError("clusterQC.voterIDs", cdcClusterQCFields[2], cadence.Array{}) - } - voterIDs := make([]flow.Identifier, 0, len(cdcVoterIDs.Values)) for _, cdcVoterID := range cdcVoterIDs.Values { voterIDHex, ok := cdcVoterID.(cadence.String) if !ok { - return nil, invalidCadenceTypeError("clusterQC[i].voterID", cdcVoterID, cadence.String("")) + return nil, invalidCadenceTypeError( + "clusterQC[i].voterID", + cdcVoterID, + cadence.String(""), + ) } voterID, err := flow.HexStringToIdentifier(string(voterIDHex)) if err != nil { @@ -379,12 +762,15 @@ func convertClusterQCVotes(cdcClusterQCs []cadence.Value) ([]flow.ClusterQCVoteD } // gather all the vote signatures - cdcRawVotes := cdcClusterQCFields[1].(cadence.Array) signatures := make([]crypto.Signature, 0, len(cdcRawVotes.Values)) for _, cdcRawVote := range cdcRawVotes.Values { rawVoteHex, ok := cdcRawVote.(cadence.String) if !ok { - return nil, invalidCadenceTypeError("clusterQC[i].vote", cdcRawVote, cadence.String("")) + return nil, invalidCadenceTypeError( + "clusterQC[i].vote", + cdcRawVote, + cadence.String(""), + ) } rawVoteBytes, err := hex.DecodeString(string(rawVoteHex)) if err != nil { @@ -436,7 +822,11 @@ func convertClusterQCVotes(cdcClusterQCs []cadence.Value) ([]flow.ClusterQCVoteD // convertDKGKeys converts hex-encoded DKG public keys as received by the DKG // smart contract into crypto.PublicKey representations suitable for inclusion // in the protocol state. -func convertDKGKeys(cdcDKGKeys []cadence.Value) (groupKey crypto.PublicKey, participantKeys []crypto.PublicKey, err error) { +func convertDKGKeys(cdcDKGKeys []cadence.Value) ( + groupKey crypto.PublicKey, + participantKeys []crypto.PublicKey, + err error, +) { hexDKGKeys := make([]string, 0, len(cdcDKGKeys)) for _, value := range cdcDKGKeys { @@ -454,7 +844,10 @@ func convertDKGKeys(cdcDKGKeys []cadence.Value) (groupKey crypto.PublicKey, part // decode group public key groupKeyBytes, err := hex.DecodeString(groupPubKeyHex) if err != nil { - return nil, nil, fmt.Errorf("could not decode group public key into bytes: %w", err) + return nil, nil, fmt.Errorf( + "could not decode group public key into bytes: %w", + err, + ) } groupKey, err = crypto.DecodePublicKey(crypto.BLSBLS12381, groupKeyBytes) if err != nil { @@ -467,7 +860,10 @@ func convertDKGKeys(cdcDKGKeys []cadence.Value) (groupKey crypto.PublicKey, part pubKeyBytes, err := hex.DecodeString(pubKeyString) if err != nil { - return nil, nil, fmt.Errorf("could not decode individual public key into bytes: %w", err) + return nil, nil, fmt.Errorf( + "could not decode individual public key into bytes: %w", + err, + ) } pubKey, err := crypto.DecodePublicKey(crypto.BLSBLS12381, pubKeyBytes) if err != nil { @@ -479,9 +875,384 @@ func convertDKGKeys(cdcDKGKeys []cadence.Value) (groupKey crypto.PublicKey, part return groupKey, dkgParticipantKeys, nil } -func invalidCadenceTypeError(fieldName string, actualType, expectedType cadence.Value) error { - return fmt.Errorf("invalid Cadence type for field %s (got=%s, expected=%s)", +func invalidCadenceTypeError( + fieldName string, + actualType, expectedType cadence.Value, +) error { + return fmt.Errorf( + "invalid Cadence type for field %s (got=%s, expected=%s)", fieldName, actualType.Type().ID(), - expectedType.Type().ID()) + expectedType.Type().ID(), + ) +} + +func convertServiceEventVersionBeacon(event flow.Event) (*flow.ServiceEvent, error) { + payload, err := ccf.Decode(nil, event.Payload) + if err != nil { + return nil, fmt.Errorf("could not unmarshal event payload: %w", err) + } + + versionBeacon, err := DecodeCadenceValue( + "VersionBeacon payload", payload, func(cdcEvent cadence.Event) ( + flow.VersionBeacon, + error, + ) { + const expectedFieldCount = 2 + if len(cdcEvent.Fields) != expectedFieldCount { + return flow.VersionBeacon{}, fmt.Errorf( + "unexpected number of fields in VersionBeacon event (%d != %d)", + len(cdcEvent.Fields), + expectedFieldCount, + ) + } + + if cdcEvent.Type() == nil { + return flow.VersionBeacon{}, fmt.Errorf("VersionBeacon event doesn't have type") + } + + var versionBoundariesValue, sequenceValue cadence.Value + var foundFieldCount int + + evt := cdcEvent.Type().(*cadence.EventType) + + for i, f := range evt.Fields { + switch f.Identifier { + case "versionBoundaries": + foundFieldCount++ + versionBoundariesValue = cdcEvent.Fields[i] + + case "sequence": + foundFieldCount++ + sequenceValue = cdcEvent.Fields[i] + } + } + + if foundFieldCount != expectedFieldCount { + return flow.VersionBeacon{}, fmt.Errorf( + "VersionBeacon event required fields not found (%d != %d)", + foundFieldCount, + expectedFieldCount, + ) + } + + versionBoundaries, err := DecodeCadenceValue( + ".versionBoundaries", versionBoundariesValue, convertVersionBoundaries, + ) + if err != nil { + return flow.VersionBeacon{}, err + } + + sequence, err := DecodeCadenceValue( + ".sequence", sequenceValue, func(cadenceVal cadence.UInt64) ( + uint64, + error, + ) { + return uint64(cadenceVal), nil + }, + ) + if err != nil { + return flow.VersionBeacon{}, err + } + + return flow.VersionBeacon{ + VersionBoundaries: versionBoundaries, + Sequence: sequence, + }, err + }, + ) + if err != nil { + return nil, err + } + + // a converted version beacon event should also be valid + if err := versionBeacon.Validate(); err != nil { + return nil, fmt.Errorf("invalid VersionBeacon event: %w", err) + } + + // create the service event + serviceEvent := &flow.ServiceEvent{ + Type: flow.ServiceEventVersionBeacon, + Event: &versionBeacon, + } + + return serviceEvent, nil +} + +func convertVersionBoundaries(array cadence.Array) ( + []flow.VersionBoundary, + error, +) { + boundaries := make([]flow.VersionBoundary, len(array.Values)) + + for i, cadenceVal := range array.Values { + boundary, err := DecodeCadenceValue( + fmt.Sprintf(".Values[%d]", i), + cadenceVal, + func(structVal cadence.Struct) ( + flow.VersionBoundary, + error, + ) { + const expectedFieldCount = 2 + if len(structVal.Fields) < expectedFieldCount { + return flow.VersionBoundary{}, fmt.Errorf( + "incorrect number of fields (%d != %d)", + len(structVal.Fields), + expectedFieldCount, + ) + } + + if structVal.Type() == nil { + return flow.VersionBoundary{}, fmt.Errorf("VersionBoundary struct doesn't have type") + } + + var blockHeightValue, versionValue cadence.Value + var foundFieldCount int + + structValType := structVal.Type().(*cadence.StructType) + + for i, f := range structValType.Fields { + switch f.Identifier { + case "blockHeight": + foundFieldCount++ + blockHeightValue = structVal.Fields[i] + + case "version": + foundFieldCount++ + versionValue = structVal.Fields[i] + } + } + + if foundFieldCount != expectedFieldCount { + return flow.VersionBoundary{}, fmt.Errorf( + "VersionBoundaries struct required fields not found (%d != %d)", + foundFieldCount, + expectedFieldCount, + ) + } + + height, err := DecodeCadenceValue( + ".blockHeight", + blockHeightValue, + func(cadenceVal cadence.UInt64) ( + uint64, + error, + ) { + return uint64(cadenceVal), nil + }, + ) + if err != nil { + return flow.VersionBoundary{}, err + } + + version, err := DecodeCadenceValue( + ".version", + versionValue, + convertSemverVersion, + ) + if err != nil { + return flow.VersionBoundary{}, err + } + + return flow.VersionBoundary{ + BlockHeight: height, + Version: version, + }, nil + }, + ) + if err != nil { + return nil, err + } + boundaries[i] = boundary + } + + return boundaries, nil +} + +func convertSemverVersion(structVal cadence.Struct) ( + string, + error, +) { + const expectedFieldCount = 4 + if len(structVal.Fields) < expectedFieldCount { + return "", fmt.Errorf( + "incorrect number of fields (%d != %d)", + len(structVal.Fields), + expectedFieldCount, + ) + } + + if structVal.Type() == nil { + return "", fmt.Errorf("Semver struct doesn't have type") + } + + var majorValue, minorValue, patchValue, preReleaseValue cadence.Value + var foundFieldCount int + + structValType := structVal.Type().(*cadence.StructType) + + for i, f := range structValType.Fields { + switch f.Identifier { + case "major": + foundFieldCount++ + majorValue = structVal.Fields[i] + + case "minor": + foundFieldCount++ + minorValue = structVal.Fields[i] + + case "patch": + foundFieldCount++ + patchValue = structVal.Fields[i] + + case "preRelease": + foundFieldCount++ + preReleaseValue = structVal.Fields[i] + } + } + + if foundFieldCount != expectedFieldCount { + return "", fmt.Errorf( + "Semver struct required fields not found (%d != %d)", + foundFieldCount, + expectedFieldCount, + ) + } + + major, err := DecodeCadenceValue( + ".major", + majorValue, + func(cadenceVal cadence.UInt8) ( + uint64, + error, + ) { + return uint64(cadenceVal), nil + }, + ) + if err != nil { + return "", err + } + + minor, err := DecodeCadenceValue( + ".minor", + minorValue, + func(cadenceVal cadence.UInt8) ( + uint64, + error, + ) { + return uint64(cadenceVal), nil + }, + ) + if err != nil { + return "", err + } + + patch, err := DecodeCadenceValue( + ".patch", + patchValue, + func(cadenceVal cadence.UInt8) ( + uint64, + error, + ) { + return uint64(cadenceVal), nil + }, + ) + if err != nil { + return "", err + } + + preRelease, err := DecodeCadenceValue( + ".preRelease", + preReleaseValue, + func(cadenceVal cadence.Optional) ( + string, + error, + ) { + if cadenceVal.Value == nil { + return "", nil + } + + return DecodeCadenceValue( + "!", + cadenceVal.Value, + func(cadenceVal cadence.String) ( + string, + error, + ) { + return string(cadenceVal), nil + }, + ) + }, + ) + if err != nil { + return "", err + } + + version := semver.Version{ + Major: int64(major), + Minor: int64(minor), + Patch: int64(patch), + PreRelease: semver.PreRelease(preRelease), + } + + return version.String(), nil + +} + +type decodeError struct { + location string + err error +} + +func (e decodeError) Error() string { + if e.err != nil { + return fmt.Sprintf("decoding error %s: %s", e.location, e.err.Error()) + } + return fmt.Sprintf("decoding error %s", e.location) +} + +func (e decodeError) Unwrap() error { + return e.err +} + +func DecodeCadenceValue[From cadence.Value, Into any]( + location string, + value cadence.Value, + decodeInner func(From) (Into, error), +) (Into, error) { + var defaultInto Into + if value == nil { + return defaultInto, decodeError{ + location: location, + err: nil, + } + } + + convertedValue, is := value.(From) + if !is { + return defaultInto, decodeError{ + location: location, + err: fmt.Errorf( + "invalid Cadence type (got=%T, expected=%T)", + value, + *new(From), + ), + } + } + + inner, err := decodeInner(convertedValue) + if err != nil { + if err, is := err.(decodeError); is { + return defaultInto, decodeError{ + location: location + err.location, + err: err.err, + } + } + return defaultInto, decodeError{ + location: location, + err: err, + } + } + + return inner, nil } diff --git a/model/convert/service_event_test.go b/model/convert/service_event_test.go index 0a14a0be7d5..88ba8c4d3ca 100644 --- a/model/convert/service_event_test.go +++ b/model/convert/service_event_test.go @@ -1,11 +1,16 @@ package convert_test import ( + "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" + + "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/model/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -15,36 +20,322 @@ func TestEventConversion(t *testing.T) { chainID := flow.Emulator - t.Run("epoch setup", func(t *testing.T) { + t.Run( + "epoch setup", func(t *testing.T) { + + fixture, expected := unittest.EpochSetupFixtureByChainID(chainID) + + // convert Cadence types to Go types + event, err := convert.ServiceEvent(chainID, fixture) + require.NoError(t, err) + require.NotNil(t, event) + + // cast event type to epoch setup + actual, ok := event.Event.(*flow.EpochSetup) + require.True(t, ok) + + assert.Equal(t, expected, actual) + + }, + ) + + t.Run( + "epoch commit", func(t *testing.T) { + + fixture, expected := unittest.EpochCommitFixtureByChainID(chainID) + + // convert Cadence types to Go types + event, err := convert.ServiceEvent(chainID, fixture) + require.NoError(t, err) + require.NotNil(t, event) + + // cast event type to epoch commit + actual, ok := event.Event.(*flow.EpochCommit) + require.True(t, ok) + + assert.Equal(t, expected, actual) + }, + ) + + t.Run( + "version beacon", func(t *testing.T) { + + fixture, expected := unittest.VersionBeaconFixtureByChainID(chainID) + + // convert Cadence types to Go types + event, err := convert.ServiceEvent(chainID, fixture) + require.NoError(t, err) + require.NotNil(t, event) + + // cast event type to version beacon + actual, ok := event.Event.(*flow.VersionBeacon) + require.True(t, ok) + + assert.Equal(t, expected, actual) + }, + ) +} + +func TestDecodeCadenceValue(t *testing.T) { + + tests := []struct { + name string + location string + value cadence.Value + decodeInner func(cadence.Value) (interface{}, error) + expected interface{} + expectError bool + expectedLocation string + }{ + { + name: "Basic", + location: "test", + value: cadence.UInt64(42), + decodeInner: func(value cadence.Value) ( + interface{}, + error, + ) { + return 42, nil + }, + expected: 42, + expectError: false, + }, + { + name: "Nil value", + location: "test", + value: nil, + decodeInner: func(value cadence.Value) ( + interface{}, + error, + ) { + return 42, nil + }, + expected: nil, + expectError: true, + }, + { + name: "Custom decode error", + location: "test", + value: cadence.String("hello"), + decodeInner: func(value cadence.Value) ( + interface{}, + error, + ) { + return nil, fmt.Errorf("custom error") + }, + expected: nil, + expectError: true, + }, + { + name: "Nested location", + location: "outer", + value: cadence.String("hello"), + decodeInner: func(value cadence.Value) (interface{}, error) { + return convert.DecodeCadenceValue( + ".inner", value, + func(value cadence.Value) (interface{}, error) { + return nil, fmt.Errorf("custom error") + }, + ) + }, + expected: nil, + expectError: true, + expectedLocation: "outer.inner", + }, + } + + for _, tt := range tests { + t.Run( + tt.name, func(t *testing.T) { + result, err := convert.DecodeCadenceValue( + tt.location, + tt.value, + tt.decodeInner, + ) + + if tt.expectError { + assert.Error(t, err) + if tt.expectedLocation != "" { + assert.Contains(t, err.Error(), tt.expectedLocation) + } + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + } + }, + ) + } +} + +func TestVersionBeaconEventConversion(t *testing.T) { + versionBoundaryType := unittest.NewNodeVersionBeaconVersionBoundaryStructType() + semverType := unittest.NewNodeVersionBeaconSemverStructType() + eventType := unittest.NewNodeVersionBeaconVersionBeaconEventType() - fixture, expected := unittest.EpochSetupFixtureByChainID(chainID) + type vbTestCase struct { + name string + event cadence.Event + converted *flow.VersionBeacon + expectAndHandleError func(t *testing.T, err error) + } - // convert Cadence types to Go types - event, err := convert.ServiceEvent(chainID, fixture) - require.NoError(t, err) - require.NotNil(t, event) + runVersionBeaconTestCase := func(t *testing.T, test vbTestCase) { + chainID := flow.Emulator + t.Run(test.name, func(t *testing.T) { + events, err := systemcontracts.ServiceEventsForChain(chainID) + if err != nil { + panic(err) + } - // cast event type to epoch setup - actual, ok := event.Event.(*flow.EpochSetup) - require.True(t, ok) + event := unittest.EventFixture(events.VersionBeacon.EventType(), 1, 1, unittest.IdentifierFixture(), 0) + event.Payload, err = ccf.Encode(test.event) + require.NoError(t, err) - assert.Equal(t, expected, actual) + // convert Cadence types to Go types + serviceEvent, err := convert.ServiceEvent(chainID, event) - }) + if test.expectAndHandleError != nil { + require.Error(t, err) + test.expectAndHandleError(t, err) + return + } - t.Run("epoch commit", func(t *testing.T) { + require.NoError(t, err) + require.NotNil(t, event) - fixture, expected := unittest.EpochCommitFixtureByChainID(chainID) + // cast event type to version beacon + actual, ok := serviceEvent.Event.(*flow.VersionBeacon) + require.True(t, ok) - // convert Cadence types to Go types - event, err := convert.ServiceEvent(chainID, fixture) - require.NoError(t, err) - require.NotNil(t, event) + require.Equal(t, test.converted, actual) + }) + } - // cast event type to epoch commit - actual, ok := event.Event.(*flow.EpochCommit) - require.True(t, ok) + runVersionBeaconTestCase(t, + vbTestCase{ + name: "with pre-release", + event: cadence.NewEvent( + []cadence.Value{ + // versionBoundaries + cadence.NewArray( + []cadence.Value{ + cadence.NewStruct( + []cadence.Value{ + // blockHeight + cadence.UInt64(44), + // version + cadence.NewStruct( + []cadence.Value{ + // major + cadence.UInt8(2), + // minor + cadence.UInt8(13), + // patch + cadence.UInt8(7), + // preRelease + cadence.NewOptional(cadence.String("test")), + }, + ).WithType(semverType), + }, + ).WithType(versionBoundaryType), + }, + ).WithType(cadence.NewVariableSizedArrayType(versionBoundaryType)), + // sequence + cadence.UInt64(5), + }, + ).WithType(eventType), + converted: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + { + BlockHeight: 44, + Version: "2.13.7-test", + }, + }, + Sequence: 5, + }, + }, + ) - assert.Equal(t, expected, actual) - }) + runVersionBeaconTestCase(t, + vbTestCase{ + name: "without pre-release", + event: cadence.NewEvent( + []cadence.Value{ + // versionBoundaries + cadence.NewArray( + []cadence.Value{ + cadence.NewStruct( + []cadence.Value{ + // blockHeight + cadence.UInt64(44), + // version + cadence.NewStruct( + []cadence.Value{ + // major + cadence.UInt8(2), + // minor + cadence.UInt8(13), + // patch + cadence.UInt8(7), + // preRelease + cadence.NewOptional(nil), + }, + ).WithType(semverType), + }, + ).WithType(versionBoundaryType), + }, + ).WithType(cadence.NewVariableSizedArrayType(versionBoundaryType)), + // sequence + cadence.UInt64(5), + }, + ).WithType(eventType), + converted: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + { + BlockHeight: 44, + Version: "2.13.7", + }, + }, + Sequence: 5, + }, + }, + ) + runVersionBeaconTestCase(t, + vbTestCase{ + name: "invalid pre-release", + event: cadence.NewEvent( + []cadence.Value{ + // versionBoundaries + cadence.NewArray( + []cadence.Value{ + cadence.NewStruct( + []cadence.Value{ + // blockHeight + cadence.UInt64(44), + // version + cadence.NewStruct( + []cadence.Value{ + // major + cadence.UInt8(2), + // minor + cadence.UInt8(13), + // patch + cadence.UInt8(7), + // preRelease + cadence.NewOptional(cadence.String("/slashes.not.allowed")), + }, + ).WithType(semverType), + }, + ).WithType(versionBoundaryType), + }, + ).WithType(cadence.NewVariableSizedArrayType(versionBoundaryType)), + // sequence + cadence.UInt64(5), + }, + ).WithType(eventType), + expectAndHandleError: func(t *testing.T, err error) { + require.ErrorContains(t, err, "failed to validate pre-release") + }, + }, + ) } diff --git a/model/flow/address_test.go b/model/flow/address_test.go index b71a0d567ed..28e99efa315 100644 --- a/model/flow/address_test.go +++ b/model/flow/address_test.go @@ -5,7 +5,6 @@ import ( "math/bits" "math/rand" "testing" - "time" "github.com/onflow/cadence" "github.com/onflow/cadence/runtime/common" @@ -167,9 +166,6 @@ func testAddressConstants(t *testing.T) { const invalidCodeWord = uint64(0xab2ae42382900010) func testAddressGeneration(t *testing.T) { - // seed random generator - rand.Seed(time.Now().UnixNano()) - // loops in each test const loop = 50 @@ -260,9 +256,6 @@ func testAddressGeneration(t *testing.T) { } func testAddressesIntersection(t *testing.T) { - // seed random generator - rand.Seed(time.Now().UnixNano()) - // loops in each test const loop = 25 @@ -329,9 +322,6 @@ func testAddressesIntersection(t *testing.T) { } func testIndexFromAddress(t *testing.T) { - // seed random generator - rand.Seed(time.Now().UnixNano()) - // loops in each test const loop = 50 @@ -370,9 +360,6 @@ func testIndexFromAddress(t *testing.T) { } func TestUint48(t *testing.T) { - // seed random generator - rand.Seed(time.Now().UnixNano()) - const loop = 50 // test consistensy of putUint48 and uint48 for i := 0; i < loop; i++ { diff --git a/model/flow/chain.go b/model/flow/chain.go index 32ceb62467d..adb4080b44b 100644 --- a/model/flow/chain.go +++ b/model/flow/chain.go @@ -12,6 +12,7 @@ import ( // // Chain IDs are used used to prevent replay attacks and to support network-specific address generation. type ChainID string +type ChainIDList []ChainID const ( // Mainnet is the chain ID for the mainnet chain. diff --git a/model/flow/chunk.go b/model/flow/chunk.go index 48102bac3e9..5fb4c0bdf68 100644 --- a/model/flow/chunk.go +++ b/model/flow/chunk.go @@ -1,39 +1,21 @@ package flow type ChunkBody struct { - // Block id of the execution result this chunk belongs to - BlockID Identifier - CollectionIndex uint - // start state when starting executing this chunk - StartState StateCommitment - - // // execution info - // - - // number of transactions inside the collection - NumberOfTransactions uint64 + StartState StateCommitment // start state when starting executing this chunk + EventCollection Identifier // Events generated by executing results + BlockID Identifier // Block id of the execution result this chunk belongs to - // Events generated by executing results - EventCollection Identifier - - // // Computation consumption info - // - - // total amount of computation used by running all txs in this chunk - TotalComputationUsed uint64 + TotalComputationUsed uint64 // total amount of computation used by running all txs in this chunk + NumberOfTransactions uint64 // number of transactions inside the collection } type Chunk struct { ChunkBody - // TODO(patrick): combine Index with body's CollectionIndex. Also typedef - // ChunkIndex (chunk index is inconsistently represented as uint64, int, - // uint) - Index uint64 // chunk index inside the ER (starts from zero) // EndState inferred from next chunk or from the ER EndState StateCommitment diff --git a/model/flow/entity.go b/model/flow/entity.go index d91a9fa6b34..963d0b15791 100644 --- a/model/flow/entity.go +++ b/model/flow/entity.go @@ -16,3 +16,11 @@ type Entity interface { // data such as signatures. Checksum() Identifier } + +func EntitiesToIDs[T Entity](entities []T) []Identifier { + ids := make([]Identifier, 0, len(entities)) + for _, entity := range entities { + ids = append(ids, entity.ID()) + } + return ids +} diff --git a/model/flow/identifier.go b/model/flow/identifier.go index 62ad2a64735..e205e74a716 100644 --- a/model/flow/identifier.go +++ b/model/flow/identifier.go @@ -6,7 +6,6 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "math/rand" "reflect" "github.com/ipfs/go-cid" @@ -16,6 +15,7 @@ import ( "github.com/onflow/flow-go/crypto/hash" "github.com/onflow/flow-go/model/fingerprint" "github.com/onflow/flow-go/storage/merkle" + "github.com/onflow/flow-go/utils/rand" ) const IdentifierLen = 32 @@ -179,21 +179,24 @@ func CheckConcatSum(sum Identifier, fps ...Identifier) bool { return sum == computed } -// Sample returns random sample of length 'size' of the ids -// [Fisher-Yates shuffle](https://en.wikipedia.org/wiki/Fisher-Yates_shuffle). -func Sample(size uint, ids ...Identifier) []Identifier { +// Sample returns non-deterministic random sample of length 'size' of the ids +func Sample(size uint, ids ...Identifier) ([]Identifier, error) { n := uint(len(ids)) dup := make([]Identifier, 0, n) dup = append(dup, ids...) // if sample size is greater than total size, return all the elements if n <= size { - return dup + return dup, nil } - for i := uint(0); i < size; i++ { - j := uint(rand.Intn(int(n - i))) - dup[i], dup[j+i] = dup[j+i], dup[i] + swap := func(i, j uint) { + dup[i], dup[j] = dup[j], dup[i] } - return dup[:size] + + err := rand.Samples(n, size, swap) + if err != nil { + return nil, fmt.Errorf("generating randoms failed: %w", err) + } + return dup[:size], nil } func CidToId(c cid.Cid) (Identifier, error) { diff --git a/model/flow/identifierList.go b/model/flow/identifierList.go index ec77a04a98f..1cf3e0263a8 100644 --- a/model/flow/identifierList.go +++ b/model/flow/identifierList.go @@ -2,7 +2,6 @@ package flow import ( "bytes" - "math/rand" "sort" "github.com/rs/zerolog/log" @@ -103,15 +102,8 @@ func (il IdentifierList) Union(other IdentifierList) IdentifierList { return union } -// DeterministicSample returns deterministic random sample from the `IdentifierList` using the given seed -func (il IdentifierList) DeterministicSample(size uint, seed int64) IdentifierList { - rand.Seed(seed) - return il.Sample(size) -} - // Sample returns random sample of length 'size' of the ids -// [Fisher-Yates shuffle](https://en.wikipedia.org/wiki/Fisher–Yates_shuffle). -func (il IdentifierList) Sample(size uint) IdentifierList { +func (il IdentifierList) Sample(size uint) (IdentifierList, error) { return Sample(size, il...) } diff --git a/model/flow/identifierList_test.go b/model/flow/identifierList_test.go index b878938a5e3..7e18b6ee921 100644 --- a/model/flow/identifierList_test.go +++ b/model/flow/identifierList_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "sort" "testing" - "time" "github.com/stretchr/testify/require" @@ -21,7 +20,7 @@ func TestIdentifierListSort(t *testing.T) { var ids flow.IdentifierList = unittest.IdentifierListFixture(count) // shuffles array before sorting to enforce some pseudo-randomness - rand.Seed(time.Now().UnixNano()) + rand.Shuffle(ids.Len(), ids.Swap) sort.Sort(ids) diff --git a/model/flow/identifier_test.go b/model/flow/identifier_test.go index a4362e95f37..7ac5dd3df89 100644 --- a/model/flow/identifier_test.go +++ b/model/flow/identifier_test.go @@ -1,10 +1,10 @@ package flow_test import ( + "crypto/rand" "encoding/binary" "encoding/json" "fmt" - "math/rand" "testing" blocks "github.com/ipfs/go-block-format" @@ -66,20 +66,23 @@ func TestIdentifierSample(t *testing.T) { t.Run("Sample creates a random sample", func(t *testing.T) { sampleSize := uint(5) - sample := flow.Sample(sampleSize, ids...) + sample, err := flow.Sample(sampleSize, ids...) + require.NoError(t, err) require.Len(t, sample, int(sampleSize)) require.NotEqual(t, sample, ids[:sampleSize]) }) t.Run("sample size greater than total size results in the original list", func(t *testing.T) { sampleSize := uint(len(ids) + 1) - sample := flow.Sample(sampleSize, ids...) + sample, err := flow.Sample(sampleSize, ids...) + require.NoError(t, err) require.Equal(t, sample, ids) }) t.Run("sample size of zero results in an empty list", func(t *testing.T) { sampleSize := uint(0) - sample := flow.Sample(sampleSize, ids...) + sample, err := flow.Sample(sampleSize, ids...) + require.NoError(t, err) require.Empty(t, sample) }) } @@ -131,7 +134,8 @@ func TestCIDConversion(t *testing.T) { // generate random CID data := make([]byte, 4) - rand.Read(data) + _, err = rand.Read(data) + require.NoError(t, err) cid = blocks.NewBlock(data).Cid() id, err = flow.CidToId(cid) diff --git a/model/flow/identity.go b/model/flow/identity.go index f05188988e6..c44c394cb06 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "math" - "math/rand" "regexp" "strconv" @@ -18,6 +17,7 @@ import ( "github.com/vmihailenco/msgpack" "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/utils/rand" ) // DefaultInitialWeight is the default initial weight for a node identity. @@ -461,40 +461,28 @@ func (il IdentityList) ByNetworkingKey(key crypto.PublicKey) (*Identity, bool) { return nil, false } -// Sample returns simple random sample from the `IdentityList` -func (il IdentityList) Sample(size uint) IdentityList { - return il.sample(size, rand.Intn) -} - -// DeterministicSample returns deterministic random sample from the `IdentityList` using the given seed -func (il IdentityList) DeterministicSample(size uint, seed int64) IdentityList { - rng := rand.New(rand.NewSource(seed)) - return il.sample(size, rng.Intn) -} - -func (il IdentityList) sample(size uint, intn func(int) int) IdentityList { +// Sample returns non-deterministic random sample from the `IdentityList` +func (il IdentityList) Sample(size uint) (IdentityList, error) { n := uint(len(il)) - if size > n { + dup := make([]*Identity, 0, n) + dup = append(dup, il...) + if n < size { size = n } - - dup := il.Copy() - for i := uint(0); i < size; i++ { - j := uint(intn(int(n - i))) - dup[i], dup[j+i] = dup[j+i], dup[i] + swap := func(i, j uint) { + dup[i], dup[j] = dup[j], dup[i] } - return dup[:size] + err := rand.Samples(n, size, swap) + if err != nil { + return nil, fmt.Errorf("failed to sample identity list: %w", err) + } + return dup[:size], nil } -// DeterministicShuffle randomly and deterministically shuffles the identity -// list, returning the shuffled list without modifying the receiver. -func (il IdentityList) DeterministicShuffle(seed int64) IdentityList { - dup := il.Copy() - rng := rand.New(rand.NewSource(seed)) - rng.Shuffle(len(il), func(i, j int) { - dup[i], dup[j] = dup[j], dup[i] - }) - return dup +// Shuffle randomly shuffles the identity list (non-deterministic), +// and returns the shuffled list without modifying the receiver. +func (il IdentityList) Shuffle() (IdentityList, error) { + return il.Sample(uint(len(il))) } // SamplePct returns a random sample from the receiver identity list. The @@ -502,9 +490,9 @@ func (il IdentityList) DeterministicShuffle(seed int64) IdentityList { // if `pct>0`, so this will always select at least one identity. // // NOTE: The input must be between 0-1. -func (il IdentityList) SamplePct(pct float64) IdentityList { +func (il IdentityList) SamplePct(pct float64) (IdentityList, error) { if pct <= 0 { - return IdentityList{} + return IdentityList{}, nil } count := float64(il.Count()) * pct diff --git a/model/flow/identity_test.go b/model/flow/identity_test.go index 9c1a137d8ab..891a854aca6 100644 --- a/model/flow/identity_test.go +++ b/model/flow/identity_test.go @@ -2,10 +2,8 @@ package flow_test import ( "encoding/json" - "math/rand" "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -198,28 +196,35 @@ func TestIdentityList_Union(t *testing.T) { func TestSample(t *testing.T) { t.Run("Sample max", func(t *testing.T) { il := unittest.IdentityListFixture(10) - require.Equal(t, uint(10), il.Sample(10).Count()) + sam, err := il.Sample(10) + require.NoError(t, err) + require.Equal(t, uint(10), sam.Count()) }) t.Run("Sample oversized", func(t *testing.T) { il := unittest.IdentityListFixture(10) - require.Equal(t, uint(10), il.Sample(11).Count()) + sam, err := il.Sample(11) + require.NoError(t, err) + require.Equal(t, uint(10), sam.Count()) }) } func TestShuffle(t *testing.T) { t.Run("should be shuffled", func(t *testing.T) { il := unittest.IdentityListFixture(15) // ~1/billion chance of shuffling to input state - shuffled := il.DeterministicShuffle(rand.Int63()) + shuffled, err := il.Shuffle() + require.NoError(t, err) assert.Equal(t, len(il), len(shuffled)) assert.ElementsMatch(t, il, shuffled) }) - t.Run("should be deterministic", func(t *testing.T) { + t.Run("should not be deterministic", func(t *testing.T) { il := unittest.IdentityListFixture(10) - seed := rand.Int63() - shuffled1 := il.DeterministicShuffle(seed) - shuffled2 := il.DeterministicShuffle(seed) - assert.Equal(t, shuffled1, shuffled2) + shuffled1, err := il.Shuffle() + require.NoError(t, err) + shuffled2, err := il.Shuffle() + require.NoError(t, err) + assert.NotEqual(t, shuffled1, shuffled2) + assert.ElementsMatch(t, shuffled1, shuffled2) }) } @@ -238,7 +243,8 @@ func TestIdentity_ID(t *testing.T) { func TestIdentity_Sort(t *testing.T) { il := unittest.IdentityListFixture(20) - random := il.DeterministicShuffle(time.Now().UnixNano()) + random, err := il.Shuffle() + require.NoError(t, err) assert.False(t, random.Sorted(order.Canonical)) canonical := il.Sort(order.Canonical) diff --git a/model/flow/ledger.go b/model/flow/ledger.go index 8e73505f214..b13dbe83e27 100644 --- a/model/flow/ledger.go +++ b/model/flow/ledger.go @@ -13,7 +13,7 @@ import ( const ( // Service level keys (owner is empty): - UUIDKey = "uuid" + UUIDKeyPrefix = "uuid" AddressStateKey = "account_address_state" // Account level keys @@ -38,9 +38,17 @@ var AddressStateRegisterID = RegisterID{ Key: AddressStateKey, } -var UUIDRegisterID = RegisterID{ - Owner: "", - Key: UUIDKey, +func UUIDRegisterID(partition byte) RegisterID { + // NOTE: partition 0 uses "uuid" as key to maintain backwards compatibility. + key := UUIDKeyPrefix + if partition != 0 { + key = fmt.Sprintf("%s_%d", UUIDKeyPrefix, partition) + } + + return RegisterID{ + Owner: "", + Key: key, + } } func AccountStatusRegisterID(address Address) RegisterID { @@ -90,10 +98,12 @@ func NewRegisterID(owner, key string) RegisterID { func (id RegisterID) IsInternalState() bool { // check if is a service level key (owner is empty) // cases: - // - "", "uuid" + // - "", "uuid" (for shard index 0) + // - "", "uuid_%d" (for shard index > 0) // - "", "account_address_state" - if len(id.Owner) == 0 && (id.Key == UUIDKey || id.Key == AddressStateKey) { - return true + if len(id.Owner) == 0 { + return strings.HasPrefix(id.Key, UUIDKeyPrefix) || + id.Key == AddressStateKey } // check account level keys @@ -118,7 +128,6 @@ func (id RegisterID) IsSlabIndex() bool { return len(id.Key) == 9 && id.Key[0] == '$' } -// TODO(patrick): pretty print flow internal register ids. // String returns formatted string representation of the RegisterID. func (id RegisterID) String() string { formattedKey := "" diff --git a/model/flow/ledger_test.go b/model/flow/ledger_test.go index 13d18e2977d..9034f884168 100644 --- a/model/flow/ledger_test.go +++ b/model/flow/ledger_test.go @@ -55,12 +55,24 @@ func TestRegisterID_IsInternalState(t *testing.T) { id := NewRegisterID(owner, key) require.False(t, id.IsInternalState()) } - require.True(t, UUIDRegisterID.IsInternalState()) - requireFalse("", UUIDKey) + + for i := 0; i < 256; i++ { + uuid := UUIDRegisterID(byte(i)) + if i == 0 { + require.Equal(t, uuid.Key, UUIDKeyPrefix) + } else { + require.Equal(t, uuid.Key, fmt.Sprintf("%s_%d", UUIDKeyPrefix, i)) + } + require.True(t, uuid.IsInternalState()) + } + requireFalse("", UUIDKeyPrefix) + for i := 0; i < 256; i++ { + requireFalse("", fmt.Sprintf("%s_%d", UUIDKeyPrefix, i)) + } require.True(t, AddressStateRegisterID.IsInternalState()) requireFalse("", AddressStateKey) requireFalse("", "other") - requireFalse("Address", UUIDKey) + requireFalse("Address", UUIDKeyPrefix) requireFalse("Address", AddressStateKey) requireTrue("Address", "public_key_12") requireTrue("Address", ContractNamesKey) diff --git a/model/flow/sealing_segment.go b/model/flow/sealing_segment.go index e0a04cb9eec..ea96d69fb64 100644 --- a/model/flow/sealing_segment.go +++ b/model/flow/sealing_segment.go @@ -18,6 +18,8 @@ import ( // Lets denote the highest block in the sealing segment as `head`. Per convention, `head` must be a // finalized block. Consider the chain of blocks leading up to `head` (included). The highest block // in chain leading up to `head` that is sealed, we denote as B. +// In other words, head is the last finalized block, and B is the last sealed block, +// block at height (B.Height + 1) is not sealed. type SealingSegment struct { // Blocks contain the chain `B <- ... <- Head` in ascending height order. // Formally, Blocks contains exactly (not more!) the history to satisfy condition @@ -65,6 +67,11 @@ func (segment *SealingSegment) Highest() *Block { return segment.Blocks[len(segment.Blocks)-1] } +// Finalized returns the last finalized block, which is an alias of Highest +func (segment *SealingSegment) Finalized() *Block { + return segment.Highest() +} + // Sealed returns the most recently sealed block based on head of sealing segment(highest block). func (segment *SealingSegment) Sealed() *Block { return segment.Blocks[0] diff --git a/model/flow/service_event.go b/model/flow/service_event.go index d1e098505c8..7467a9e8f2f 100644 --- a/model/flow/service_event.go +++ b/model/flow/service_event.go @@ -10,9 +10,18 @@ import ( cborcodec "github.com/onflow/flow-go/model/encoding/cbor" ) +type ServiceEventType string + +// String returns the string representation of the service event type. +// TODO: this should not be needed. We should use ServiceEventType directly everywhere. +func (set ServiceEventType) String() string { + return string(set) +} + const ( - ServiceEventSetup = "setup" - ServiceEventCommit = "commit" + ServiceEventSetup ServiceEventType = "setup" + ServiceEventCommit ServiceEventType = "commit" + ServiceEventVersionBeacon ServiceEventType = "version-beacon" ) // ServiceEvent represents a service event, which is a special event that when @@ -23,7 +32,7 @@ const ( // This type represents a generic service event and primarily exists to simplify // encoding and decoding. type ServiceEvent struct { - Type string + Type ServiceEventType Event interface{} } @@ -38,7 +47,11 @@ func (sel ServiceEventList) EqualTo(other ServiceEventList) (bool, error) { for i, se := range sel { equalTo, err := se.EqualTo(&other[i]) if err != nil { - return false, fmt.Errorf("error while comparing service event index %d: %w", i, err) + return false, fmt.Errorf( + "error while comparing service event index %d: %w", + i, + err, + ) } if !equalTo { return false, nil @@ -48,152 +61,121 @@ func (sel ServiceEventList) EqualTo(other ServiceEventList) (bool, error) { return true, nil } -func (se *ServiceEvent) UnmarshalJSON(b []byte) error { +type ServiceEventMarshaller interface { + Unmarshal(b []byte) (ServiceEvent, error) + UnmarshalWithType( + b []byte, + eventType ServiceEventType, + ) ( + ServiceEvent, + error, + ) +} - var enc map[string]interface{} - err := json.Unmarshal(b, &enc) - if err != nil { - return err - } +type marshallerImpl struct { + MarshalFunc func(v interface{}) ([]byte, error) + UnmarshalFunc func(data []byte, v interface{}) error +} - tp, ok := enc["Type"].(string) - if !ok { - return fmt.Errorf("missing type key") +var ( + ServiceEventJSONMarshaller = marshallerImpl{ + MarshalFunc: json.Marshal, + UnmarshalFunc: json.Unmarshal, } - ev, ok := enc["Event"] - if !ok { - return fmt.Errorf("missing event key") + ServiceEventMSGPACKMarshaller = marshallerImpl{ + MarshalFunc: msgpack.Marshal, + UnmarshalFunc: msgpack.Unmarshal, } - - // re-marshal the event, we'll unmarshal it into the appropriate type - evb, err := json.Marshal(ev) - if err != nil { - return err + ServiceEventCBORMarshaller = marshallerImpl{ + MarshalFunc: cborcodec.EncMode.Marshal, + UnmarshalFunc: cbor.Unmarshal, } +) - var event interface{} - switch tp { - case ServiceEventSetup: - setup := new(EpochSetup) - err = json.Unmarshal(evb, setup) - if err != nil { - return err - } - event = setup - case ServiceEventCommit: - commit := new(EpochCommit) - err = json.Unmarshal(evb, commit) - if err != nil { - return err - } - event = commit - default: - return fmt.Errorf("invalid type: %s", tp) - } - - *se = ServiceEvent{ - Type: tp, - Event: event, - } - return nil -} - -func (se *ServiceEvent) UnmarshalMsgpack(b []byte) error { - +func (marshaller marshallerImpl) Unmarshal(b []byte) ( + ServiceEvent, + error, +) { var enc map[string]interface{} - err := msgpack.Unmarshal(b, &enc) + err := marshaller.UnmarshalFunc(b, &enc) if err != nil { - return err + return ServiceEvent{}, err } tp, ok := enc["Type"].(string) if !ok { - return fmt.Errorf("missing type key") + return ServiceEvent{}, fmt.Errorf("missing type key") } ev, ok := enc["Event"] if !ok { - return fmt.Errorf("missing event key") + return ServiceEvent{}, fmt.Errorf("missing event key") } // re-marshal the event, we'll unmarshal it into the appropriate type - evb, err := msgpack.Marshal(ev) + evb, err := marshaller.MarshalFunc(ev) if err != nil { - return err + return ServiceEvent{}, err } + return marshaller.UnmarshalWithType(evb, ServiceEventType(tp)) +} + +func (marshaller marshallerImpl) UnmarshalWithType( + b []byte, + eventType ServiceEventType, +) (ServiceEvent, error) { var event interface{} - switch tp { + switch eventType { case ServiceEventSetup: - setup := new(EpochSetup) - err = msgpack.Unmarshal(evb, setup) - if err != nil { - return err - } - event = setup + event = new(EpochSetup) case ServiceEventCommit: - commit := new(EpochCommit) - err = msgpack.Unmarshal(evb, commit) - if err != nil { - return err - } - event = commit + event = new(EpochCommit) + case ServiceEventVersionBeacon: + event = new(VersionBeacon) default: - return fmt.Errorf("invalid type: %s", tp) + return ServiceEvent{}, fmt.Errorf("invalid type: %s", eventType) } - *se = ServiceEvent{ - Type: tp, - Event: event, + err := marshaller.UnmarshalFunc(b, event) + if err != nil { + return ServiceEvent{}, + fmt.Errorf( + "failed to unmarshal to service event ot type %s: %w", + eventType, + err, + ) } - return nil -} -func (se *ServiceEvent) UnmarshalCBOR(b []byte) error { + return ServiceEvent{ + Type: eventType, + Event: event, + }, nil +} - var enc map[string]interface{} - err := cbor.Unmarshal(b, &enc) +func (se *ServiceEvent) UnmarshalJSON(b []byte) error { + e, err := ServiceEventJSONMarshaller.Unmarshal(b) if err != nil { return err } + *se = e + return nil +} - tp, ok := enc["Type"].(string) - if !ok { - return fmt.Errorf("missing type key") - } - ev, ok := enc["Event"] - if !ok { - return fmt.Errorf("missing event key") - } - - evb, err := cborcodec.EncMode.Marshal(ev) +func (se *ServiceEvent) UnmarshalMsgpack(b []byte) error { + e, err := ServiceEventMSGPACKMarshaller.Unmarshal(b) if err != nil { return err } + *se = e + return nil +} - var event interface{} - switch tp { - case ServiceEventSetup: - setup := new(EpochSetup) - err = cbor.Unmarshal(evb, setup) - if err != nil { - return err - } - event = setup - case ServiceEventCommit: - commit := new(EpochCommit) - err = cbor.Unmarshal(evb, commit) - if err != nil { - return err - } - event = commit - default: - return fmt.Errorf("invalid type: %s", tp) - } - - *se = ServiceEvent{ - Type: tp, - Event: event, +func (se *ServiceEvent) UnmarshalCBOR(b []byte) error { + e, err := ServiceEventCBORMarshaller.Unmarshal(b) + if err != nil { + return err } + *se = e return nil } @@ -205,24 +187,55 @@ func (se *ServiceEvent) EqualTo(other *ServiceEvent) (bool, error) { case ServiceEventSetup: setup, ok := se.Event.(*EpochSetup) if !ok { - return false, fmt.Errorf("internal invalid type for ServiceEventSetup: %T", se.Event) + return false, fmt.Errorf( + "internal invalid type for ServiceEventSetup: %T", + se.Event, + ) } otherSetup, ok := other.Event.(*EpochSetup) if !ok { - return false, fmt.Errorf("internal invalid type for ServiceEventSetup: %T", other.Event) + return false, fmt.Errorf( + "internal invalid type for ServiceEventSetup: %T", + other.Event, + ) } return setup.EqualTo(otherSetup), nil case ServiceEventCommit: commit, ok := se.Event.(*EpochCommit) if !ok { - return false, fmt.Errorf("internal invalid type for ServiceEventCommit: %T", se.Event) + return false, fmt.Errorf( + "internal invalid type for ServiceEventCommit: %T", + se.Event, + ) } otherCommit, ok := other.Event.(*EpochCommit) if !ok { - return false, fmt.Errorf("internal invalid type for ServiceEventCommit: %T", other.Event) + return false, fmt.Errorf( + "internal invalid type for ServiceEventCommit: %T", + other.Event, + ) } return commit.EqualTo(otherCommit), nil + + case ServiceEventVersionBeacon: + version, ok := se.Event.(*VersionBeacon) + if !ok { + return false, fmt.Errorf( + "internal invalid type for ServiceEventVersionBeacon: %T", + se.Event, + ) + } + otherVersion, ok := other.Event.(*VersionBeacon) + if !ok { + return false, + fmt.Errorf( + "internal invalid type for ServiceEventVersionBeacon: %T", + other.Event, + ) + } + return version.EqualTo(otherVersion), nil + default: return false, fmt.Errorf("unknown serice event type: %s", se.Type) } diff --git a/model/flow/service_event_test.go b/model/flow/service_event_test.go index 47ec937b0f9..90c571fc4ba 100644 --- a/model/flow/service_event_test.go +++ b/model/flow/service_event_test.go @@ -20,6 +20,7 @@ func TestEncodeDecode(t *testing.T) { setup := unittest.EpochSetupFixture() commit := unittest.EpochCommitFixture() + versionBeacon := unittest.VersionBeaconFixture() comparePubKey := cmp.FilterValues(func(a, b crypto.PublicKey) bool { return true @@ -32,6 +33,7 @@ func TestEncodeDecode(t *testing.T) { t.Run("json", func(t *testing.T) { t.Run("specific event types", func(t *testing.T) { + // EpochSetup b, err := json.Marshal(setup) require.NoError(t, err) @@ -40,6 +42,7 @@ func TestEncodeDecode(t *testing.T) { require.NoError(t, err) assert.DeepEqual(t, setup, gotSetup, comparePubKey) + // EpochCommit b, err = json.Marshal(commit) require.NoError(t, err) @@ -47,9 +50,19 @@ func TestEncodeDecode(t *testing.T) { err = json.Unmarshal(b, gotCommit) require.NoError(t, err) assert.DeepEqual(t, commit, gotCommit, comparePubKey) + + // VersionBeacon + b, err = json.Marshal(versionBeacon) + require.NoError(t, err) + + gotVersionBeacon := new(flow.VersionBeacon) + err = json.Unmarshal(b, gotVersionBeacon) + require.NoError(t, err) + assert.DeepEqual(t, versionBeacon, gotVersionBeacon) }) t.Run("generic type", func(t *testing.T) { + // EpochSetup b, err := json.Marshal(setup.ServiceEvent()) require.NoError(t, err) @@ -60,6 +73,7 @@ func TestEncodeDecode(t *testing.T) { require.True(t, ok) assert.DeepEqual(t, setup, gotSetup, comparePubKey) + // EpochCommit t.Logf("- debug: setup.ServiceEvent()=%+v\n", setup.ServiceEvent()) b, err = json.Marshal(commit.ServiceEvent()) require.NoError(t, err) @@ -72,11 +86,26 @@ func TestEncodeDecode(t *testing.T) { gotCommit, ok := outer.Event.(*flow.EpochCommit) require.True(t, ok) assert.DeepEqual(t, commit, gotCommit, comparePubKey) + + // VersionBeacon + t.Logf("- debug: versionBeacon.ServiceEvent()=%+v\n", versionBeacon.ServiceEvent()) + b, err = json.Marshal(versionBeacon.ServiceEvent()) + require.NoError(t, err) + + outer = new(flow.ServiceEvent) + t.Logf("- debug: outer=%+v <-- before .Unmarshal()\n", outer) + err = json.Unmarshal(b, outer) + t.Logf("- debug: outer=%+v <-- after .Unmarshal()\n", outer) + require.NoError(t, err) + gotVersionTable, ok := outer.Event.(*flow.VersionBeacon) + require.True(t, ok) + assert.DeepEqual(t, versionBeacon, gotVersionTable) }) }) t.Run("msgpack", func(t *testing.T) { t.Run("specific event types", func(t *testing.T) { + // EpochSetup b, err := msgpack.Marshal(setup) require.NoError(t, err) @@ -85,6 +114,7 @@ func TestEncodeDecode(t *testing.T) { require.NoError(t, err) assert.DeepEqual(t, setup, gotSetup, comparePubKey) + // EpochCommit b, err = msgpack.Marshal(commit) require.NoError(t, err) @@ -92,6 +122,15 @@ func TestEncodeDecode(t *testing.T) { err = msgpack.Unmarshal(b, gotCommit) require.NoError(t, err) assert.DeepEqual(t, commit, gotCommit, comparePubKey) + + // VersionBeacon + b, err = msgpack.Marshal(versionBeacon) + require.NoError(t, err) + + gotVersionTable := new(flow.VersionBeacon) + err = msgpack.Unmarshal(b, gotVersionTable) + require.NoError(t, err) + assert.DeepEqual(t, versionBeacon, gotVersionTable) }) t.Run("generic type", func(t *testing.T) { @@ -105,6 +144,7 @@ func TestEncodeDecode(t *testing.T) { require.True(t, ok) assert.DeepEqual(t, setup, gotSetup, comparePubKey) + // EpochCommit t.Logf("- debug: setup.ServiceEvent()=%+v\n", setup.ServiceEvent()) b, err = msgpack.Marshal(commit.ServiceEvent()) require.NoError(t, err) @@ -117,11 +157,26 @@ func TestEncodeDecode(t *testing.T) { gotCommit, ok := outer.Event.(*flow.EpochCommit) require.True(t, ok) assert.DeepEqual(t, commit, gotCommit, comparePubKey) + + // VersionBeacon + t.Logf("- debug: versionTable.ServiceEvent()=%+v\n", versionBeacon.ServiceEvent()) + b, err = msgpack.Marshal(versionBeacon.ServiceEvent()) + require.NoError(t, err) + + outer = new(flow.ServiceEvent) + t.Logf("- debug: outer=%+v <-- before .Unmarshal()\n", outer) + err = msgpack.Unmarshal(b, outer) + t.Logf("- debug: outer=%+v <-- after .Unmarshal()\n", outer) + require.NoError(t, err) + gotVersionTable, ok := outer.Event.(*flow.VersionBeacon) + require.True(t, ok) + assert.DeepEqual(t, versionBeacon, gotVersionTable, comparePubKey) }) }) t.Run("cbor", func(t *testing.T) { t.Run("specific event types", func(t *testing.T) { + // EpochSetup b, err := cborcodec.EncMode.Marshal(setup) require.NoError(t, err) @@ -130,6 +185,7 @@ func TestEncodeDecode(t *testing.T) { require.NoError(t, err) assert.DeepEqual(t, setup, gotSetup, comparePubKey) + // EpochCommit b, err = cborcodec.EncMode.Marshal(commit) require.NoError(t, err) @@ -137,9 +193,20 @@ func TestEncodeDecode(t *testing.T) { err = cbor.Unmarshal(b, gotCommit) require.NoError(t, err) assert.DeepEqual(t, commit, gotCommit, comparePubKey) + + // VersionBeacon + b, err = cborcodec.EncMode.Marshal(versionBeacon) + require.NoError(t, err) + + gotVersionTable := new(flow.VersionBeacon) + err = cbor.Unmarshal(b, gotVersionTable) + require.NoError(t, err) + assert.DeepEqual(t, versionBeacon, gotVersionTable) + }) t.Run("generic type", func(t *testing.T) { + // EpochSetup t.Logf("- debug: setup.ServiceEvent()=%+v\n", setup.ServiceEvent()) b, err := cborcodec.EncMode.Marshal(setup.ServiceEvent()) require.NoError(t, err) @@ -153,6 +220,7 @@ func TestEncodeDecode(t *testing.T) { require.True(t, ok) assert.DeepEqual(t, setup, gotSetup, comparePubKey) + // EpochCommit b, err = cborcodec.EncMode.Marshal(commit.ServiceEvent()) require.NoError(t, err) @@ -162,6 +230,18 @@ func TestEncodeDecode(t *testing.T) { gotCommit, ok := outer.Event.(*flow.EpochCommit) require.True(t, ok) assert.DeepEqual(t, commit, gotCommit, comparePubKey) + + // VersionBeacon + t.Logf("- debug: setup.ServiceEvent()=%+v\n", versionBeacon.ServiceEvent()) + b, err = cborcodec.EncMode.Marshal(versionBeacon.ServiceEvent()) + require.NoError(t, err) + + outer = new(flow.ServiceEvent) + err = cbor.Unmarshal(b, outer) + require.NoError(t, err) + gotVersionTable, ok := outer.Event.(*flow.VersionBeacon) + require.True(t, ok) + assert.DeepEqual(t, versionBeacon, gotVersionTable) }) }) } diff --git a/model/flow/version_beacon.go b/model/flow/version_beacon.go new file mode 100644 index 00000000000..9638446ed16 --- /dev/null +++ b/model/flow/version_beacon.go @@ -0,0 +1,148 @@ +package flow + +import ( + "fmt" + + "github.com/coreos/go-semver/semver" +) + +// VersionBoundary represents a boundary between semver versions. +// BlockHeight is the first block height that must be run by the given Version (inclusive). +// Version is a semver string. +type VersionBoundary struct { + BlockHeight uint64 + Version string +} + +func (v VersionBoundary) Semver() (*semver.Version, error) { + return semver.NewVersion(v.Version) +} + +// VersionBeacon represents a service event specifying the required software versions +// for upcoming blocks. +// +// It contains a VersionBoundaries field, which is an ordered list of VersionBoundary +// (sorted by VersionBoundary.BlockHeight). While heights are strictly +// increasing, versions must be equal or greater when compared using semver semantics. +// It must contain at least one entry. The first entry is for a past block height. +// The remaining entries are for all future block heights. Future version boundaries +// can be removed, in which case the emitted event will not contain the removed version +// boundaries. +// VersionBeacon is produced by the NodeVersionBeacon smart contract. +// +// Sequence is the event sequence number, which can be used to verify that no event has been +// skipped by the follower. Every time the smart contract emits a new event, it increments +// the sequence number by one. +type VersionBeacon struct { + VersionBoundaries []VersionBoundary + Sequence uint64 +} + +// SealedVersionBeacon is a VersionBeacon with a SealHeight field. +// Version beacons are effective only after the results containing the version beacon +// are sealed. +type SealedVersionBeacon struct { + *VersionBeacon + SealHeight uint64 +} + +func (v *VersionBeacon) ServiceEvent() ServiceEvent { + return ServiceEvent{ + Type: ServiceEventVersionBeacon, + Event: v, + } +} + +// EqualTo returns true if two VersionBeacons are equal. +// If any of the VersionBeacons has a malformed version, it will return false. +func (v *VersionBeacon) EqualTo(other *VersionBeacon) bool { + + if v.Sequence != other.Sequence { + return false + } + + if len(v.VersionBoundaries) != len(other.VersionBoundaries) { + return false + } + + for i, v := range v.VersionBoundaries { + other := other.VersionBoundaries[i] + + if v.BlockHeight != other.BlockHeight { + return false + } + + v1, err := v.Semver() + if err != nil { + return false + } + v2, err := other.Semver() + if err != nil { + return false + } + if !v1.Equal(*v2) { + return false + } + } + + return true +} + +// Validate validates the internal structure of a flow.VersionBeacon. +// An error with an appropriate message is returned +// if any validation fails. +func (v *VersionBeacon) Validate() error { + eventError := func(format string, args ...interface{}) error { + args = append([]interface{}{v.Sequence}, args...) + return fmt.Errorf( + "version beacon (sequence=%d) error: "+format, + args..., + ) + } + + if len(v.VersionBoundaries) == 0 { + return eventError("required version boundaries empty") + } + + var previousHeight uint64 + var previousVersion *semver.Version + for i, boundary := range v.VersionBoundaries { + version, err := boundary.Semver() + if err != nil { + return eventError( + "invalid semver %s for version boundary (height=%d) (index=%d): %w", + boundary.Version, + boundary.BlockHeight, + i, + err, + ) + } + + if i != 0 && previousHeight >= boundary.BlockHeight { + return eventError( + "higher requirement (index=%d) height %d "+ + "at or below previous height (index=%d) %d", + i, + boundary.BlockHeight, + i-1, + previousHeight, + ) + } + + if i != 0 && version.LessThan(*previousVersion) { + return eventError( + "higher requirement (index=%d) semver %s "+ + "lower than previous (index=%d) %s", + i, + version, + i-1, + previousVersion, + ) + } + + previousVersion = version + previousHeight = boundary.BlockHeight + } + + return nil +} diff --git a/model/flow/version_beacon_test.go b/model/flow/version_beacon_test.go new file mode 100644 index 00000000000..83f4542e827 --- /dev/null +++ b/model/flow/version_beacon_test.go @@ -0,0 +1,215 @@ +package flow_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" +) + +func TestEqualTo(t *testing.T) { + testCases := []struct { + name string + vb1 flow.VersionBeacon + vb2 flow.VersionBeacon + result bool + }{ + { + name: "Equal version beacons", + vb1: flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "1.0.0"}, + {BlockHeight: 2, Version: "1.1.0"}, + }, + Sequence: 1, + }, + vb2: flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "1.0.0"}, + {BlockHeight: 2, Version: "1.1.0"}, + }, + Sequence: 1, + }, + result: true, + }, + { + name: "Different sequence", + vb1: flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "1.0.0"}, + {BlockHeight: 2, Version: "1.1.0"}, + }, + Sequence: 1, + }, + vb2: flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "1.0.0"}, + {BlockHeight: 2, Version: "1.1.0"}, + }, + Sequence: 2, + }, + result: false, + }, + { + name: "Equal sequence, but invalid version", + vb1: flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "v1.0.0"}, + }, + Sequence: 1, + }, + vb2: flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "v1.0.0"}, + }, + Sequence: 1, + }, + result: false, + }, + { + name: "Different version boundaries", + vb1: flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "1.0.0"}, + {BlockHeight: 2, Version: "1.1.0"}, + }, + Sequence: 1, + }, + vb2: flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "1.0.0"}, + {BlockHeight: 2, Version: "1.2.0"}, + }, + Sequence: 1, + }, + result: false, + }, + { + name: "Different length of version boundaries", + vb1: flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "1.0.0"}, + {BlockHeight: 2, Version: "1.1.0"}, + }, + Sequence: 1, + }, + vb2: flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "1.0.0"}, + }, + Sequence: 1, + }, + result: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.result, tc.vb1.EqualTo(&tc.vb2)) + }) + } +} + +func TestValidate(t *testing.T) { + testCases := []struct { + name string + vb *flow.VersionBeacon + expected bool + }{ + { + name: "empty requirements table is invalid", + vb: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{}, + Sequence: 1, + }, + expected: false, + }, + { + name: "single version required requirement must be valid semver", + vb: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "v0.21.37"}, + }, + Sequence: 1, + }, + expected: false, + }, + { + name: "ordered by height ascending is valid", + vb: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "0.21.37"}, + {BlockHeight: 100, Version: "0.21.37"}, + {BlockHeight: 200, Version: "0.21.37"}, + {BlockHeight: 300, Version: "0.21.37"}, + }, + Sequence: 1, + }, + expected: true, + }, + { + name: "decreasing height is invalid", + vb: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "0.21.37"}, + {BlockHeight: 200, Version: "0.21.37"}, + {BlockHeight: 180, Version: "0.21.37"}, + {BlockHeight: 300, Version: "0.21.37"}, + }, + Sequence: 1, + }, + expected: false, + }, + { + name: "version higher or equal to the previous one is valid", + vb: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "0.21.37"}, + {BlockHeight: 200, Version: "0.21.37"}, + {BlockHeight: 300, Version: "0.21.38"}, + {BlockHeight: 400, Version: "1.0.0"}, + }, + Sequence: 1, + }, + expected: true, + }, + { + name: "any version lower than previous one is invalid", + vb: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "0.21.37"}, + {BlockHeight: 200, Version: "1.2.3"}, + {BlockHeight: 300, Version: "1.2.4"}, + {BlockHeight: 400, Version: "1.2.3"}, + }, + Sequence: 1, + }, + expected: false, + }, + { + name: "all version must be valid semver string to be valid", + vb: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + {BlockHeight: 1, Version: "0.21.37"}, + {BlockHeight: 200, Version: "0.21.37"}, + {BlockHeight: 300, Version: "0.21.38"}, + {BlockHeight: 400, Version: "v0.21.39"}, + }, + Sequence: 1, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.vb.Validate() + if tc.expected { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} diff --git a/model/verification/chunkDataPackRequest.go b/model/verification/chunkDataPackRequest.go index 0c0cd4cd92a..9f2bf42c52c 100644 --- a/model/verification/chunkDataPackRequest.go +++ b/model/verification/chunkDataPackRequest.go @@ -1,6 +1,8 @@ package verification import ( + "fmt" + "github.com/onflow/flow-go/model/chunks" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" @@ -23,10 +25,14 @@ type ChunkDataPackRequestInfo struct { // SampleTargets returns identifier of execution nodes that can be asked for the chunk data pack, based on // the agreeing and disagreeing execution nodes of the chunk data pack request. -func (c ChunkDataPackRequestInfo) SampleTargets(count int) flow.IdentifierList { +func (c ChunkDataPackRequestInfo) SampleTargets(count int) (flow.IdentifierList, error) { // if there are enough receipts produced the same result (agrees), we sample from them. if len(c.Agrees) >= count { - return c.Targets.Filter(filter.HasNodeID(c.Agrees...)).Sample(uint(count)).NodeIDs() + sample, err := c.Targets.Filter(filter.HasNodeID(c.Agrees...)).Sample(uint(count)) + if err != nil { + return nil, fmt.Errorf("sampling target failed: %w", err) + } + return sample.NodeIDs(), nil } // since there is at least one agree, then usually, we just need `count - 1` extra nodes as backup. @@ -35,8 +41,11 @@ func (c ChunkDataPackRequestInfo) SampleTargets(count int) flow.IdentifierList { // fetch from the one produced the same result (the only agree) need := uint(count - len(c.Agrees)) - nonResponders := c.Targets.Filter(filter.Not(filter.HasNodeID(c.Disagrees...))).Sample(need).NodeIDs() - return append(c.Agrees, nonResponders...) + nonResponders, err := c.Targets.Filter(filter.Not(filter.HasNodeID(c.Disagrees...))).Sample(need) + if err != nil { + return nil, fmt.Errorf("sampling target failed: %w", err) + } + return append(c.Agrees, nonResponders.NodeIDs()...), nil } type ChunkDataPackRequestInfoList []*ChunkDataPackRequestInfo diff --git a/model/verification/verifiableChunkData.go b/model/verification/verifiableChunkData.go index 298beece37f..2f6f1e22579 100644 --- a/model/verification/verifiableChunkData.go +++ b/model/verification/verifiableChunkData.go @@ -2,6 +2,7 @@ package verification import ( "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" ) // VerifiableChunkData represents a ready-to-verify chunk @@ -10,6 +11,7 @@ type VerifiableChunkData struct { IsSystemChunk bool // indicates whether this is a system chunk Chunk *flow.Chunk // the chunk to be verified Header *flow.Header // BlockHeader that contains this chunk + Snapshot protocol.Snapshot // state snapshot at the chunk's block Result *flow.ExecutionResult // execution result of this block ChunkDataPack *flow.ChunkDataPack // chunk data package needed to verify this chunk EndState flow.StateCommitment // state commitment at the end of this chunk diff --git a/module/buffer.go b/module/buffer.go index 30bd3df6f5f..06ee349908d 100644 --- a/module/buffer.go +++ b/module/buffer.go @@ -11,7 +11,7 @@ import ( // children once the parent is received. // Safe for concurrent use. type PendingBlockBuffer interface { - Add(originID flow.Identifier, block *flow.Block) bool + Add(block flow.Slashable[*flow.Block]) bool ByID(blockID flow.Identifier) (flow.Slashable[*flow.Block], bool) @@ -29,7 +29,7 @@ type PendingBlockBuffer interface { // collection node cluster consensus. // Safe for concurrent use. type PendingClusterBlockBuffer interface { - Add(originID flow.Identifier, block *cluster.Block) bool + Add(block flow.Slashable[*cluster.Block]) bool ByID(blockID flow.Identifier) (flow.Slashable[*cluster.Block], bool) diff --git a/module/buffer/backend.go b/module/buffer/backend.go index 328cdc3b534..c57dd78adb5 100644 --- a/module/buffer/backend.go +++ b/module/buffer/backend.go @@ -9,21 +9,20 @@ import ( // item represents an item in the cache: a block header, payload, and the ID // of the node that sent it to us. The payload is generic. type item struct { - originID flow.Identifier - header *flow.Header - payload interface{} + header flow.Slashable[*flow.Header] + payload interface{} } // backend implements a simple cache of pending blocks, indexed by parent ID. type backend struct { mu sync.RWMutex - // map of pending block IDs, keyed by parent ID for ByParentID lookups + // map of pending header IDs, keyed by parent ID for ByParentID lookups blocksByParent map[flow.Identifier][]flow.Identifier // set of pending blocks, keyed by ID to avoid duplication blocksByID map[flow.Identifier]*item } -// newBackend returns a new pending block cache. +// newBackend returns a new pending header cache. func newBackend() *backend { cache := &backend{ blocksByParent: make(map[flow.Identifier][]flow.Identifier), @@ -34,12 +33,12 @@ func newBackend() *backend { // add adds the item to the cache, returning false if it already exists and // true otherwise. -func (b *backend) add(originID flow.Identifier, header *flow.Header, payload interface{}) bool { +func (b *backend) add(block flow.Slashable[*flow.Header], payload interface{}) bool { b.mu.Lock() defer b.mu.Unlock() - blockID := header.ID() + blockID := block.Message.ID() _, exists := b.blocksByID[blockID] if exists { @@ -47,13 +46,12 @@ func (b *backend) add(originID flow.Identifier, header *flow.Header, payload int } item := &item{ - header: header, - originID: originID, - payload: payload, + header: block, + payload: payload, } b.blocksByID[blockID] = item - b.blocksByParent[header.ParentID] = append(b.blocksByParent[header.ParentID], blockID) + b.blocksByParent[block.Message.ParentID] = append(b.blocksByParent[block.Message.ParentID], blockID) return true } @@ -116,9 +114,9 @@ func (b *backend) pruneByView(view uint64) { defer b.mu.Unlock() for id, item := range b.blocksByID { - if item.header.View <= view { + if item.header.Message.View <= view { delete(b.blocksByID, id) - delete(b.blocksByParent, item.header.ParentID) + delete(b.blocksByParent, item.header.Message.ParentID) } } } diff --git a/module/buffer/backend_test.go b/module/buffer/backend_test.go index fa0fd3165b4..8812ff1876d 100644 --- a/module/buffer/backend_test.go +++ b/module/buffer/backend_test.go @@ -23,33 +23,35 @@ func (suite *BackendSuite) SetupTest() { suite.backend = newBackend() } -func (suite *BackendSuite) Item() *item { +func (suite *BackendSuite) item() *item { parent := unittest.BlockHeaderFixture() - return suite.ItemWithParent(parent) + return suite.itemWithParent(parent) } -func (suite *BackendSuite) ItemWithParent(parent *flow.Header) *item { +func (suite *BackendSuite) itemWithParent(parent *flow.Header) *item { header := unittest.BlockHeaderWithParentFixture(parent) return &item{ - header: header, - payload: unittest.IdentifierFixture(), - originID: unittest.IdentifierFixture(), + header: flow.Slashable[*flow.Header]{ + OriginID: unittest.IdentifierFixture(), + Message: header, + }, + payload: unittest.IdentifierFixture(), } } func (suite *BackendSuite) Add(item *item) { - suite.backend.add(item.originID, item.header, item.payload) + suite.backend.add(item.header, item.payload) } func (suite *BackendSuite) TestAdd() { - expected := suite.Item() - suite.backend.add(expected.originID, expected.header, expected.payload) + expected := suite.item() + suite.backend.add(expected.header, expected.payload) - actual, ok := suite.backend.byID(expected.header.ID()) + actual, ok := suite.backend.byID(expected.header.Message.ID()) suite.Assert().True(ok) suite.Assert().Equal(expected, actual) - byParent, ok := suite.backend.byParentID(expected.header.ParentID) + byParent, ok := suite.backend.byParentID(expected.header.Message.ParentID) suite.Assert().True(ok) suite.Assert().Len(byParent, 1) suite.Assert().Equal(expected, byParent[0]) @@ -57,11 +59,11 @@ func (suite *BackendSuite) TestAdd() { func (suite *BackendSuite) TestChildIndexing() { - parent := suite.Item() - child1 := suite.ItemWithParent(parent.header) - child2 := suite.ItemWithParent(parent.header) - grandchild := suite.ItemWithParent(child1.header) - unrelated := suite.Item() + parent := suite.item() + child1 := suite.itemWithParent(parent.header.Message) + child2 := suite.itemWithParent(parent.header.Message) + grandchild := suite.itemWithParent(child1.header.Message) + unrelated := suite.item() suite.Add(child1) suite.Add(child2) @@ -69,7 +71,7 @@ func (suite *BackendSuite) TestChildIndexing() { suite.Add(unrelated) suite.Run("retrieve by parent ID", func() { - byParent, ok := suite.backend.byParentID(parent.header.ID()) + byParent, ok := suite.backend.byParentID(parent.header.Message.ID()) suite.Assert().True(ok) // should only include direct children suite.Assert().Len(byParent, 2) @@ -78,22 +80,22 @@ func (suite *BackendSuite) TestChildIndexing() { }) suite.Run("drop for parent ID", func() { - suite.backend.dropForParent(parent.header.ID()) + suite.backend.dropForParent(parent.header.Message.ID()) // should only drop direct children - _, exists := suite.backend.byID(child1.header.ID()) + _, exists := suite.backend.byID(child1.header.Message.ID()) suite.Assert().False(exists) - _, exists = suite.backend.byID(child2.header.ID()) + _, exists = suite.backend.byID(child2.header.Message.ID()) suite.Assert().False(exists) // grandchildren should be unaffected - _, exists = suite.backend.byParentID(child1.header.ID()) + _, exists = suite.backend.byParentID(child1.header.Message.ID()) suite.Assert().True(exists) - _, exists = suite.backend.byID(grandchild.header.ID()) + _, exists = suite.backend.byID(grandchild.header.Message.ID()) suite.Assert().True(exists) // nothing else should be affected - _, exists = suite.backend.byID(unrelated.header.ID()) + _, exists = suite.backend.byID(unrelated.header.Message.ID()) suite.Assert().True(exists) }) } @@ -106,31 +108,31 @@ func (suite *BackendSuite) TestPruneByView() { // build a pending buffer for i := 0; i < N; i++ { - // 10% of the time, add a new unrelated pending block + // 10% of the time, add a new unrelated pending header if i%10 == 0 { - item := suite.Item() + item := suite.item() suite.Add(item) items = append(items, item) continue } - // 90% of the time, build on an existing block + // 90% of the time, build on an existing header if i%2 == 1 { parent := items[rand.Intn(len(items))] - item := suite.ItemWithParent(parent.header) + item := suite.itemWithParent(parent.header.Message) suite.Add(item) items = append(items, item) } } // pick a height to prune that's guaranteed to prune at least one item - pruneAt := items[rand.Intn(len(items))].header.View + pruneAt := items[rand.Intn(len(items))].header.Message.View suite.backend.pruneByView(pruneAt) for _, item := range items { - view := item.header.View - id := item.header.ID() - parentID := item.header.ParentID + view := item.header.Message.View + id := item.header.Message.ID() + parentID := item.header.Message.ParentID // check that items below the prune view were removed if view <= pruneAt { @@ -141,7 +143,7 @@ func (suite *BackendSuite) TestPruneByView() { } // check that other items were not removed - if view > item.header.View { + if view > item.header.Message.View { _, exists := suite.backend.byID(id) suite.Assert().True(exists) _, exists = suite.backend.byParentID(parentID) diff --git a/module/buffer/pending_blocks.go b/module/buffer/pending_blocks.go index ca4e54b4924..e842ee5f361 100644 --- a/module/buffer/pending_blocks.go +++ b/module/buffer/pending_blocks.go @@ -16,8 +16,11 @@ func NewPendingBlocks() *PendingBlocks { return b } -func (b *PendingBlocks) Add(originID flow.Identifier, block *flow.Block) bool { - return b.backend.add(originID, block.Header, block.Payload) +func (b *PendingBlocks) Add(block flow.Slashable[*flow.Block]) bool { + return b.backend.add(flow.Slashable[*flow.Header]{ + OriginID: block.OriginID, + Message: block.Message.Header, + }, block.Message.Payload) } func (b *PendingBlocks) ByID(blockID flow.Identifier) (flow.Slashable[*flow.Block], bool) { @@ -27,9 +30,9 @@ func (b *PendingBlocks) ByID(blockID flow.Identifier) (flow.Slashable[*flow.Bloc } block := flow.Slashable[*flow.Block]{ - OriginID: item.originID, + OriginID: item.header.OriginID, Message: &flow.Block{ - Header: item.header, + Header: item.header.Message, Payload: item.payload.(*flow.Payload), }, } @@ -46,9 +49,9 @@ func (b *PendingBlocks) ByParentID(parentID flow.Identifier) ([]flow.Slashable[* blocks := make([]flow.Slashable[*flow.Block], 0, len(items)) for _, item := range items { block := flow.Slashable[*flow.Block]{ - OriginID: item.originID, + OriginID: item.header.OriginID, Message: &flow.Block{ - Header: item.header, + Header: item.header.Message, Payload: item.payload.(*flow.Payload), }, } diff --git a/module/buffer/pending_cluster_blocks.go b/module/buffer/pending_cluster_blocks.go index df4a3324770..f806271defd 100644 --- a/module/buffer/pending_cluster_blocks.go +++ b/module/buffer/pending_cluster_blocks.go @@ -14,8 +14,11 @@ func NewPendingClusterBlocks() *PendingClusterBlocks { return b } -func (b *PendingClusterBlocks) Add(originID flow.Identifier, block *cluster.Block) bool { - return b.backend.add(originID, block.Header, block.Payload) +func (b *PendingClusterBlocks) Add(block flow.Slashable[*cluster.Block]) bool { + return b.backend.add(flow.Slashable[*flow.Header]{ + OriginID: flow.Identifier{}, + Message: block.Message.Header, + }, block.Message.Payload) } func (b *PendingClusterBlocks) ByID(blockID flow.Identifier) (flow.Slashable[*cluster.Block], bool) { @@ -25,9 +28,9 @@ func (b *PendingClusterBlocks) ByID(blockID flow.Identifier) (flow.Slashable[*cl } block := flow.Slashable[*cluster.Block]{ - OriginID: item.originID, + OriginID: item.header.OriginID, Message: &cluster.Block{ - Header: item.header, + Header: item.header.Message, Payload: item.payload.(*cluster.Payload), }, } @@ -44,9 +47,9 @@ func (b *PendingClusterBlocks) ByParentID(parentID flow.Identifier) ([]flow.Slas blocks := make([]flow.Slashable[*cluster.Block], 0, len(items)) for _, item := range items { block := flow.Slashable[*cluster.Block]{ - OriginID: item.originID, + OriginID: item.header.OriginID, Message: &cluster.Block{ - Header: item.header, + Header: item.header.Message, Payload: item.payload.(*cluster.Payload), }, } diff --git a/module/builder/collection/build_ctx.go b/module/builder/collection/build_ctx.go new file mode 100644 index 00000000000..794f09af2a5 --- /dev/null +++ b/module/builder/collection/build_ctx.go @@ -0,0 +1,56 @@ +package collection + +import ( + "github.com/onflow/flow-go/model/flow" +) + +// blockBuildContext encapsulates required information about the cluster chain and +// main chain state needed to build a new cluster block proposal. +type blockBuildContext struct { + parentID flow.Identifier // ID of the parent we are extending + parent *flow.Header // parent of the block we are building + clusterChainFinalizedBlock *flow.Header // finalized block on the cluster chain + refChainFinalizedHeight uint64 // finalized height on reference chain + refChainFinalizedID flow.Identifier // finalized block ID on reference chain + refEpochFirstHeight uint64 // first height of this cluster's operating epoch + refEpochFinalHeight *uint64 // last height of this cluster's operating epoch (nil if epoch not ended) + refEpochFinalID *flow.Identifier // ID of last block in this cluster's operating epoch (nil if epoch not ended) + config Config + limiter *rateLimiter + lookup *transactionLookup +} + +// highestPossibleReferenceBlockHeight returns the height of the highest possible valid reference block. +// It is the highest finalized block which is in this cluster's operating epoch. +func (ctx *blockBuildContext) highestPossibleReferenceBlockHeight() uint64 { + if ctx.refEpochFinalHeight != nil { + return *ctx.refEpochFinalHeight + } + return ctx.refChainFinalizedHeight +} + +// highestPossibleReferenceBlockID returns the ID of the highest possible valid reference block. +// It is the highest finalized block which is in this cluster's operating epoch. +func (ctx *blockBuildContext) highestPossibleReferenceBlockID() flow.Identifier { + if ctx.refEpochFinalID != nil { + return *ctx.refEpochFinalID + } + return ctx.refChainFinalizedID +} + +// lowestPossibleReferenceBlockHeight returns the height of the lowest possible valid reference block. +// This is the higher of: +// - the first block in this cluster's operating epoch +// - the lowest block which could be used as a reference block without being +// immediately expired (accounting for the configured expiry buffer) +func (ctx *blockBuildContext) lowestPossibleReferenceBlockHeight() uint64 { + // By default, the lowest possible reference block for a non-expired collection has a height + // δ below the latest finalized block, for `δ := flow.DefaultTransactionExpiry - ctx.config.ExpiryBuffer` + // However, our current Epoch might not have δ finalized blocks yet, in which case the lowest + // possible reference block is the first block in the Epoch. + delta := uint64(flow.DefaultTransactionExpiry - ctx.config.ExpiryBuffer) + if ctx.refChainFinalizedHeight <= ctx.refEpochFirstHeight+delta { + return ctx.refEpochFirstHeight + } + return ctx.refChainFinalizedHeight - delta +} diff --git a/module/builder/collection/builder.go b/module/builder/collection/builder.go index 41865bfd5a1..91f7fe93e37 100644 --- a/module/builder/collection/builder.go +++ b/module/builder/collection/builder.go @@ -4,19 +4,20 @@ import ( "context" "errors" "fmt" - "math" "time" "github.com/dgraph-io/badger/v2" "github.com/rs/zerolog" - otelTrace "go.opentelemetry.io/otel/trace" "github.com/onflow/flow-go/model/cluster" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/trace" + clusterstate "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/state/fork" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/operation" "github.com/onflow/flow-go/storage/badger/procedure" @@ -33,27 +34,50 @@ type Builder struct { db *badger.DB mainHeaders storage.Headers clusterHeaders storage.Headers + protoState protocol.State + clusterState clusterstate.State payloads storage.ClusterPayloads transactions mempool.Transactions tracer module.Tracer config Config log zerolog.Logger + clusterEpoch uint64 // the operating epoch for this cluster + // cache of values about the operating epoch which never change + refEpochFirstHeight uint64 // first height of this cluster's operating epoch + epochFinalHeight *uint64 // last height of this cluster's operating epoch (nil if epoch not ended) + epochFinalID *flow.Identifier // ID of last block in this cluster's operating epoch (nil if epoch not ended) } -// TODO: #6435 -// - pass in epoch (minimally counter, preferably cluster chain ID as well) -// - check candidate reference blocks by view (need to get whole header each time - cheap if header in cache) -// - if outside view boundary, look up first+final block height of epoch (can cache both) -func NewBuilder(db *badger.DB, tracer module.Tracer, mainHeaders storage.Headers, clusterHeaders storage.Headers, payloads storage.ClusterPayloads, transactions mempool.Transactions, log zerolog.Logger, opts ...Opt) (*Builder, error) { +func NewBuilder( + db *badger.DB, + tracer module.Tracer, + protoState protocol.State, + clusterState clusterstate.State, + mainHeaders storage.Headers, + clusterHeaders storage.Headers, + payloads storage.ClusterPayloads, + transactions mempool.Transactions, + log zerolog.Logger, + epochCounter uint64, + opts ...Opt, +) (*Builder, error) { b := Builder{ db: db, tracer: tracer, + protoState: protoState, + clusterState: clusterState, mainHeaders: mainHeaders, clusterHeaders: clusterHeaders, payloads: payloads, transactions: transactions, config: DefaultConfig(), log: log.With().Str("component", "cluster_builder").Logger(), + clusterEpoch: epochCounter, + } + + err := db.View(operation.RetrieveEpochFirstHeight(epochCounter, &b.refEpochFirstHeight)) + if err != nil { + return nil, fmt.Errorf("could not get epoch first height: %w", err) } for _, apply := range opts { @@ -71,15 +95,10 @@ func NewBuilder(db *badger.DB, tracer module.Tracer, mainHeaders storage.Headers // BuildOn creates a new block built on the given parent. It produces a payload // that is valid with respect to the un-finalized chain it extends. func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) error) (*flow.Header, error) { - var proposal cluster.Block // proposal we are building - var parent flow.Header // parent of the proposal we are building - var clusterChainFinalizedBlock flow.Header // finalized block on the cluster chain - var refChainFinalizedHeight uint64 // finalized height on reference chain - var refChainFinalizedID flow.Identifier // finalized block ID on reference chain - - startTime := time.Now() + parentSpan, ctx := b.tracer.StartSpanFromContext(context.Background(), trace.COLBuildOn) + defer parentSpan.End() - // STEP ONE: build a lookup for excluding duplicated transactions. + // STEP 1: build a lookup for excluding duplicated transactions. // This is briefly how it works: // // Let E be the global transaction expiry. @@ -97,7 +116,8 @@ func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) er // // A collection with overlapping expiry window can be finalized or un-finalized. // * to find all non-expired and finalized collections, we make use of an index - // (main_chain_finalized_height -> cluster_block_ids with respective reference height), to search for a range of main chain heights // which could be only referenced by collections with overlapping expiry windows. + // (main_chain_finalized_height -> cluster_block_ids with respective reference height), to search for a range of main chain heights + // which could be only referenced by collections with overlapping expiry windows. // * to find all overlapping and un-finalized collections, we can't use the above index, because it's // only for finalized collections. Instead, we simply traverse along the chain up to the last // finalized block. This could possibly include some collections with expiry windows that DON'T @@ -105,102 +125,245 @@ func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) er // // After combining both the finalized and un-finalized cluster blocks that overlap with our expiry window, // we can iterate through their transactions, and build a lookup for excluding duplicated transactions. - err := b.db.View(func(btx *badger.Txn) error { + // + // RATE LIMITING: the builder module can be configured to limit the + // rate at which transactions with a common payer are included in + // blocks. Depending on the configured limit, we either allow 1 + // transaction every N sequential collections, or we allow K transactions + // per collection. The rate limiter tracks transactions included previously + // to enforce rate limit rules for the constructed block. - // TODO (ramtin): enable this again - // b.tracer.StartSpan(parentID, trace.COLBuildOnSetup) - // defer b.tracer.FinishSpan(parentID, trace.COLBuildOnSetup) + span, _ := b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnGetBuildCtx) + buildCtx, err := b.getBlockBuildContext(parentID) + span.End() + if err != nil { + return nil, fmt.Errorf("could not get block build context: %w", err) + } - err := operation.RetrieveHeader(parentID, &parent)(btx) - if err != nil { - return fmt.Errorf("could not retrieve parent: %w", err) - } + log := b.log.With(). + Hex("parent_id", parentID[:]). + Str("chain_id", buildCtx.parent.ChainID.String()). + Uint64("final_ref_height", buildCtx.refChainFinalizedHeight). + Logger() + log.Debug().Msg("building new cluster block") - // retrieve the height and ID of the latest finalized block ON THE MAIN CHAIN - // this is used as the reference point for transaction expiry - err = operation.RetrieveFinalizedHeight(&refChainFinalizedHeight)(btx) - if err != nil { - return fmt.Errorf("could not retrieve main finalized height: %w", err) - } - err = operation.LookupBlockHeight(refChainFinalizedHeight, &refChainFinalizedID)(btx) - if err != nil { - return fmt.Errorf("could not retrieve main finalized ID: %w", err) - } + // STEP 1a: create a lookup of all transactions included in UN-FINALIZED ancestors. + // In contrast to the transactions collected in step 1b, transactions in un-finalized + // collections cannot be removed from the mempool, as we would want to include + // such transactions in other forks. + span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnUnfinalizedLookup) + err = b.populateUnfinalizedAncestryLookup(buildCtx) + span.End() + if err != nil { + return nil, fmt.Errorf("could not populate un-finalized ancestry lookout (parent_id=%x): %w", parentID, err) + } - // retrieve the finalized boundary ON THE CLUSTER CHAIN - err = procedure.RetrieveLatestFinalizedClusterHeader(parent.ChainID, &clusterChainFinalizedBlock)(btx) - if err != nil { - return fmt.Errorf("could not retrieve cluster final: %w", err) - } - return nil - }) + // STEP 1b: create a lookup of all transactions previously included in + // the finalized collections. Any transactions already included in finalized + // collections can be removed from the mempool. + span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnFinalizedLookup) + err = b.populateFinalizedAncestryLookup(buildCtx) + span.End() + if err != nil { + return nil, fmt.Errorf("could not populate finalized ancestry lookup: %w", err) + } + + // STEP 2: build a payload of valid transactions, while at the same + // time figuring out the correct reference block ID for the collection. + span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnCreatePayload) + payload, err := b.buildPayload(buildCtx) + span.End() if err != nil { - return nil, err + return nil, fmt.Errorf("could not build payload: %w", err) } - // pre-compute the minimum possible reference block height for transactions - // included in this collection (actual reference height may be greater) - minPossibleRefHeight := refChainFinalizedHeight - uint64(flow.DefaultTransactionExpiry-b.config.ExpiryBuffer) - if minPossibleRefHeight > refChainFinalizedHeight { - minPossibleRefHeight = 0 // overflow check + // STEP 3: we have a set of transactions that are valid to include on this fork. + // Now we create the header for the cluster block. + span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnCreateHeader) + header, err := b.buildHeader(buildCtx, payload, setter) + span.End() + if err != nil { + return nil, fmt.Errorf("could not build header: %w", err) } - log := b.log.With(). - Hex("parent_id", parentID[:]). - Str("chain_id", parent.ChainID.String()). - Uint64("final_ref_height", refChainFinalizedHeight). - Logger() - log.Debug().Msg("building new cluster block") + proposal := cluster.Block{ + Header: header, + Payload: payload, + } - // TODO (ramtin): enable this again - // b.tracer.FinishSpan(parentID, trace.COLBuildOnSetup) - // b.tracer.StartSpan(parentID, trace.COLBuildOnUnfinalizedLookup) - // defer b.tracer.FinishSpan(parentID, trace.COLBuildOnUnfinalizedLookup) + // STEP 4: insert the cluster block to the database. + span, _ = b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnDBInsert) + err = operation.RetryOnConflict(b.db.Update, procedure.InsertClusterBlock(&proposal)) + span.End() + if err != nil { + return nil, fmt.Errorf("could not insert built block: %w", err) + } - // STEP TWO: create a lookup of all previously used transactions on the - // part of the chain we care about. We do this separately for - // un-finalized and finalized sections of the chain to decide whether to - // remove conflicting transactions from the mempool. + return proposal.Header, nil +} - // keep track of transactions in the ancestry to avoid duplicates - lookup := newTransactionLookup() - // keep track of transactions to enforce rate limiting - limiter := newRateLimiter(b.config, parent.Height+1) +// getBlockBuildContext retrieves the required contextual information from the database +// required to build a new block proposal. +// No errors are expected during normal operation. +func (b *Builder) getBlockBuildContext(parentID flow.Identifier) (*blockBuildContext, error) { + ctx := new(blockBuildContext) + ctx.config = b.config + ctx.parentID = parentID + ctx.lookup = newTransactionLookup() + + var err error + ctx.parent, err = b.clusterHeaders.ByBlockID(parentID) + if err != nil { + return nil, fmt.Errorf("could not get parent: %w", err) + } + ctx.limiter = newRateLimiter(b.config, ctx.parent.Height+1) - // RATE LIMITING: the builder module can be configured to limit the - // rate at which transactions with a common payer are included in - // blocks. Depending on the configured limit, we either allow 1 - // transaction every N sequential collections, or we allow K transactions - // per collection. + // retrieve the finalized boundary ON THE CLUSTER CHAIN + ctx.clusterChainFinalizedBlock, err = b.clusterState.Final().Head() + if err != nil { + return nil, fmt.Errorf("could not retrieve cluster chain finalized header: %w", err) + } - // first, look up previously included transactions in UN-FINALIZED ancestors - err = b.populateUnfinalizedAncestryLookup(parentID, clusterChainFinalizedBlock.Height, lookup, limiter) + // retrieve the height and ID of the latest finalized block ON THE MAIN CHAIN + // this is used as the reference point for transaction expiry + mainChainFinalizedHeader, err := b.protoState.Final().Head() if err != nil { - return nil, fmt.Errorf("could not populate un-finalized ancestry lookout (parent_id=%x): %w", parentID, err) + return nil, fmt.Errorf("could not retrieve main chain finalized header: %w", err) + } + ctx.refChainFinalizedHeight = mainChainFinalizedHeader.Height + ctx.refChainFinalizedID = mainChainFinalizedHeader.ID() + + // if the epoch has ended and the final block is cached, use the cached values + if b.epochFinalHeight != nil && b.epochFinalID != nil { + ctx.refEpochFinalID = b.epochFinalID + ctx.refEpochFinalHeight = b.epochFinalHeight + return ctx, nil } - // TODO (ramtin): enable this again - // b.tracer.FinishSpan(parentID, trace.COLBuildOnUnfinalizedLookup) - // b.tracer.StartSpan(parentID, trace.COLBuildOnFinalizedLookup) - // defer b.tracer.FinishSpan(parentID, trace.COLBuildOnFinalizedLookup) + // otherwise, attempt to read them from storage + err = b.db.View(func(btx *badger.Txn) error { + var refEpochFinalHeight uint64 + var refEpochFinalID flow.Identifier - // second, look up previously included transactions in FINALIZED ancestors - err = b.populateFinalizedAncestryLookup(minPossibleRefHeight, refChainFinalizedHeight, lookup, limiter) + err = operation.RetrieveEpochLastHeight(b.clusterEpoch, &refEpochFinalHeight)(btx) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil + } + return fmt.Errorf("unexpected failure to retrieve final height of operating epoch: %w", err) + } + err = operation.LookupBlockHeight(refEpochFinalHeight, &refEpochFinalID)(btx) + if err != nil { + // if we are able to retrieve the epoch's final height, the block must be finalized + // therefore failing to look up its height here is an unexpected error + return irrecoverable.NewExceptionf("could not retrieve ID of finalized final block of operating epoch: %w", err) + } + + // cache the values + b.epochFinalHeight = &refEpochFinalHeight + b.epochFinalID = &refEpochFinalID + // store the values in the build context + ctx.refEpochFinalID = b.epochFinalID + ctx.refEpochFinalHeight = b.epochFinalHeight + + return nil + }) if err != nil { - return nil, fmt.Errorf("could not populate finalized ancestry lookup: %w", err) + return nil, fmt.Errorf("could not get block build context: %w", err) } + return ctx, nil +} - // TODO (ramtin): enable this again - // b.tracer.FinishSpan(parentID, trace.COLBuildOnFinalizedLookup) - // b.tracer.StartSpan(parentID, trace.COLBuildOnCreatePayload) - // defer b.tracer.FinishSpan(parentID, trace.COLBuildOnCreatePayload) +// populateUnfinalizedAncestryLookup traverses the unfinalized ancestry backward +// to populate the transaction lookup (used for deduplication) and the rate limiter +// (used to limit transaction submission by payer). +// +// The traversal begins with the block specified by parentID (the block we are +// building on top of) and ends with the oldest unfinalized block in the ancestry. +func (b *Builder) populateUnfinalizedAncestryLookup(ctx *blockBuildContext) error { + err := fork.TraverseBackward(b.clusterHeaders, ctx.parentID, func(ancestor *flow.Header) error { + payload, err := b.payloads.ByBlockID(ancestor.ID()) + if err != nil { + return fmt.Errorf("could not retrieve ancestor payload: %w", err) + } - // STEP THREE: build a payload of valid transactions, while at the same - // time figuring out the correct reference block ID for the collection. + for _, tx := range payload.Collection.Transactions { + ctx.lookup.addUnfinalizedAncestor(tx.ID()) + ctx.limiter.addAncestor(ancestor.Height, tx) + } + return nil + }, fork.ExcludingHeight(ctx.clusterChainFinalizedBlock.Height)) + return err +} +// populateFinalizedAncestryLookup traverses the reference block height index to +// populate the transaction lookup (used for deduplication) and the rate limiter +// (used to limit transaction submission by payer). +// +// The traversal is structured so that we check every collection whose reference +// block height translates to a possible constituent transaction which could also +// appear in the collection we are building. +func (b *Builder) populateFinalizedAncestryLookup(ctx *blockBuildContext) error { + minRefHeight := ctx.lowestPossibleReferenceBlockHeight() + maxRefHeight := ctx.highestPossibleReferenceBlockHeight() + lookup := ctx.lookup + limiter := ctx.limiter + + // Let E be the global transaction expiry constant, measured in blocks. For each + // T ∈ `includedTransactions`, we have to decide whether the transaction + // already appeared in _any_ finalized cluster block. + // Notation: + // - consider a valid cluster block C and let c be its reference block height + // - consider a transaction T ∈ `includedTransactions` and let t denote its + // reference block height + // + // Boundary conditions: + // 1. C's reference block height is equal to the lowest reference block height of + // all its constituent transactions. Hence, for collection C to potentially contain T, it must satisfy c <= t. + // 2. For T to be eligible for inclusion in collection C, _none_ of the transactions within C are allowed + // to be expired w.r.t. C's reference block. Hence, for collection C to potentially contain T, it must satisfy t < c + E. + // + // Therefore, for collection C to potentially contain transaction T, it must satisfy t - E < c <= t. + // In other words, we only need to inspect collections with reference block height c ∈ (t-E, t]. + // Consequently, for a set of transactions, with `minRefHeight` (`maxRefHeight`) being the smallest (largest) + // reference block height, we only need to inspect collections with c ∈ (minRefHeight-E, maxRefHeight]. + + // the finalized cluster blocks which could possibly contain any conflicting transactions + var clusterBlockIDs []flow.Identifier + start, end := findRefHeightSearchRangeForConflictingClusterBlocks(minRefHeight, maxRefHeight) + err := b.db.View(operation.LookupClusterBlocksByReferenceHeightRange(start, end, &clusterBlockIDs)) + if err != nil { + return fmt.Errorf("could not lookup finalized cluster blocks by reference height range [%d,%d]: %w", start, end, err) + } + + for _, blockID := range clusterBlockIDs { + header, err := b.clusterHeaders.ByBlockID(blockID) + if err != nil { + return fmt.Errorf("could not retrieve cluster header (id=%x): %w", blockID, err) + } + payload, err := b.payloads.ByBlockID(blockID) + if err != nil { + return fmt.Errorf("could not retrieve cluster payload (block_id=%x): %w", blockID, err) + } + for _, tx := range payload.Collection.Transactions { + lookup.addFinalizedAncestor(tx.ID()) + limiter.addAncestor(header.Height, tx) + } + } + + return nil +} + +// buildPayload constructs a valid payload based on transactions available in the mempool. +// If the mempool is empty, an empty payload will be returned. +// No errors are expected during normal operation. +func (b *Builder) buildPayload(buildCtx *blockBuildContext) (*cluster.Payload, error) { + lookup := buildCtx.lookup + limiter := buildCtx.limiter + maxRefHeight := buildCtx.highestPossibleReferenceBlockHeight() // keep track of the actual smallest reference height of all included transactions - minRefHeight := uint64(math.MaxUint64) - minRefID := refChainFinalizedID + minRefHeight := maxRefHeight + minRefID := buildCtx.highestPossibleReferenceBlockID() var transactions []*flow.TransactionBody var totalByteSize uint64 @@ -247,29 +410,30 @@ func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) er return nil, fmt.Errorf("could not retrieve reference header: %w", err) } - // disallow un-finalized reference blocks - if refChainFinalizedHeight < refHeader.Height { + // disallow un-finalized reference blocks, and reference blocks beyond the cluster's operating epoch + if refHeader.Height > maxRefHeight { continue } + + txID := tx.ID() // make sure the reference block is finalized and not orphaned - blockFinalizedAtReferenceHeight, err := b.mainHeaders.ByHeight(refHeader.Height) + blockIDFinalizedAtRefHeight, err := b.mainHeaders.BlockIDByHeight(refHeader.Height) if err != nil { - return nil, fmt.Errorf("could not check that reference block (id=%x) is finalized: %w", tx.ReferenceBlockID, err) + return nil, fmt.Errorf("could not check that reference block (id=%x) for transaction (id=%x) is finalized: %w", tx.ReferenceBlockID, txID, err) } - if blockFinalizedAtReferenceHeight.ID() != tx.ReferenceBlockID { + if blockIDFinalizedAtRefHeight != tx.ReferenceBlockID { // the transaction references an orphaned block - it will never be valid - b.transactions.Remove(tx.ID()) + b.transactions.Remove(txID) continue } // ensure the reference block is not too old - if refHeader.Height < minPossibleRefHeight { + if refHeader.Height < buildCtx.lowestPossibleReferenceBlockHeight() { // the transaction is expired, it will never be valid - b.transactions.Remove(tx.ID()) + b.transactions.Remove(txID) continue } - txID := tx.ID() // check that the transaction was not already used in un-finalized history if lookup.isUnfinalizedAncestor(txID) { continue @@ -315,21 +479,20 @@ func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) er totalGas += tx.GasLimit } - // STEP FOUR: we have a set of transactions that are valid to include - // on this fork. Now we need to create the collection that will be - // used in the payload and construct the final proposal model - // TODO (ramtin): enable this again - // b.tracer.FinishSpan(parentID, trace.COLBuildOnCreatePayload) - // b.tracer.StartSpan(parentID, trace.COLBuildOnCreateHeader) - // defer b.tracer.FinishSpan(parentID, trace.COLBuildOnCreateHeader) - // build the payload from the transactions payload := cluster.PayloadFromTransactions(minRefID, transactions...) + return &payload, nil +} + +// buildHeader constructs the header for the cluster block being built. +// It invokes the HotStuff setter to set fields related to HotStuff (QC, etc.). +// No errors are expected during normal operation. +func (b *Builder) buildHeader(ctx *blockBuildContext, payload *cluster.Payload, setter func(header *flow.Header) error) (*flow.Header, error) { header := &flow.Header{ - ChainID: parent.ChainID, - ParentID: parentID, - Height: parent.Height + 1, + ChainID: ctx.parent.ChainID, + ParentID: ctx.parentID, + Height: ctx.parent.Height + 1, PayloadHash: payload.Hash(), Timestamp: time.Now().UTC(), @@ -338,110 +501,11 @@ func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) er } // set fields specific to the consensus algorithm - err = setter(header) + err := setter(header) if err != nil { return nil, fmt.Errorf("could not set fields to header: %w", err) } - - proposal = cluster.Block{ - Header: header, - Payload: &payload, - } - - // TODO (ramtin): enable this again - // b.tracer.FinishSpan(parentID, trace.COLBuildOnCreateHeader) - - span, ctx := b.tracer.StartCollectionSpan(context.Background(), proposal.ID(), trace.COLBuildOn, otelTrace.WithTimestamp(startTime)) - defer span.End() - - dbInsertSpan, _ := b.tracer.StartSpanFromContext(ctx, trace.COLBuildOnDBInsert) - defer dbInsertSpan.End() - - // finally we insert the block in a write transaction - err = operation.RetryOnConflict(b.db.Update, procedure.InsertClusterBlock(&proposal)) - if err != nil { - return nil, fmt.Errorf("could not insert built block: %w", err) - } - - return proposal.Header, nil -} - -// populateUnfinalizedAncestryLookup traverses the unfinalized ancestry backward -// to populate the transaction lookup (used for deduplication) and the rate limiter -// (used to limit transaction submission by payer). -// -// The traversal begins with the block specified by parentID (the block we are -// building on top of) and ends with the oldest unfinalized block in the ancestry. -func (b *Builder) populateUnfinalizedAncestryLookup(parentID flow.Identifier, finalHeight uint64, lookup *transactionLookup, limiter *rateLimiter) error { - - err := fork.TraverseBackward(b.clusterHeaders, parentID, func(ancestor *flow.Header) error { - payload, err := b.payloads.ByBlockID(ancestor.ID()) - if err != nil { - return fmt.Errorf("could not retrieve ancestor payload: %w", err) - } - - for _, tx := range payload.Collection.Transactions { - lookup.addUnfinalizedAncestor(tx.ID()) - limiter.addAncestor(ancestor.Height, tx) - } - return nil - }, fork.ExcludingHeight(finalHeight)) - - return err -} - -// populateFinalizedAncestryLookup traverses the reference block height index to -// populate the transaction lookup (used for deduplication) and the rate limiter -// (used to limit transaction submission by payer). -// -// The traversal is structured so that we check every collection whose reference -// block height translates to a possible constituent transaction which could also -// appear in the collection we are building. -func (b *Builder) populateFinalizedAncestryLookup(minRefHeight, maxRefHeight uint64, lookup *transactionLookup, limiter *rateLimiter) error { - - // Let E be the global transaction expiry constant, measured in blocks. For each - // T ∈ `includedTransactions`, we have to decide whether the transaction - // already appeared in _any_ finalized cluster block. - // Notation: - // - consider a valid cluster block C and let c be its reference block height - // - consider a transaction T ∈ `includedTransactions` and let t denote its - // reference block height - // - // Boundary conditions: - // 1. C's reference block height is equal to the lowest reference block height of - // all its constituent transactions. Hence, for collection C to potentially contain T, it must satisfy c <= t. - // 2. For T to be eligible for inclusion in collection C, _none_ of the transactions within C are allowed - // to be expired w.r.t. C's reference block. Hence, for collection C to potentially contain T, it must satisfy t < c + E. - // - // Therefore, for collection C to potentially contain transaction T, it must satisfy t - E < c <= t. - // In other words, we only need to inspect collections with reference block height c ∈ (t-E, t]. - // Consequently, for a set of transactions, with `minRefHeight` (`maxRefHeight`) being the smallest (largest) - // reference block height, we only need to inspect collections with c ∈ (minRefHeight-E, maxRefHeight]. - - // the finalized cluster blocks which could possibly contain any conflicting transactions - var clusterBlockIDs []flow.Identifier - start, end := findRefHeightSearchRangeForConflictingClusterBlocks(minRefHeight, maxRefHeight) - err := b.db.View(operation.LookupClusterBlocksByReferenceHeightRange(start, end, &clusterBlockIDs)) - if err != nil { - return fmt.Errorf("could not lookup finalized cluster blocks by reference height range [%d,%d]: %w", start, end, err) - } - - for _, blockID := range clusterBlockIDs { - header, err := b.clusterHeaders.ByBlockID(blockID) - if err != nil { - return fmt.Errorf("could not retrieve cluster header (id=%x): %w", blockID, err) - } - payload, err := b.payloads.ByBlockID(blockID) - if err != nil { - return fmt.Errorf("could not retrieve cluster payload (block_id=%x): %w", blockID, err) - } - for _, tx := range payload.Collection.Transactions { - lookup.addFinalizedAncestor(tx.ID()) - limiter.addAncestor(header.Height, tx) - } - } - - return nil + return header, nil } // findRefHeightSearchRangeForConflictingClusterBlocks computes the range of reference diff --git a/module/builder/collection/builder_test.go b/module/builder/collection/builder_test.go index 91677776730..9641b7c934a 100644 --- a/module/builder/collection/builder_test.go +++ b/module/builder/collection/builder_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "os" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/rs/zerolog" @@ -42,8 +41,9 @@ type BuilderSuite struct { db *badger.DB dbdir string - genesis *model.Block - chainID flow.ChainID + genesis *model.Block + chainID flow.ChainID + epochCounter uint64 headers storage.Headers payloads storage.ClusterPayloads @@ -62,9 +62,6 @@ type BuilderSuite struct { func (suite *BuilderSuite) SetupTest() { var err error - // seed the RNG - rand.Seed(time.Now().UnixNano()) - suite.genesis = model.Genesis() suite.chainID = suite.genesis.Header.ChainID @@ -78,12 +75,22 @@ func (suite *BuilderSuite) SetupTest() { log := zerolog.Nop() all := sutil.StorageLayer(suite.T(), suite.db) consumer := events.NewNoop() + suite.headers = all.Headers suite.blocks = all.Blocks suite.payloads = bstorage.NewClusterPayloads(metrics, suite.db) + // just bootstrap with a genesis block, we'll use this as reference + root, result, seal := unittest.BootstrapFixture(unittest.IdentityListFixture(5, unittest.WithAllRoles())) + // ensure we don't enter a new epoch for tests that build many blocks + result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = root.Header.View + 100000 + seal.ResultID = result.ID() + rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID()))) + require.NoError(suite.T(), err) + suite.epochCounter = rootSnapshot.Encodable().Epochs.Current.Counter + clusterQC := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID())) - clusterStateRoot, err := clusterkv.NewStateRoot(suite.genesis, clusterQC) + clusterStateRoot, err := clusterkv.NewStateRoot(suite.genesis, clusterQC, suite.epochCounter) suite.Require().NoError(err) clusterState, err := clusterkv.Bootstrap(suite.db, clusterStateRoot) suite.Require().NoError(err) @@ -91,17 +98,20 @@ func (suite *BuilderSuite) SetupTest() { suite.state, err = clusterkv.NewMutableState(clusterState, tracer, suite.headers, suite.payloads) suite.Require().NoError(err) - // just bootstrap with a genesis block, we'll use this as reference - participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) - root, result, seal := unittest.BootstrapFixture(participants) - // ensure we don't enter a new epoch for tests that build many blocks - result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = root.Header.View + 100000 - seal.ResultID = result.ID() - - rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID()))) - require.NoError(suite.T(), err) - - state, err := pbadger.Bootstrap(metrics, suite.db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := pbadger.Bootstrap( + metrics, + suite.db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(suite.T(), err) suite.protoState, err = pbadger.NewFollowerState( @@ -126,7 +136,7 @@ func (suite *BuilderSuite) SetupTest() { suite.Assert().True(added) } - suite.builder, _ = builder.NewBuilder(suite.db, tracer, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger()) + suite.builder, _ = builder.NewBuilder(suite.db, tracer, suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter) } // runs after each test finishes @@ -169,7 +179,7 @@ func (suite *BuilderSuite) Payload(transactions ...*flow.TransactionBody) model. // ProtoStateRoot returns the root block of the protocol state. func (suite *BuilderSuite) ProtoStateRoot() *flow.Header { - root, err := suite.protoState.Params().Root() + root, err := suite.protoState.Params().FinalizedRoot() suite.Require().NoError(err) return root } @@ -479,7 +489,7 @@ func (suite *BuilderSuite) TestBuildOn_LargeHistory() { // use a mempool with 2000 transactions, one per block suite.pool = herocache.NewTransactions(2000, unittest.Logger(), metrics.NewNoopCollector()) - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), builder.WithMaxCollectionSize(10000)) + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(10000)) // get a valid reference block ID final, err := suite.protoState.Final().Head() @@ -559,7 +569,7 @@ func (suite *BuilderSuite) TestBuildOn_LargeHistory() { func (suite *BuilderSuite) TestBuildOn_MaxCollectionSize() { // set the max collection size to 1 - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), builder.WithMaxCollectionSize(1)) + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(1)) // build a block header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) @@ -577,7 +587,7 @@ func (suite *BuilderSuite) TestBuildOn_MaxCollectionSize() { func (suite *BuilderSuite) TestBuildOn_MaxCollectionByteSize() { // set the max collection byte size to 400 (each tx is about 150 bytes) - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), builder.WithMaxCollectionByteSize(400)) + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionByteSize(400)) // build a block header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) @@ -595,7 +605,7 @@ func (suite *BuilderSuite) TestBuildOn_MaxCollectionByteSize() { func (suite *BuilderSuite) TestBuildOn_MaxCollectionTotalGas() { // set the max gas to 20,000 - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), builder.WithMaxCollectionTotalGas(20000)) + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionTotalGas(20000)) // build a block header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) @@ -632,7 +642,7 @@ func (suite *BuilderSuite) TestBuildOn_ExpiredTransaction() { // reset the pool and builder suite.pool = herocache.NewTransactions(10, unittest.Logger(), metrics.NewNoopCollector()) - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger()) + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter) // insert a transaction referring genesis (now expired) tx1 := unittest.TransactionBodyFixture(func(tx *flow.TransactionBody) { @@ -674,7 +684,7 @@ func (suite *BuilderSuite) TestBuildOn_EmptyMempool() { // start with an empty mempool suite.pool = herocache.NewTransactions(1000, unittest.Logger(), metrics.NewNoopCollector()) - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger()) + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter) header, err := suite.builder.BuildOn(suite.genesis.ID(), noopSetter) suite.Require().NoError(err) @@ -701,7 +711,7 @@ func (suite *BuilderSuite) TestBuildOn_NoRateLimiting() { suite.ClearPool() // create builder with no rate limit and max 10 tx/collection - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(10), builder.WithMaxPayerTransactionRate(0), ) @@ -742,7 +752,7 @@ func (suite *BuilderSuite) TestBuildOn_RateLimitNonPayer() { suite.ClearPool() // create builder with 5 tx/payer and max 10 tx/collection - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(10), builder.WithMaxPayerTransactionRate(5), ) @@ -786,7 +796,7 @@ func (suite *BuilderSuite) TestBuildOn_HighRateLimit() { suite.ClearPool() // create builder with 5 tx/payer and max 10 tx/collection - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(10), builder.WithMaxPayerTransactionRate(5), ) @@ -824,7 +834,7 @@ func (suite *BuilderSuite) TestBuildOn_LowRateLimit() { suite.ClearPool() // create builder with .5 tx/payer and max 10 tx/collection - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(10), builder.WithMaxPayerTransactionRate(.5), ) @@ -866,7 +876,7 @@ func (suite *BuilderSuite) TestBuildOn_UnlimitedPayer() { // create builder with 5 tx/payer and max 10 tx/collection // configure an unlimited payer payer := unittest.RandomAddressFixture() - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(10), builder.WithMaxPayerTransactionRate(5), builder.WithUnlimitedPayers(payer), @@ -907,7 +917,7 @@ func (suite *BuilderSuite) TestBuildOn_RateLimitDryRun() { // create builder with 5 tx/payer and max 10 tx/collection // configure an unlimited payer payer := unittest.RandomAddressFixture() - suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), + suite.builder, _ = builder.NewBuilder(suite.db, trace.NewNoopTracer(), suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter, builder.WithMaxCollectionSize(10), builder.WithMaxPayerTransactionRate(5), builder.WithRateLimitDryRun(true), @@ -996,7 +1006,7 @@ func benchmarkBuildOn(b *testing.B, size int) { suite.payloads = bstorage.NewClusterPayloads(metrics, suite.db) qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID())) - stateRoot, err := clusterkv.NewStateRoot(suite.genesis, qc) + stateRoot, err := clusterkv.NewStateRoot(suite.genesis, qc, suite.epochCounter) state, err := clusterkv.Bootstrap(suite.db, stateRoot) assert.NoError(b, err) @@ -1012,7 +1022,7 @@ func benchmarkBuildOn(b *testing.B, size int) { } // create the builder - suite.builder, _ = builder.NewBuilder(suite.db, tracer, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger()) + suite.builder, _ = builder.NewBuilder(suite.db, tracer, suite.protoState, suite.state, suite.headers, suite.headers, suite.payloads, suite.pool, unittest.Logger(), suite.epochCounter) } // create a block history to test performance against diff --git a/module/builder/collection/rate_limiter.go b/module/builder/collection/rate_limiter.go index 615e50e15fa..3643beae89b 100644 --- a/module/builder/collection/rate_limiter.go +++ b/module/builder/collection/rate_limiter.go @@ -11,6 +11,12 @@ type rateLimiter struct { // maximum rate of transactions/payer/collection (from Config) rate float64 + // Derived fields based on the rate. Only one will be used: + // - if rate >= 1, then txPerBlock is set to the number of transactions allowed per payer per block + // - if rate < 1, then blocksPerTx is set to the number of consecutive blocks in which one transaction per payer is allowed + txPerBlock uint + blocksPerTx uint64 + // set of unlimited payer address (from Config) unlimited map[flow.Address]struct{} // height of the collection we are building @@ -31,6 +37,11 @@ func newRateLimiter(conf Config, height uint64) *rateLimiter { latestCollectionHeight: make(map[flow.Address]uint64), txIncludedCount: make(map[flow.Address]uint), } + if limiter.rate >= 1 { + limiter.txPerBlock = uint(math.Floor(limiter.rate)) + } else { + limiter.blocksPerTx = uint64(math.Ceil(1 / limiter.rate)) + } return limiter } @@ -62,14 +73,14 @@ func (limiter *rateLimiter) shouldRateLimit(tx *flow.TransactionBody) bool { // skip rate limiting if it is turned off or the payer is unlimited _, isUnlimited := limiter.unlimited[payer] - if limiter.rate == 0 || isUnlimited { + if limiter.rate <= 0 || isUnlimited { return false } // if rate >=1, we only consider the current collection and rate limit once // the number of transactions for the payer exceeds rate if limiter.rate >= 1 { - if limiter.txIncludedCount[payer] >= uint(math.Floor(limiter.rate)) { + if limiter.txIncludedCount[payer] >= limiter.txPerBlock { return true } } @@ -94,7 +105,7 @@ func (limiter *rateLimiter) shouldRateLimit(tx *flow.TransactionBody) bool { return false } - if limiter.height-latestHeight < uint64(math.Ceil(1/limiter.rate)) { + if limiter.height-latestHeight < limiter.blocksPerTx { return true } } diff --git a/module/cache.go b/module/cache.go new file mode 100644 index 00000000000..96c8d3a6128 --- /dev/null +++ b/module/cache.go @@ -0,0 +1,10 @@ +package module + +import ( + "github.com/onflow/flow-go/model/flow" +) + +// FinalizedHeaderCache is a cache of the latest finalized block header. +type FinalizedHeaderCache interface { + Get() *flow.Header +} diff --git a/module/chunks/chunkVerifier.go b/module/chunks/chunkVerifier.go index f5d1d3804b8..fb1b81fbf98 100644 --- a/module/chunks/chunkVerifier.go +++ b/module/chunks/chunkVerifier.go @@ -11,11 +11,11 @@ import ( "github.com/onflow/flow-go/engine/execution/computation/computer" executionState "github.com/onflow/flow-go/engine/execution/state" - "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/fvm" - fvmState "github.com/onflow/flow-go/fvm/state" - "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/derived" + "github.com/onflow/flow-go/fvm/storage/logical" + "github.com/onflow/flow-go/fvm/storage/snapshot" + fvmState "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/partial" chmodels "github.com/onflow/flow-go/model/chunks" @@ -57,7 +57,15 @@ func (fcv *ChunkVerifier) Verify( if vc.IsSystemChunk { ctx = fvm.NewContextFromParent( fcv.systemChunkCtx, - fvm.WithBlockHeader(vc.Header)) + fvm.WithBlockHeader(vc.Header), + // `protocol.Snapshot` implements `EntropyProvider` interface + // Note that `Snapshot` possible errors for RandomSource() are: + // - storage.ErrNotFound if the QC is unknown. + // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown + // However, at this stage, snapshot reference block should be known and the QC should also be known, + // so no error is expected in normal operations, as required by `EntropyProvider`. + fvm.WithEntropyProvider(vc.Snapshot), + ) txBody, err := blueprints.SystemChunkTransaction(fcv.vmCtx.Chain) if err != nil { @@ -70,7 +78,15 @@ func (fcv *ChunkVerifier) Verify( } else { ctx = fvm.NewContextFromParent( fcv.vmCtx, - fvm.WithBlockHeader(vc.Header)) + fvm.WithBlockHeader(vc.Header), + // `protocol.Snapshot` implements `EntropyProvider` interface + // Note that `Snapshot` possible errors for RandomSource() are: + // - storage.ErrNotFound if the QC is unknown. + // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown + // However, at this stage, snapshot reference block should be known and the QC should also be known, + // so no error is expected in normal operations, as required by `EntropyProvider`. + fvm.WithEntropyProvider(vc.Snapshot), + ) transactions = make( []*fvm.TransactionProcedure, @@ -94,7 +110,7 @@ func (fcv *ChunkVerifier) Verify( } type partialLedgerStorageSnapshot struct { - snapshot fvmState.StorageSnapshot + snapshot snapshot.StorageSnapshot unknownRegTouch map[flow.RegisterID]struct{} } @@ -166,21 +182,20 @@ func (fcv *ChunkVerifier) verifyTransactionsInContext( context = fvm.NewContextFromParent( context, fvm.WithDerivedBlockData( - derived.NewEmptyDerivedBlockDataWithTransactionOffset( - transactionOffset))) + derived.NewEmptyDerivedBlockData(logical.Time(transactionOffset)))) // chunk view construction // unknown register tracks access to parts of the partial trie which // are not expanded and values are unknown. unknownRegTouch := make(map[flow.RegisterID]struct{}) - snapshotTree := storage.NewSnapshotTree( + snapshotTree := snapshot.NewSnapshotTree( &partialLedgerStorageSnapshot{ snapshot: executionState.NewLedgerStorageSnapshot( psmt, chunkDataPack.StartState), unknownRegTouch: unknownRegTouch, }) - chunkView := delta.NewDeltaView(nil) + chunkState := fvmState.NewExecutionState(nil, fvmState.DefaultParameters()) var problematicTx flow.Identifier // executes all transactions in this chunk @@ -203,7 +218,7 @@ func (fcv *ChunkVerifier) verifyTransactionsInContext( serviceEvents = append(serviceEvents, output.ConvertedServiceEvents...) snapshotTree = snapshotTree.Append(executionSnapshot) - err = chunkView.Merge(executionSnapshot) + err = chunkState.Merge(executionSnapshot) if err != nil { return nil, nil, fmt.Errorf("failed to merge: %d (%w)", i, err) } @@ -223,9 +238,11 @@ func (fcv *ChunkVerifier) verifyTransactionsInContext( return nil, nil, fmt.Errorf("cannot calculate events collection hash: %w", err) } if chunk.EventCollection != eventsHash { - + collectionID := "" + if chunkDataPack.Collection != nil { + collectionID = chunkDataPack.Collection.ID().String() + } for i, event := range events { - fcv.logger.Warn().Int("list_index", i). Str("event_id", event.ID().String()). Hex("event_fingerptint", event.Fingerprint()). @@ -235,7 +252,7 @@ func (fcv *ChunkVerifier) verifyTransactionsInContext( Uint32("event_index", event.EventIndex). Bytes("event_payload", event.Payload). Str("block_id", chunk.BlockID.String()). - Str("collection_id", chunkDataPack.Collection.ID().String()). + Str("collection_id", collectionID). Str("result_id", result.ID().String()). Uint64("chunk_index", chunk.Index). Msg("not matching events debug") @@ -257,7 +274,7 @@ func (fcv *ChunkVerifier) verifyTransactionsInContext( // Applying chunk updates to the partial trie. This returns the expected // end state commitment after updates and the list of register keys that // was not provided by the chunk data package (err). - chunkExecutionSnapshot := chunkView.Finalize() + chunkExecutionSnapshot := chunkState.Finalize() keys, values := executionState.RegisterEntriesToKeysValues( chunkExecutionSnapshot.UpdatedRegisters()) diff --git a/module/chunks/chunkVerifier_test.go b/module/chunks/chunkVerifier_test.go index 14f4a509962..ba01a33c49b 100644 --- a/module/chunks/chunkVerifier_test.go +++ b/module/chunks/chunkVerifier_test.go @@ -2,9 +2,7 @@ package chunks_test import ( "fmt" - "math/rand" "testing" - "time" "github.com/onflow/cadence/runtime" "github.com/rs/zerolog" @@ -15,7 +13,8 @@ import ( executionState "github.com/onflow/flow-go/engine/execution/state" "github.com/onflow/flow-go/fvm" fvmErrors "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger" completeLedger "github.com/onflow/flow-go/ledger/complete" "github.com/onflow/flow-go/ledger/complete/wal/fixtures" @@ -67,9 +66,6 @@ type ChunkVerifierTestSuite struct { // Make sure variables are set properly // SetupTest is executed prior to each individual test in this test suite func (s *ChunkVerifierTestSuite) SetupSuite() { - // seed the RNG - rand.Seed(time.Now().UnixNano()) - vm := new(vmMock) systemOkVm := new(vmSystemOkMock) systemBadVm := new(vmSystemBadMock) @@ -354,12 +350,20 @@ func GetBaselineVerifiableChunk(t *testing.T, script string, system bool) *verif type vmMock struct{} +func (vm *vmMock) NewExecutor( + ctx fvm.Context, + proc fvm.Procedure, + txn storage.TransactionPreparer, +) fvm.ProcedureExecutor { + panic("not implemented") +} + func (vm *vmMock) Run( ctx fvm.Context, proc fvm.Procedure, - storage state.StorageSnapshot, + storage snapshot.StorageSnapshot, ) ( - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, fvm.ProcedureOutput, error, ) { @@ -369,7 +373,7 @@ func (vm *vmMock) Run( "invokable is not a transaction") } - snapshot := &state.ExecutionSnapshot{} + snapshot := &snapshot.ExecutionSnapshot{} output := fvm.ProcedureOutput{} id0 := flow.NewRegisterID("00", "") @@ -413,7 +417,7 @@ func (vm *vmMock) Run( func (vmMock) GetAccount( _ fvm.Context, _ flow.Address, - _ state.StorageSnapshot, + _ snapshot.StorageSnapshot, ) ( *flow.Account, error) { @@ -422,12 +426,20 @@ func (vmMock) GetAccount( type vmSystemOkMock struct{} +func (vm *vmSystemOkMock) NewExecutor( + ctx fvm.Context, + proc fvm.Procedure, + txn storage.TransactionPreparer, +) fvm.ProcedureExecutor { + panic("not implemented") +} + func (vm *vmSystemOkMock) Run( ctx fvm.Context, proc fvm.Procedure, - storage state.StorageSnapshot, + storage snapshot.StorageSnapshot, ) ( - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, fvm.ProcedureOutput, error, ) { @@ -441,7 +453,7 @@ func (vm *vmSystemOkMock) Run( id5 := flow.NewRegisterID("05", "") // add "default" interaction expected in tests - snapshot := &state.ExecutionSnapshot{ + snapshot := &snapshot.ExecutionSnapshot{ ReadSet: map[flow.RegisterID]struct{}{ id0: struct{}{}, id5: struct{}{}, @@ -461,7 +473,7 @@ func (vm *vmSystemOkMock) Run( func (vmSystemOkMock) GetAccount( _ fvm.Context, _ flow.Address, - _ state.StorageSnapshot, + _ snapshot.StorageSnapshot, ) ( *flow.Account, error, @@ -471,12 +483,20 @@ func (vmSystemOkMock) GetAccount( type vmSystemBadMock struct{} +func (vm *vmSystemBadMock) NewExecutor( + ctx fvm.Context, + proc fvm.Procedure, + txn storage.TransactionPreparer, +) fvm.ProcedureExecutor { + panic("not implemented") +} + func (vm *vmSystemBadMock) Run( ctx fvm.Context, proc fvm.Procedure, - storage state.StorageSnapshot, + storage snapshot.StorageSnapshot, ) ( - *state.ExecutionSnapshot, + *snapshot.ExecutionSnapshot, fvm.ProcedureOutput, error, ) { @@ -492,13 +512,13 @@ func (vm *vmSystemBadMock) Run( ConvertedServiceEvents: flow.ServiceEventList{*epochCommitServiceEvent}, } - return &state.ExecutionSnapshot{}, output, nil + return &snapshot.ExecutionSnapshot{}, output, nil } func (vmSystemBadMock) GetAccount( _ fvm.Context, _ flow.Address, - _ state.StorageSnapshot, + _ snapshot.StorageSnapshot, ) ( *flow.Account, error, diff --git a/module/chunks/chunk_assigner.go b/module/chunks/chunk_assigner.go index 7ac2247c997..dc34816a784 100644 --- a/module/chunks/chunk_assigner.go +++ b/module/chunks/chunk_assigner.go @@ -12,7 +12,7 @@ import ( "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/mempool/stdmap" "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/state/protocol/seed" + "github.com/onflow/flow-go/state/protocol/prg" ) // ChunkAssigner implements an instance of the Public Chunk Assignment @@ -98,7 +98,7 @@ func (p *ChunkAssigner) rngByBlockID(stateSnapshot protocol.Snapshot) (random.Ra return nil, fmt.Errorf("failed to retrieve source of randomness: %w", err) } - rng, err := seed.PRGFromRandomSource(randomSource, seed.ProtocolVerificationChunkAssignment) + rng, err := prg.New(randomSource, prg.VerificationChunkAssignment, nil) if err != nil { return nil, fmt.Errorf("failed to instantiate random number generator: %w", err) } diff --git a/module/chunks/chunk_assigner_test.go b/module/chunks/chunk_assigner_test.go index 1c65c91d817..ea6907c2e70 100644 --- a/module/chunks/chunk_assigner_test.go +++ b/module/chunks/chunk_assigner_test.go @@ -1,7 +1,7 @@ package chunks import ( - "math/rand" + "crypto/rand" "testing" "github.com/stretchr/testify/mock" @@ -10,8 +10,8 @@ import ( chmodels "github.com/onflow/flow-go/model/chunks" "github.com/onflow/flow-go/model/flow" - protocolMock "github.com/onflow/flow-go/state/protocol/mock" - "github.com/onflow/flow-go/state/protocol/seed" + protocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/state/protocol/prg" "github.com/onflow/flow-go/utils/unittest" ) @@ -21,7 +21,7 @@ type PublicAssignmentTestSuite struct { } // Setup test with n verification nodes -func (a *PublicAssignmentTestSuite) SetupTest(n int) (*flow.Header, *protocolMock.Snapshot, *protocolMock.State) { +func (a *PublicAssignmentTestSuite) SetupTest(n int) (*flow.Header, *protocol.Snapshot, *protocol.State) { nodes := make([]flow.Role, 0) for i := 1; i < n; i++ { nodes = append(nodes, flow.RoleVerification) @@ -361,7 +361,7 @@ func (a *PublicAssignmentTestSuite) CreateResult(head *flow.Header, num int, t * } func (a *PublicAssignmentTestSuite) GetSeed(t *testing.T) []byte { - seed := make([]byte, seed.RandomSourceLength) + seed := make([]byte, prg.RandomSourceLength) _, err := rand.Read(seed) require.NoError(t, err) return seed diff --git a/module/compliance/config.go b/module/compliance/config.go index 7409b0acd4a..97707cfaac8 100644 --- a/module/compliance/config.go +++ b/module/compliance/config.go @@ -18,6 +18,15 @@ func DefaultConfig() Config { } } +// GetSkipNewProposalsThreshold returns stored value in config possibly applying a lower bound. +func (c *Config) GetSkipNewProposalsThreshold() uint64 { + if c.SkipNewProposalsThreshold < MinSkipNewProposalsThreshold { + return MinSkipNewProposalsThreshold + } + + return c.SkipNewProposalsThreshold +} + type Opt func(*Config) // WithSkipNewProposalsThreshold returns an option to set the skip new proposals diff --git a/module/component/component.go b/module/component/component.go index 34f8f61cf14..07fc387e077 100644 --- a/module/component/component.go +++ b/module/component/component.go @@ -157,10 +157,10 @@ func NewComponentManagerBuilder() ComponentManagerBuilder { return &componentManagerBuilderImpl{} } -// AddWorker adds a ComponentWorker closure to the ComponentManagerBuilder // All worker functions will be run in parallel when the ComponentManager is started. // Note: AddWorker is not concurrency-safe, and should only be called on an individual builder -// within a single goroutine. +// within a single goroutine.// AddWorker adds a ComponentWorker closure to the ComponentManagerBuilder + func (c *componentManagerBuilderImpl) AddWorker(worker ComponentWorker) ComponentManagerBuilder { c.workers = append(c.workers, worker) return c diff --git a/engine/consensus/sealing/counters/monotonous_counter.go b/module/counters/monotonous_counter.go similarity index 100% rename from engine/consensus/sealing/counters/monotonous_counter.go rename to module/counters/monotonous_counter.go diff --git a/engine/consensus/sealing/counters/monotonous_counter_test.go b/module/counters/monotonous_counter_test.go similarity index 100% rename from engine/consensus/sealing/counters/monotonous_counter_test.go rename to module/counters/monotonous_counter_test.go diff --git a/module/dkg/broker.go b/module/dkg/broker.go index 13709d02ed3..f94fbc981fe 100644 --- a/module/dkg/broker.go +++ b/module/dkg/broker.go @@ -99,7 +99,7 @@ func NewBroker( b := &Broker{ config: config, - log: log.With().Str("component", "broker").Str("dkg_instance_id", dkgInstanceID).Logger(), + log: log.With().Str("component", "dkg_broker").Str("dkg_instance_id", dkgInstanceID).Logger(), unit: engine.NewUnit(), dkgInstanceID: dkgInstanceID, committee: committee, diff --git a/module/dkg/controller.go b/module/dkg/controller.go index 5c9adf4994a..ae4b54ecb38 100644 --- a/module/dkg/controller.go +++ b/module/dkg/controller.go @@ -3,7 +3,6 @@ package dkg import ( "fmt" "math" - "math/rand" "sync" "time" @@ -12,6 +11,7 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/utils/rand" ) const ( @@ -304,7 +304,10 @@ func (c *Controller) doBackgroundWork() { isFirstMessage := false c.once.Do(func() { isFirstMessage = true - delay := c.preHandleFirstBroadcastDelay() + delay, err := c.preHandleFirstBroadcastDelay() + if err != nil { + c.log.Err(err).Msg("pre handle first broadcast delay failed") + } c.log.Info().Msgf("sleeping for %s before processing first phase 1 broadcast message", delay) time.Sleep(delay) }) @@ -337,12 +340,15 @@ func (c *Controller) start() error { // before starting the DKG, sleep for a random delay to avoid synchronizing // this expensive operation across all consensus nodes - delay := c.preStartDelay() + delay, err := c.preStartDelay() + if err != nil { + return fmt.Errorf("pre start delay failed: %w", err) + } c.log.Debug().Msgf("sleeping for %s before starting DKG", delay) time.Sleep(delay) c.dkgLock.Lock() - err := c.dkg.Start(c.seed) + err = c.dkg.Start(c.seed) c.dkgLock.Unlock() if err != nil { return fmt.Errorf("Error starting DKG: %w", err) @@ -421,18 +427,16 @@ func (c *Controller) phase3() error { // preStartDelay returns a duration to delay prior to starting the DKG process. // This prevents synchronization of the DKG starting (an expensive operation) // across the network, which can impact finalization. -func (c *Controller) preStartDelay() time.Duration { - delay := computePreprocessingDelay(c.config.BaseStartDelay, c.dkg.Size()) - return delay +func (c *Controller) preStartDelay() (time.Duration, error) { + return computePreprocessingDelay(c.config.BaseStartDelay, c.dkg.Size()) } // preHandleFirstBroadcastDelay returns a duration to delay prior to handling // the first broadcast message. This delay is used only during phase 1 of the DKG. // This prevents synchronization of processing verification vectors (an // expensive operation) across the network, which can impact finalization. -func (c *Controller) preHandleFirstBroadcastDelay() time.Duration { - delay := computePreprocessingDelay(c.config.BaseHandleFirstBroadcastDelay, c.dkg.Size()) - return delay +func (c *Controller) preHandleFirstBroadcastDelay() (time.Duration, error) { + return computePreprocessingDelay(c.config.BaseHandleFirstBroadcastDelay, c.dkg.Size()) } // computePreprocessingDelay computes a random delay to introduce before an @@ -441,15 +445,18 @@ func (c *Controller) preHandleFirstBroadcastDelay() time.Duration { // The maximum delay is m=b*n^2 where: // * b is a configurable base delay // * n is the size of the DKG committee -func computePreprocessingDelay(baseDelay time.Duration, dkgSize int) time.Duration { +func computePreprocessingDelay(baseDelay time.Duration, dkgSize int) (time.Duration, error) { maxDelay := computePreprocessingDelayMax(baseDelay, dkgSize) if maxDelay <= 0 { - return 0 + return 0, nil } // select delay from [0,m) - delay := time.Duration(rand.Int63n(maxDelay.Nanoseconds())) - return delay + r, err := rand.Uint64n(uint64(maxDelay.Nanoseconds())) + if err != nil { + return time.Duration(0), fmt.Errorf("delay generation failed %w", err) + } + return time.Duration(r), nil } // computePreprocessingDelayMax computes the maximum dely for computePreprocessingDelay. diff --git a/module/dkg/controller_test.go b/module/dkg/controller_test.go index 03f10adf1c1..e8f8d253537 100644 --- a/module/dkg/controller_test.go +++ b/module/dkg/controller_test.go @@ -333,20 +333,26 @@ func checkArtifacts(t *testing.T, nodes []*node, totalNodes int) { func TestDelay(t *testing.T) { t.Run("should return 0 delay for <=0 inputs", func(t *testing.T) { - delay := computePreprocessingDelay(0, 100) + delay, err := computePreprocessingDelay(0, 100) + require.NoError(t, err) assert.Equal(t, delay, time.Duration(0)) - delay = computePreprocessingDelay(time.Hour, 0) + delay, err = computePreprocessingDelay(time.Hour, 0) + require.NoError(t, err) assert.Equal(t, delay, time.Duration(0)) - delay = computePreprocessingDelay(time.Millisecond, -1) + delay, err = computePreprocessingDelay(time.Millisecond, -1) + require.NoError(t, err) assert.Equal(t, delay, time.Duration(0)) - delay = computePreprocessingDelay(-time.Millisecond, 100) + delay, err = computePreprocessingDelay(-time.Millisecond, 100) + require.NoError(t, err) assert.Equal(t, delay, time.Duration(0)) }) // NOTE: this is a probabilistic test. It will (extremely infrequently) fail. t.Run("should return different values for same inputs", func(t *testing.T) { - d1 := computePreprocessingDelay(time.Hour, 100) - d2 := computePreprocessingDelay(time.Hour, 100) + d1, err := computePreprocessingDelay(time.Hour, 100) + require.NoError(t, err) + d2, err := computePreprocessingDelay(time.Hour, 100) + require.NoError(t, err) assert.NotEqual(t, d1, d2) }) @@ -360,7 +366,8 @@ func TestDelay(t *testing.T) { maxDelay := computePreprocessingDelayMax(baseDelay, dkgSize) assert.Equal(t, expectedMaxDelay, maxDelay) - delay := computePreprocessingDelay(baseDelay, dkgSize) + delay, err := computePreprocessingDelay(baseDelay, dkgSize) + require.NoError(t, err) assert.LessOrEqual(t, minDelay, delay) assert.GreaterOrEqual(t, expectedMaxDelay, delay) }) @@ -375,7 +382,8 @@ func TestDelay(t *testing.T) { maxDelay := computePreprocessingDelayMax(baseDelay, dkgSize) assert.Equal(t, expectedMaxDelay, maxDelay) - delay := computePreprocessingDelay(baseDelay, dkgSize) + delay, err := computePreprocessingDelay(baseDelay, dkgSize) + require.NoError(t, err) assert.LessOrEqual(t, minDelay, delay) assert.GreaterOrEqual(t, expectedMaxDelay, delay) }) diff --git a/module/dkg/tunnel.go b/module/dkg/tunnel.go index 934ee8820cb..f5615a1fc5c 100644 --- a/module/dkg/tunnel.go +++ b/module/dkg/tunnel.go @@ -6,13 +6,14 @@ import ( // BrokerTunnel allows the DKG MessagingEngine to relay messages to and from a // loosely-coupled Broker and Controller. The same BrokerTunnel is intended -// to be reused across epochs. +// to be reused across epochs (multiple DKG instances). The BrokerTunnel does +// not internally queue messages, so sends through the tunnel are blocking. type BrokerTunnel struct { MsgChIn chan messages.PrivDKGMessageIn // from network engine to broker MsgChOut chan messages.PrivDKGMessageOut // from broker to network engine } -// NewBrokerTunnel instantiates a new BrokerTunnel +// NewBrokerTunnel instantiates a new BrokerTunnel. func NewBrokerTunnel() *BrokerTunnel { return &BrokerTunnel{ MsgChIn: make(chan messages.PrivDKGMessageIn), @@ -20,14 +21,15 @@ func NewBrokerTunnel() *BrokerTunnel { } } -// SendIn pushes incoming messages in the MsgChIn channel to be received by the -// Broker. +// SendIn pushes incoming messages in the MsgChIn channel to be received by the Broker. +// This is a blocking call (messages are not queued within the tunnel) func (t *BrokerTunnel) SendIn(msg messages.PrivDKGMessageIn) { t.MsgChIn <- msg } -// SendOut pushes outcoing messages in the MsgChOut channel to be received and +// SendOut pushes outbound messages in the MsgChOut channel to be received and // forwarded by the network engine. +// This is a blocking call (messages are not queued within the tunnel) func (t *BrokerTunnel) SendOut(msg messages.PrivDKGMessageOut) { t.MsgChOut <- msg } diff --git a/module/epochs/qc_voter_test.go b/module/epochs/qc_voter_test.go index 71a2fdd3b97..47a54483200 100644 --- a/module/epochs/qc_voter_test.go +++ b/module/epochs/qc_voter_test.go @@ -69,7 +69,7 @@ func (suite *Suite) SetupTest() { suite.counter = rand.Uint64() suite.nodes = unittest.IdentityListFixture(4, unittest.WithRole(flow.RoleCollection)) - suite.me = suite.nodes.Sample(1)[0] + suite.me = suite.nodes[rand.Intn(len(suite.nodes))] suite.local.On("NodeID").Return(func() flow.Identifier { return suite.me.NodeID }) diff --git a/module/events/finalization_actor.go b/module/events/finalization_actor.go new file mode 100644 index 00000000000..7a16e013991 --- /dev/null +++ b/module/events/finalization_actor.go @@ -0,0 +1,76 @@ +package events + +import ( + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/tracker" + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" +) + +// ProcessLatestFinalizedBlock is invoked when a new block is finalized. +// It is possible that blocks will be skipped. +type ProcessLatestFinalizedBlock func(block *model.Block) error + +// FinalizationActor is an event responder worker which can be embedded in a component +// to simplify the plumbing required to respond to block finalization events. +// This worker is designed to respond to a newly finalized blocks on a best-effort basis, +// meaning that it may skip blocks when finalization occurs more quickly. +// CAUTION: This is suitable for use only when the handler can tolerate skipped blocks. +type FinalizationActor struct { + newestFinalized *tracker.NewestBlockTracker + notifier engine.Notifier + handler ProcessLatestFinalizedBlock +} + +var _ hotstuff.FinalizationConsumer = (*FinalizationActor)(nil) + +// NewFinalizationActor creates a new FinalizationActor, and returns the worker routine +// and event consumer required to operate it. +// The caller MUST: +// - start the returned component.ComponentWorker function +// - subscribe the returned FinalizationActor to ProcessLatestFinalizedBlock events +func NewFinalizationActor(handler ProcessLatestFinalizedBlock) (*FinalizationActor, component.ComponentWorker) { + actor := &FinalizationActor{ + newestFinalized: tracker.NewNewestBlockTracker(), + notifier: engine.NewNotifier(), + handler: handler, + } + return actor, actor.workerLogic +} + +// workerLogic is the worker function exposed by the FinalizationActor. It should be +// attached to a ComponentBuilder by the higher-level component. +// It processes each new finalized block by invoking the ProcessLatestFinalizedBlock callback. +func (actor *FinalizationActor) workerLogic(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + doneSignal := ctx.Done() + blockFinalizedSignal := actor.notifier.Channel() + + for { + select { + case <-doneSignal: + return + case <-blockFinalizedSignal: + block := actor.newestFinalized.NewestBlock() + err := actor.handler(block) + if err != nil { + ctx.Throw(err) + return + } + } + } +} + +// OnFinalizedBlock receives block finalization events. It updates the newest finalized +// block tracker and notifies the worker thread. +func (actor *FinalizationActor) OnFinalizedBlock(block *model.Block) { + if actor.newestFinalized.Track(block) { + actor.notifier.Notify() + } +} + +func (actor *FinalizationActor) OnBlockIncorporated(*model.Block) {} +func (actor *FinalizationActor) OnDoubleProposeDetected(*model.Block, *model.Block) {} diff --git a/module/events/finalization_actor_test.go b/module/events/finalization_actor_test.go new file mode 100644 index 00000000000..37bd2239c45 --- /dev/null +++ b/module/events/finalization_actor_test.go @@ -0,0 +1,29 @@ +package events + +import ( + "context" + "testing" + "time" + + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestFinalizationActor_SubscribeDuringConstruction tests that the FinalizationActor +// subscribes to the provided distributor at construction and can subsequently receive notifications. +func TestFinalizationActor_SubscribeDuringConstruction(t *testing.T) { + + // to ensure the actor is subscribed, create and start the worker, then register the callback + done := make(chan struct{}) + actor, worker := NewFinalizationActor(func(_ *model.Block) error { + close(done) + return nil + }) + ctx, cancel := irrecoverable.NewMockSignalerContextWithCancel(t, context.Background()) + defer cancel() + go worker(ctx, func() {}) + actor.OnFinalizedBlock(nil) + + unittest.AssertClosesBefore(t, done, time.Second) +} diff --git a/module/events/finalized_header_cache.go b/module/events/finalized_header_cache.go new file mode 100644 index 00000000000..dd40eba577c --- /dev/null +++ b/module/events/finalized_header_cache.go @@ -0,0 +1,70 @@ +package events + +import ( + "fmt" + "sync/atomic" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/state/protocol" +) + +// FinalizedHeaderCache caches a copy of the most recently finalized block header by +// consuming BlockFinalized events from HotStuff, using a FinalizationActor. +// The constructor returns both the cache and a worker function. +// +// NOTE: The protocol state already guarantees that state.Final().Head() will be cached, however, +// since the protocol state is shared among many components, there may be high contention on its cache. +// The FinalizedHeaderCache can be used in place of state.Final().Head() to avoid read contention with other components. +type FinalizedHeaderCache struct { + state protocol.State + val *atomic.Pointer[flow.Header] + *FinalizationActor // implement hotstuff.FinalizationConsumer +} + +var _ module.FinalizedHeaderCache = (*FinalizedHeaderCache)(nil) +var _ hotstuff.FinalizationConsumer = (*FinalizedHeaderCache)(nil) + +// Get returns the most recently finalized block. +// Guaranteed to be non-nil after construction. +func (cache *FinalizedHeaderCache) Get() *flow.Header { + return cache.val.Load() +} + +// update reads the latest finalized header and updates the cache. +// No errors are expected during normal operation. +func (cache *FinalizedHeaderCache) update() error { + final, err := cache.state.Final().Head() + if err != nil { + return fmt.Errorf("could not retrieve latest finalized header: %w", err) + } + cache.val.Store(final) + return nil +} + +// NewFinalizedHeaderCache returns a new FinalizedHeaderCache and the ComponentWorker function. +// The caller MUST: +// - subscribe the `FinalizedHeaderCache` (first return value) to the `FinalizationDistributor` +// that is distributing the consensus logic's `OnFinalizedBlock` events +// - start the returned ComponentWorker logic (second return value) in a goroutine to maintain the cache. +func NewFinalizedHeaderCache(state protocol.State) (*FinalizedHeaderCache, component.ComponentWorker, error) { + cache := &FinalizedHeaderCache{ + state: state, + val: new(atomic.Pointer[flow.Header]), + } + // initialize the cache with the current finalized header + if err := cache.update(); err != nil { + return nil, nil, fmt.Errorf("could not initialize cache: %w", err) + } + + // create a worker to continuously track the latest finalized header + actor, worker := NewFinalizationActor(func(_ *model.Block) error { + return cache.update() + }) + cache.FinalizationActor = actor + + return cache, worker, nil +} diff --git a/module/events/finalized_header_cache_test.go b/module/events/finalized_header_cache_test.go new file mode 100644 index 00000000000..6b4f7d1bfdc --- /dev/null +++ b/module/events/finalized_header_cache_test.go @@ -0,0 +1,50 @@ +package events + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + protocolmock "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestFinalizedHeaderCache validates that the FinalizedHeaderCache can be constructed +// with an initial value, and updated with events through the FinalizationActor. +func TestFinalizedHeaderCache(t *testing.T) { + final := unittest.BlockHeaderFixture() + + state := protocolmock.NewState(t) + snap := protocolmock.NewSnapshot(t) + state.On("Final").Return(snap) + snap.On("Head").Return( + func() *flow.Header { return final }, + func() error { return nil }) + + cache, worker, err := NewFinalizedHeaderCache(state) + require.NoError(t, err) + + // cache should be initialized + assert.Equal(t, final, cache.Get()) + + ctx, cancel := irrecoverable.NewMockSignalerContextWithCancel(t, context.Background()) + defer cancel() + go worker(ctx, func() {}) + + // change the latest finalized block and mock a BlockFinalized event + final = unittest.BlockHeaderFixture( + unittest.HeaderWithView(final.View+1), + unittest.WithHeaderHeight(final.Height+1)) + cache.OnFinalizedBlock(model.BlockFromFlow(final)) + + // the cache should be updated + assert.Eventually(t, func() bool { + return final.ID() == cache.Get().ID() + }, time.Second, time.Millisecond*10) +} diff --git a/module/executiondatasync/execution_data/cache/cache.go b/module/executiondatasync/execution_data/cache/cache.go new file mode 100644 index 00000000000..bfe497aac82 --- /dev/null +++ b/module/executiondatasync/execution_data/cache/cache.go @@ -0,0 +1,117 @@ +package cache + +import ( + "context" + "fmt" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/mempool" + "github.com/onflow/flow-go/storage" +) + +// ExecutionDataCache is a read-through cache for ExecutionData. +type ExecutionDataCache struct { + backend execution_data.ExecutionDataGetter + + headers storage.Headers + seals storage.Seals + results storage.ExecutionResults + cache mempool.ExecutionData +} + +// NewExecutionDataCache returns a new ExecutionDataCache. +func NewExecutionDataCache( + backend execution_data.ExecutionDataGetter, + headers storage.Headers, + seals storage.Seals, + results storage.ExecutionResults, + cache mempool.ExecutionData, +) *ExecutionDataCache { + return &ExecutionDataCache{ + backend: backend, + + headers: headers, + seals: seals, + results: results, + cache: cache, + } +} + +// ByID returns the execution data for the given ExecutionDataID. +// +// Expected errors during normal operations: +// - BlobNotFoundError if some CID in the blob tree could not be found from the blobstore +// - MalformedDataError if some level of the blob tree cannot be properly deserialized +// - BlobSizeLimitExceededError if some blob in the blob tree exceeds the maximum allowed size +func (c *ExecutionDataCache) ByID(ctx context.Context, executionDataID flow.Identifier) (*execution_data.BlockExecutionDataEntity, error) { + execData, err := c.backend.Get(ctx, executionDataID) + if err != nil { + return nil, err + } + + return execution_data.NewBlockExecutionDataEntity(executionDataID, execData), nil +} + +// ByBlockID returns the execution data for the given block ID. +// +// Expected errors during normal operations: +// - storage.ErrNotFound if a seal or execution result is not available for the block +// - BlobNotFoundError if some CID in the blob tree could not be found from the blobstore +// - MalformedDataError if some level of the blob tree cannot be properly deserialized +// - BlobSizeLimitExceededError if some blob in the blob tree exceeds the maximum allowed size +func (c *ExecutionDataCache) ByBlockID(ctx context.Context, blockID flow.Identifier) (*execution_data.BlockExecutionDataEntity, error) { + if execData, ok := c.cache.ByID(blockID); ok { + return execData, nil + } + + executionDataID, err := c.LookupID(blockID) + if err != nil { + return nil, err + } + + execData, err := c.backend.Get(ctx, executionDataID) + if err != nil { + return nil, err + } + + execDataEntity := execution_data.NewBlockExecutionDataEntity(executionDataID, execData) + + _ = c.cache.Add(execDataEntity) + + return execDataEntity, nil +} + +// ByHeight returns the execution data for the given block height. +// +// Expected errors during normal operations: +// - storage.ErrNotFound if a seal or execution result is not available for the block +// - BlobNotFoundError if some CID in the blob tree could not be found from the blobstore +// - MalformedDataError if some level of the blob tree cannot be properly deserialized +// - BlobSizeLimitExceededError if some blob in the blob tree exceeds the maximum allowed size +func (c *ExecutionDataCache) ByHeight(ctx context.Context, height uint64) (*execution_data.BlockExecutionDataEntity, error) { + blockID, err := c.headers.BlockIDByHeight(height) + if err != nil { + return nil, err + } + + return c.ByBlockID(ctx, blockID) +} + +// LookupID returns the ExecutionDataID for the given block ID. +// +// Expected errors during normal operations: +// - storage.ErrNotFound if a seal or execution result is not available for the block +func (c *ExecutionDataCache) LookupID(blockID flow.Identifier) (flow.Identifier, error) { + seal, err := c.seals.FinalizedSealForBlock(blockID) + if err != nil { + return flow.ZeroID, fmt.Errorf("failed to lookup seal for block %s: %w", blockID, err) + } + + result, err := c.results.ByID(seal.ResultID) + if err != nil { + return flow.ZeroID, fmt.Errorf("failed to lookup execution result for block %s: %w", blockID, err) + } + + return result.ExecutionDataID, nil +} diff --git a/module/executiondatasync/execution_data/downloader.go b/module/executiondatasync/execution_data/downloader.go index d0470428bfe..406b3a15e4b 100644 --- a/module/executiondatasync/execution_data/downloader.go +++ b/module/executiondatasync/execution_data/downloader.go @@ -18,15 +18,11 @@ import ( // Downloader is used to download execution data blobs from the network via a blob service. type Downloader interface { module.ReadyDoneAware - - // Download downloads and returns a Block Execution Data from the network. - // The returned error will be: - // - MalformedDataError if some level of the blob tree cannot be properly deserialized - // - BlobNotFoundError if some CID in the blob tree could not be found from the blob service - // - BlobSizeLimitExceededError if some blob in the blob tree exceeds the maximum allowed size - Download(ctx context.Context, executionDataID flow.Identifier) (*BlockExecutionData, error) + ExecutionDataGetter } +var _ Downloader = (*downloader)(nil) + type downloader struct { blobService network.BlobService maxBlobSize int @@ -35,12 +31,14 @@ type downloader struct { type DownloaderOption func(*downloader) +// WithSerializer configures the serializer for the downloader func WithSerializer(serializer Serializer) DownloaderOption { return func(d *downloader) { d.serializer = serializer } } +// NewDownloader creates a new Downloader instance func NewDownloader(blobService network.BlobService, opts ...DownloaderOption) *downloader { d := &downloader{ blobService, @@ -55,19 +53,23 @@ func NewDownloader(blobService network.BlobService, opts ...DownloaderOption) *d return d } +// Ready returns a channel that will be closed when the downloader is ready to be used func (d *downloader) Ready() <-chan struct{} { return d.blobService.Ready() } + +// Done returns a channel that will be closed when the downloader is finished shutting down func (d *downloader) Done() <-chan struct{} { return d.blobService.Done() } -// Download downloads a blob tree identified by executionDataID from the network and returns the deserialized BlockExecutionData struct -// During normal operation, the returned error will be: -// - MalformedDataError if some level of the blob tree cannot be properly deserialized +// Get downloads a blob tree identified by executionDataID from the network and returns the deserialized BlockExecutionData struct +// +// Expected errors during normal operations: // - BlobNotFoundError if some CID in the blob tree could not be found from the blob service +// - MalformedDataError if some level of the blob tree cannot be properly deserialized // - BlobSizeLimitExceededError if some blob in the blob tree exceeds the maximum allowed size -func (d *downloader) Download(ctx context.Context, executionDataID flow.Identifier) (*BlockExecutionData, error) { +func (d *downloader) Get(ctx context.Context, executionDataID flow.Identifier) (*BlockExecutionData, error) { blobGetter := d.blobService.GetSession(ctx) // First, download the root execution data record which contains a list of chunk execution data @@ -115,6 +117,13 @@ func (d *downloader) Download(ctx context.Context, executionDataID flow.Identifi return bed, nil } +// getExecutionDataRoot downloads the root execution data record from the network and returns the +// deserialized BlockExecutionDataRoot struct. +// +// Expected errors during normal operations: +// - BlobNotFoundError if the root blob could not be found from the blob service +// - MalformedDataError if the root blob cannot be properly deserialized +// - BlobSizeLimitExceededError if the root blob exceeds the maximum allowed size func (d *downloader) getExecutionDataRoot( ctx context.Context, rootID flow.Identifier, @@ -150,6 +159,14 @@ func (d *downloader) getExecutionDataRoot( return edRoot, nil } +// getChunkExecutionData downloads a chunk execution data blob from the network and returns the +// deserialized ChunkExecutionData struct. +// +// Expected errors during normal operations: +// - context.Canceled or context.DeadlineExceeded if the context is canceled or times out +// - BlobNotFoundError if the root blob could not be found from the blob service +// - MalformedDataError if the root blob cannot be properly deserialized +// - BlobSizeLimitExceededError if the root blob exceeds the maximum allowed size func (d *downloader) getChunkExecutionData( ctx context.Context, chunkExecutionDataID cid.Cid, @@ -177,10 +194,21 @@ func (d *downloader) getChunkExecutionData( } // getBlobs gets the given CIDs from the blobservice, reassembles the blobs, and deserializes the reassembled data into an object. +// +// Expected errors during normal operations: +// - context.Canceled or context.DeadlineExceeded if the context is canceled or times out +// - BlobNotFoundError if the root blob could not be found from the blob service +// - MalformedDataError if the root blob cannot be properly deserialized +// - BlobSizeLimitExceededError if the root blob exceeds the maximum allowed size func (d *downloader) getBlobs(ctx context.Context, blobGetter network.BlobGetter, cids []cid.Cid) (interface{}, error) { + // this uses an optimization to deserialize the data in a streaming fashion as it is received + // from the network, reducing the amount of memory required to deserialize large objects. blobCh, errCh := d.retrieveBlobs(ctx, blobGetter, cids) bcr := blobs.NewBlobChannelReader(blobCh) + v, deserializeErr := d.serializer.Deserialize(bcr) + + // blocks until all blobs have been retrieved or an error is encountered err := <-errCh if err != nil { @@ -195,6 +223,13 @@ func (d *downloader) getBlobs(ctx context.Context, blobGetter network.BlobGetter } // retrieveBlobs asynchronously retrieves the blobs for the given CIDs with the given BlobGetter. +// Blobs corresponding to the requested CIDs are returned in order on the response channel. +// +// Expected errors during normal operations: +// - context.Canceled or context.DeadlineExceeded if the context is canceled or times out +// - BlobNotFoundError if the root blob could not be found from the blob service +// - MalformedDataError if the root blob cannot be properly deserialized +// - BlobSizeLimitExceededError if the root blob exceeds the maximum allowed size func (d *downloader) retrieveBlobs(parent context.Context, blobGetter network.BlobGetter, cids []cid.Cid) (<-chan blobs.Blob, <-chan error) { blobsOut := make(chan blobs.Blob, len(cids)) errCh := make(chan error, 1) @@ -214,8 +249,10 @@ func (d *downloader) retrieveBlobs(parent context.Context, blobGetter network.Bl cachedBlobs := make(map[cid.Cid]blobs.Blob) cidCounts := make(map[cid.Cid]int) // used to account for duplicate CIDs + // record the number of times each CID appears in the list. this is later used to determine + // when it's safe to delete cached blobs during processing for _, c := range cids { - cidCounts[c] += 1 + cidCounts[c]++ } // for each cid, find the corresponding blob from the incoming blob channel and send it to @@ -235,7 +272,8 @@ func (d *downloader) retrieveBlobs(parent context.Context, blobGetter network.Bl } } - cidCounts[c] -= 1 + // remove the blob from the cache if it's no longer needed + cidCounts[c]-- if cidCounts[c] == 0 { delete(cachedBlobs, c) @@ -251,12 +289,20 @@ func (d *downloader) retrieveBlobs(parent context.Context, blobGetter network.Bl // findBlob retrieves blobs from the given channel, caching them along the way, until it either // finds the target blob or exhausts the channel. +// +// This is necessary to ensure blobs can be reassembled in order from the underlying blobservice +// which provides no guarantees for blob order on the response channel. +// +// Expected errors during normal operations: +// - BlobNotFoundError if the root blob could not be found from the blob service +// - BlobSizeLimitExceededError if the root blob exceeds the maximum allowed size func (d *downloader) findBlob( blobChan <-chan blobs.Blob, target cid.Cid, cache map[cid.Cid]blobs.Blob, ) (blobs.Blob, error) { - // Note: blobs are returned as they are found, in no particular order + // pull blobs off the blob channel until the target blob is found or the channel is closed + // Note: blobs are returned on the blob channel as they are found, in no particular order for blob := range blobChan { // check blob size blobSize := len(blob.RawData()) diff --git a/module/executiondatasync/execution_data/downloader_test.go b/module/executiondatasync/execution_data/downloader_test.go index e2267e03395..775f4a68107 100644 --- a/module/executiondatasync/execution_data/downloader_test.go +++ b/module/executiondatasync/execution_data/downloader_test.go @@ -23,7 +23,7 @@ func TestCIDNotFound(t *testing.T) { downloader := execution_data.NewDownloader(blobService) edStore := execution_data.NewExecutionDataStore(blobstore, execution_data.DefaultSerializer) bed := generateBlockExecutionData(t, 10, 3*execution_data.DefaultMaxBlobSize) - edID, err := edStore.AddExecutionData(context.Background(), bed) + edID, err := edStore.Add(context.Background(), bed) require.NoError(t, err) blobGetter := new(mocknetwork.BlobGetter) @@ -54,7 +54,7 @@ func TestCIDNotFound(t *testing.T) { }, ) - _, err = downloader.Download(context.Background(), edID) + _, err = downloader.Get(context.Background(), edID) var blobNotFoundError *execution_data.BlobNotFoundError assert.ErrorAs(t, err, &blobNotFoundError) } diff --git a/module/executiondatasync/execution_data/entity.go b/module/executiondatasync/execution_data/entity.go index 85a220100fd..6facd5ad580 100644 --- a/module/executiondatasync/execution_data/entity.go +++ b/module/executiondatasync/execution_data/entity.go @@ -23,10 +23,10 @@ func NewBlockExecutionDataEntity(id flow.Identifier, executionData *BlockExecuti } } -func (c *BlockExecutionDataEntity) ID() flow.Identifier { +func (c BlockExecutionDataEntity) ID() flow.Identifier { return c.id } -func (c *BlockExecutionDataEntity) Checksum() flow.Identifier { +func (c BlockExecutionDataEntity) Checksum() flow.Identifier { return c.id } diff --git a/module/executiondatasync/execution_data/execution_data.go b/module/executiondatasync/execution_data/execution_data.go index 5cdca9c775b..752ea4b9ddb 100644 --- a/module/executiondatasync/execution_data/execution_data.go +++ b/module/executiondatasync/execution_data/execution_data.go @@ -7,6 +7,8 @@ import ( "github.com/onflow/flow-go/model/flow" ) +// DefaultMaxBlobSize is the default maximum size of a blob. +// This is calibrated to fit within a libp2p message and not exceed the max size recommended by bitswap. const DefaultMaxBlobSize = 1 << 20 // 1MiB // ChunkExecutionData represents the execution data of a chunk @@ -16,11 +18,18 @@ type ChunkExecutionData struct { TrieUpdate *ledger.TrieUpdate } +// BlockExecutionDataRoot represents the root of a serialized BlockExecutionData. +// The hash of the serialized BlockExecutionDataRoot is the ExecutionDataID used within an flow.ExecutionResult. type BlockExecutionDataRoot struct { - BlockID flow.Identifier + // BlockID is the ID of the block who's result this execution data is for. + BlockID flow.Identifier + + // ChunkExecutionDataIDs is a list of the root CIDs for each serialized ChunkExecutionData + // associated with this block. ChunkExecutionDataIDs []cid.Cid } +// BlockExecutionData represents the execution data of a block. type BlockExecutionData struct { BlockID flow.Identifier ChunkExecutionDatas []*ChunkExecutionData diff --git a/module/executiondatasync/execution_data/mock/downloader.go b/module/executiondatasync/execution_data/mock/downloader.go index a79dbbe2483..dfeafeeffbe 100644 --- a/module/executiondatasync/execution_data/mock/downloader.go +++ b/module/executiondatasync/execution_data/mock/downloader.go @@ -32,17 +32,17 @@ func (_m *Downloader) Done() <-chan struct{} { return r0 } -// Download provides a mock function with given fields: ctx, executionDataID -func (_m *Downloader) Download(ctx context.Context, executionDataID flow.Identifier) (*execution_data.BlockExecutionData, error) { - ret := _m.Called(ctx, executionDataID) +// Get provides a mock function with given fields: ctx, rootID +func (_m *Downloader) Get(ctx context.Context, rootID flow.Identifier) (*execution_data.BlockExecutionData, error) { + ret := _m.Called(ctx, rootID) var r0 *execution_data.BlockExecutionData var r1 error if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*execution_data.BlockExecutionData, error)); ok { - return rf(ctx, executionDataID) + return rf(ctx, rootID) } if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *execution_data.BlockExecutionData); ok { - r0 = rf(ctx, executionDataID) + r0 = rf(ctx, rootID) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*execution_data.BlockExecutionData) @@ -50,7 +50,7 @@ func (_m *Downloader) Download(ctx context.Context, executionDataID flow.Identif } if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { - r1 = rf(ctx, executionDataID) + r1 = rf(ctx, rootID) } else { r1 = ret.Error(1) } diff --git a/module/executiondatasync/execution_data/mock/execution_data_store.go b/module/executiondatasync/execution_data/mock/execution_data_store.go index f4360871bea..c11c0f1cbce 100644 --- a/module/executiondatasync/execution_data/mock/execution_data_store.go +++ b/module/executiondatasync/execution_data/mock/execution_data_store.go @@ -16,8 +16,8 @@ type ExecutionDataStore struct { mock.Mock } -// AddExecutionData provides a mock function with given fields: ctx, executionData -func (_m *ExecutionDataStore) AddExecutionData(ctx context.Context, executionData *execution_data.BlockExecutionData) (flow.Identifier, error) { +// Add provides a mock function with given fields: ctx, executionData +func (_m *ExecutionDataStore) Add(ctx context.Context, executionData *execution_data.BlockExecutionData) (flow.Identifier, error) { ret := _m.Called(ctx, executionData) var r0 flow.Identifier @@ -42,8 +42,8 @@ func (_m *ExecutionDataStore) AddExecutionData(ctx context.Context, executionDat return r0, r1 } -// GetExecutionData provides a mock function with given fields: ctx, rootID -func (_m *ExecutionDataStore) GetExecutionData(ctx context.Context, rootID flow.Identifier) (*execution_data.BlockExecutionData, error) { +// Get provides a mock function with given fields: ctx, rootID +func (_m *ExecutionDataStore) Get(ctx context.Context, rootID flow.Identifier) (*execution_data.BlockExecutionData, error) { ret := _m.Called(ctx, rootID) var r0 *execution_data.BlockExecutionData diff --git a/module/executiondatasync/execution_data/serializer.go b/module/executiondatasync/execution_data/serializer.go index e47b6d9ed9b..0cc175cb178 100644 --- a/module/executiondatasync/execution_data/serializer.go +++ b/module/executiondatasync/execution_data/serializer.go @@ -14,6 +14,8 @@ import ( "github.com/onflow/flow-go/network/compressor" ) +// DefaultSerializer is the default implementation for an Execution Data serializer. +// It is configured to use cbor encoding with LZ4 compression. var DefaultSerializer Serializer func init() { @@ -33,17 +35,19 @@ func init() { DefaultSerializer = NewSerializer(codec, compressor.NewLz4Compressor()) } -// header codes to distinguish between different types of data -// these codes provide simple versioning of execution state data blobs and indicate how the data -// should be deserialized into their original form. Therefore, each input format must have a unique -// code, and the codes must never be reused. This allows for libraries that can accurately decode -// the data without juggling software versions. +// header codes are used to distinguish between the different data types serialized within a blob. +// they provide simple versioning of execution state data blobs and indicate how the data should +// be deserialized back into their original form. Therefore, each input format must have a unique +// code, and the codes must never be reused. This allows libraries to accurately decode the data +// without juggling software versions. const ( codeRecursiveCIDs = iota + 1 codeExecutionDataRoot codeChunkExecutionData ) +// getCode returns the header code for the given value's type. +// It returns an error if the type is not supported. func getCode(v interface{}) (byte, error) { switch v.(type) { case *BlockExecutionDataRoot: @@ -57,6 +61,8 @@ func getCode(v interface{}) (byte, error) { } } +// getPrototype returns a new instance of the type that corresponds to the given header code. +// It returns an error if the code is not supported. func getPrototype(code byte) (interface{}, error) { switch code { case codeExecutionDataRoot: @@ -73,7 +79,12 @@ func getPrototype(code byte) (interface{}, error) { // Serializer is used to serialize / deserialize Execution Data and CID lists for the // Execution Data Service. type Serializer interface { + // Serialize encodes and compresses the given value to the given writer. + // No errors are expected during normal operation. Serialize(io.Writer, interface{}) error + + // Deserialize decompresses and decodes the data from the given reader. + // No errors are expected during normal operation. Deserialize(io.Reader) (interface{}, error) } @@ -87,6 +98,7 @@ type serializer struct { compressor network.Compressor } +// NewSerializer returns a new Execution Data serializer using the provided encoder and compressor. func NewSerializer(codec encoding.Codec, compressor network.Compressor) *serializer { return &serializer{ codec: codec, @@ -116,7 +128,8 @@ func (s *serializer) writePrototype(w io.Writer, v interface{}) error { return nil } -// Serialize encodes and compresses the given value to the given writer +// Serialize encodes and compresses the given value to the given writer. +// No errors are expected during normal operation. func (s *serializer) Serialize(w io.Writer, v interface{}) error { if err := s.writePrototype(w, v); err != nil { return fmt.Errorf("failed to write prototype: %w", err) @@ -162,7 +175,8 @@ func (s *serializer) readPrototype(r io.Reader) (interface{}, error) { return getPrototype(code) } -// Deserialize decompresses and decodes the data from the given reader +// Deserialize decompresses and decodes the data from the given reader. +// No errors are expected during normal operation. func (s *serializer) Deserialize(r io.Reader) (interface{}, error) { v, err := s.readPrototype(r) diff --git a/module/executiondatasync/execution_data/store.go b/module/executiondatasync/execution_data/store.go index a082a97fe8c..a49796ea8e9 100644 --- a/module/executiondatasync/execution_data/store.go +++ b/module/executiondatasync/execution_data/store.go @@ -12,17 +12,24 @@ import ( "github.com/onflow/flow-go/module/blobs" ) -// ExecutionDataStore handles adding / getting execution data to / from a local blobstore -type ExecutionDataStore interface { - // GetExecutionData gets the BlockExecutionData for the given root ID from the blobstore. - // The returned error will be: - // - MalformedDataError if some level of the blob tree cannot be properly deserialized +// ExecutionDataGetter handles getting execution data from a blobstore +type ExecutionDataGetter interface { + // Get gets the BlockExecutionData for the given root ID from the blobstore. + // Expected errors during normal operations: // - BlobNotFoundError if some CID in the blob tree could not be found from the blobstore - GetExecutionData(ctx context.Context, rootID flow.Identifier) (*BlockExecutionData, error) + // - MalformedDataError if some level of the blob tree cannot be properly deserialized + // - BlobSizeLimitExceededError if some blob in the blob tree exceeds the maximum allowed size + Get(ctx context.Context, rootID flow.Identifier) (*BlockExecutionData, error) +} - // AddExecutionData constructs a blob tree for the given BlockExecutionData and adds it to the - // blobstore, and then returns the root CID. - AddExecutionData(ctx context.Context, executionData *BlockExecutionData) (flow.Identifier, error) +// ExecutionDataStore handles adding / getting execution data to / from a blobstore +type ExecutionDataStore interface { + ExecutionDataGetter + + // Add constructs a blob tree for the given BlockExecutionData, adds it to the blobstore, + // then returns the root CID. + // No errors are expected during normal operation. + Add(ctx context.Context, executionData *BlockExecutionData) (flow.Identifier, error) } type ExecutionDataStoreOption func(*store) @@ -34,6 +41,8 @@ func WithMaxBlobSize(size int) ExecutionDataStoreOption { } } +var _ ExecutionDataStore = (*store)(nil) + type store struct { blobstore blobs.Blobstore serializer Serializer @@ -55,7 +64,10 @@ func NewExecutionDataStore(blobstore blobs.Blobstore, serializer Serializer, opt return s } -func (s *store) AddExecutionData(ctx context.Context, executionData *BlockExecutionData) (flow.Identifier, error) { +// Add constructs a blob tree for the given BlockExecutionData, adds it to the blobstore, +// then returns the rootID. +// No errors are expected during normal operation. +func (s *store) Add(ctx context.Context, executionData *BlockExecutionData) (flow.Identifier, error) { executionDataRoot := &BlockExecutionDataRoot{ BlockID: executionData.BlockID, ChunkExecutionDataIDs: make([]cid.Cid, len(executionData.ChunkExecutionDatas)), @@ -75,6 +87,13 @@ func (s *store) AddExecutionData(ctx context.Context, executionData *BlockExecut return flow.ZeroID, fmt.Errorf("could not serialize execution data root: %w", err) } + // this should never happen unless either: + // - maxBlobSize is set too low + // - an enormous number of chunks are included in the block + // e.g. given a 1MB max size, 32 byte CID and 32 byte blockID: + // 1MB/32byes - 1 = 32767 chunk CIDs + // if the number of chunks in a block ever exceeds this, we will need to update the root blob + // generation to support splitting it up into a tree similar to addChunkExecutionData if buf.Len() > s.maxBlobSize { return flow.ZeroID, errors.New("root blob exceeds blob size limit") } @@ -92,24 +111,38 @@ func (s *store) AddExecutionData(ctx context.Context, executionData *BlockExecut return rootID, nil } +// addChunkExecutionData constructs a blob tree for the given ChunkExecutionData, adds it to the +// blobstore, and returns the root CID. +// No errors are expected during normal operation. func (s *store) addChunkExecutionData(ctx context.Context, chunkExecutionData *ChunkExecutionData) (cid.Cid, error) { var v interface{} = chunkExecutionData + // given an arbitrarily large v, split it into blobs of size up to maxBlobSize, adding them to + // the blobstore. Then, combine the list of CIDs added into a second level of blobs, and repeat. + // This produces a tree of blobs, where the leaves are the actual data, and each internal node + // contains a list of CIDs for its children. for i := 0; ; i++ { + // chunk and store the data, then get the list of CIDs added cids, err := s.addBlobs(ctx, v) if err != nil { return cid.Undef, fmt.Errorf("failed to add blob tree level at height %d: %w", i, err) } + // once a single CID is left, we have reached the root of the tree if len(cids) == 1 { return cids[0], nil } + // the next level is the list of CIDs added in this level v = cids } } +// addBlobs splits the given value into blobs of size up to maxBlobSize, adds them to the blobstore, +// then returns the CIDs for each blob added. +// No errors are expected during normal operation. func (s *store) addBlobs(ctx context.Context, v interface{}) ([]cid.Cid, error) { + // first, serialize the data into a large byte slice buf := new(bytes.Buffer) if err := s.serializer.Serialize(buf, v); err != nil { return nil, fmt.Errorf("could not serialize execution data root: %w", err) @@ -119,6 +152,7 @@ func (s *store) addBlobs(ctx context.Context, v interface{}) ([]cid.Cid, error) var cids []cid.Cid var blbs []blobs.Blob + // next, chunk the data into blobs of size up to maxBlobSize for len(data) > 0 { blobLen := s.maxBlobSize if len(data) < blobLen { @@ -131,6 +165,7 @@ func (s *store) addBlobs(ctx context.Context, v interface{}) ([]cid.Cid, error) cids = append(cids, blob.Cid()) } + // finally, add the blobs to the blobstore and return the list of CIDs if err := s.blobstore.PutMany(ctx, blbs); err != nil { return nil, fmt.Errorf("could not add blobs: %w", err) } @@ -138,9 +173,14 @@ func (s *store) addBlobs(ctx context.Context, v interface{}) ([]cid.Cid, error) return cids, nil } -func (s *store) GetExecutionData(ctx context.Context, rootID flow.Identifier) (*BlockExecutionData, error) { +// Get gets the BlockExecutionData for the given root ID from the blobstore. +// Expected errors during normal operations: +// - BlobNotFoundError if some CID in the blob tree could not be found from the blobstore +// - MalformedDataError if some level of the blob tree cannot be properly deserialized +func (s *store) Get(ctx context.Context, rootID flow.Identifier) (*BlockExecutionData, error) { rootCid := flow.IdToCid(rootID) + // first, get the root blob. it will contain a list of blobs, one for each chunk rootBlob, err := s.blobstore.Get(ctx, rootCid) if err != nil { if errors.Is(err, blobs.ErrNotFound) { @@ -160,6 +200,7 @@ func (s *store) GetExecutionData(ctx context.Context, rootID flow.Identifier) (* return nil, NewMalformedDataError(fmt.Errorf("root blob does not deserialize to a BlockExecutionDataRoot, got %T instead", rootData)) } + // next, get each chunk blob and deserialize it blockExecutionData := &BlockExecutionData{ BlockID: executionDataRoot.BlockID, ChunkExecutionDatas: make([]*ChunkExecutionData, len(executionDataRoot.ChunkExecutionDataIDs)), @@ -177,9 +218,14 @@ func (s *store) GetExecutionData(ctx context.Context, rootID flow.Identifier) (* return blockExecutionData, nil } +// getChunkExecutionData gets the ChunkExecutionData for the given CID from the blobstore. +// Expected errors during normal operations: +// - BlobNotFoundError if some CID in the blob tree could not be found from the blobstore +// - MalformedDataError if some level of the blob tree cannot be properly deserialized func (s *store) getChunkExecutionData(ctx context.Context, chunkExecutionDataID cid.Cid) (*ChunkExecutionData, error) { cids := []cid.Cid{chunkExecutionDataID} + // given a root CID, get the blob tree level by level, until we reach the full ChunkExecutionData for i := 0; ; i++ { v, err := s.getBlobs(ctx, cids) if err != nil { @@ -197,9 +243,14 @@ func (s *store) getChunkExecutionData(ctx context.Context, chunkExecutionDataID } } +// getBlobs gets the blobs for the given CIDs from the blobstore, deserializes them, and returns +// the deserialized value. +// - BlobNotFoundError if any of the CIDs could not be found from the blobstore +// - MalformedDataError if any of the blobs cannot be properly deserialized func (s *store) getBlobs(ctx context.Context, cids []cid.Cid) (interface{}, error) { buf := new(bytes.Buffer) + // get each blob and append the raw data to the buffer for _, cid := range cids { blob, err := s.blobstore.Get(ctx, cid) if err != nil { @@ -216,6 +267,7 @@ func (s *store) getBlobs(ctx context.Context, cids []cid.Cid) (interface{}, erro } } + // deserialize the buffer into a value, and return it v, err := s.serializer.Deserialize(buf) if err != nil { return nil, NewMalformedDataError(err) diff --git a/module/executiondatasync/execution_data/store_test.go b/module/executiondatasync/execution_data/store_test.go index 39d00d93044..f1784201766 100644 --- a/module/executiondatasync/execution_data/store_test.go +++ b/module/executiondatasync/execution_data/store_test.go @@ -3,9 +3,10 @@ package execution_data_test import ( "bytes" "context" + "crypto/rand" "fmt" "io" - "math/rand" + mrand "math/rand" "testing" "github.com/ipfs/go-cid" @@ -103,9 +104,9 @@ func TestHappyPath(t *testing.T) { test := func(numChunks int, minSerializedSizePerChunk uint64) { expected := generateBlockExecutionData(t, numChunks, minSerializedSizePerChunk) - rootId, err := eds.AddExecutionData(context.Background(), expected) + rootId, err := eds.Add(context.Background(), expected) require.NoError(t, err) - actual, err := eds.GetExecutionData(context.Background(), rootId) + actual, err := eds.Get(context.Background(), rootId) require.NoError(t, err) deepEqual(t, expected, actual) } @@ -134,7 +135,7 @@ type corruptedTailSerializer struct { func newCorruptedTailSerializer(numChunks int) *corruptedTailSerializer { return &corruptedTailSerializer{ - corruptedChunk: rand.Intn(numChunks) + 1, + corruptedChunk: mrand.Intn(numChunks) + 1, } } @@ -171,9 +172,9 @@ func TestMalformedData(t *testing.T) { blobstore := getBlobstore() defaultEds := getExecutionDataStore(blobstore, execution_data.DefaultSerializer) malformedEds := getExecutionDataStore(blobstore, serializer) - rootID, err := malformedEds.AddExecutionData(context.Background(), bed) + rootID, err := malformedEds.Add(context.Background(), bed) require.NoError(t, err) - _, err = defaultEds.GetExecutionData(context.Background(), rootID) + _, err = defaultEds.Get(context.Background(), rootID) assert.True(t, execution_data.IsMalformedDataError(err)) } @@ -191,16 +192,16 @@ func TestGetIncompleteData(t *testing.T) { eds := getExecutionDataStore(blobstore, execution_data.DefaultSerializer) bed := generateBlockExecutionData(t, 5, 10*execution_data.DefaultMaxBlobSize) - rootID, err := eds.AddExecutionData(context.Background(), bed) + rootID, err := eds.Add(context.Background(), bed) require.NoError(t, err) cids := getAllKeys(t, blobstore) t.Logf("%d blobs in blob tree", len(cids)) - cidToDelete := cids[rand.Intn(len(cids))] + cidToDelete := cids[mrand.Intn(len(cids))] require.NoError(t, blobstore.DeleteBlob(context.Background(), cidToDelete)) - _, err = eds.GetExecutionData(context.Background(), rootID) + _, err = eds.Get(context.Background(), rootID) var blobNotFoundError *execution_data.BlobNotFoundError assert.ErrorAs(t, err, &blobNotFoundError) } diff --git a/module/executiondatasync/execution_data/util.go b/module/executiondatasync/execution_data/util.go index f2585f4c61f..4180904639f 100644 --- a/module/executiondatasync/execution_data/util.go +++ b/module/executiondatasync/execution_data/util.go @@ -7,10 +7,12 @@ import ( "github.com/onflow/flow-go/module/blobs" ) +// CalculateID calculates the root ID of the given execution data without storing any data. +// No errors are expected during normal operation. func CalculateID(ctx context.Context, execData *BlockExecutionData, serializer Serializer) (flow.Identifier, error) { executionDatastore := NewExecutionDataStore(&blobs.NoopBlobstore{}, serializer) - id, err := executionDatastore.AddExecutionData(ctx, execData) + id, err := executionDatastore.Add(ctx, execData) if err != nil { return flow.ZeroID, err } diff --git a/module/executiondatasync/provider/provider_test.go b/module/executiondatasync/provider/provider_test.go index 3ebd216767b..b8d90a68433 100644 --- a/module/executiondatasync/provider/provider_test.go +++ b/module/executiondatasync/provider/provider_test.go @@ -118,7 +118,7 @@ func TestHappyPath(t *testing.T) { expected := generateBlockExecutionData(t, numChunks, minSerializedSizePerChunk) executionDataID, err := provider.Provide(context.Background(), 0, expected) require.NoError(t, err) - actual, err := store.GetExecutionData(context.Background(), executionDataID) + actual, err := store.Get(context.Background(), executionDataID) require.NoError(t, err) deepEqual(t, expected, actual) } diff --git a/module/finalizer/collection/finalizer_test.go b/module/finalizer/collection/finalizer_test.go index 921e8cc6c57..fa92d3eeafe 100644 --- a/module/finalizer/collection/finalizer_test.go +++ b/module/finalizer/collection/finalizer_test.go @@ -1,9 +1,7 @@ package collection_test import ( - "math/rand" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -25,10 +23,6 @@ import ( func TestFinalizer(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { - - // seed the RNG - rand.Seed(time.Now().UnixNano()) - // reference block on the main consensus chain refBlock := unittest.BlockHeaderFixture() // genesis block for the cluster chain @@ -53,7 +47,7 @@ func TestFinalizer(t *testing.T) { // a helper function to bootstrap with the genesis block bootstrap := func() { - stateRoot, err := cluster.NewStateRoot(genesis, unittest.QuorumCertificateFixture()) + stateRoot, err := cluster.NewStateRoot(genesis, unittest.QuorumCertificateFixture(), 0) require.NoError(t, err) state, err = cluster.Bootstrap(db, stateRoot) require.NoError(t, err) diff --git a/module/forest/leveled_forest.go b/module/forest/leveled_forest.go index 970cff8a07f..9dd65d47543 100644 --- a/module/forest/leveled_forest.go +++ b/module/forest/leveled_forest.go @@ -196,11 +196,17 @@ func (f *LevelledForest) AddVertex(vertex Vertex) { f.size += 1 } +// registerWithParent retrieves the parent and registers the given vertex as a child. +// For a block, whose level equal to the pruning threshold, we do not inspect the parent at all. +// Thereby, this implementation can gracefully handle the corner case where the tree has a defined +// end vertex (distinct root). This is commonly the case in blockchain (genesis, or spork root block). +// Mathematically, this means that this library can also represent bounded trees. func (f *LevelledForest) registerWithParent(vertexContainer *vertexContainer) { - // caution: do not modify this combination of check (a) and (a) - // Deliberate handling of root vertex (genesis block) whose view is _exactly_ at LowestLevel - // For this block, we don't care about its parent and the exception is allowed where - // vertex.level = vertex.Parent().Level = LowestLevel = 0 + // caution, necessary for handling bounded trees: + // For root vertex (genesis block) the view is _exactly_ at LowestLevel. For these blocks, + // a parent does not exist. In the implementation, we deliberately do not call the `Parent()` method, + // as its output is conceptually undefined. Thereby, we can gracefully handle the corner case of + // vertex.level = vertex.Parent().Level = LowestLevel = 0 if vertexContainer.level <= f.LowestLevel { // check (a) return } diff --git a/module/grpcserver/server.go b/module/grpcserver/server.go new file mode 100644 index 00000000000..309cb9315f2 --- /dev/null +++ b/module/grpcserver/server.go @@ -0,0 +1,88 @@ +package grpcserver + +import ( + "net" + "sync" + + "github.com/rs/zerolog" + + "google.golang.org/grpc" + + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" +) + +// GrpcServer wraps `grpc.Server` and allows to manage it using `component.Component` interface. It can be injected +// into different engines making it possible to use single grpc server for multiple services which live in different modules. +type GrpcServer struct { + component.Component + log zerolog.Logger + Server *grpc.Server + + grpcListenAddr string // the GRPC server address as ip:port + + addrLock sync.RWMutex + grpcAddress net.Addr +} + +var _ component.Component = (*GrpcServer)(nil) + +// NewGrpcServer returns a new grpc server. +func NewGrpcServer(log zerolog.Logger, + grpcListenAddr string, + grpcServer *grpc.Server, +) *GrpcServer { + server := &GrpcServer{ + log: log, + Server: grpcServer, + grpcListenAddr: grpcListenAddr, + } + server.Component = component.NewComponentManagerBuilder(). + AddWorker(server.serveGRPCWorker). + AddWorker(server.shutdownWorker). + Build() + return server +} + +// serveGRPCWorker is a worker routine which starts the gRPC server. +// The ready callback is called after the server address is bound and set. +func (g *GrpcServer) serveGRPCWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + g.log = g.log.With().Str("grpc_address", g.grpcListenAddr).Logger() + g.log.Info().Msg("starting grpc server on address") + + l, err := net.Listen("tcp", g.grpcListenAddr) + if err != nil { + g.log.Err(err).Msg("failed to start the grpc server") + ctx.Throw(err) + return + } + + // save the actual address on which we are listening (may be different from g.config.GRPCListenAddr if not port + // was specified) + g.addrLock.Lock() + g.grpcAddress = l.Addr() + g.addrLock.Unlock() + g.log.Debug().Msg("listening on port") + ready() + + err = g.Server.Serve(l) // blocking call + if err != nil { + g.log.Err(err).Msg("fatal error in grpc server") + ctx.Throw(err) + } +} + +// GRPCAddress returns the listen address of the GRPC server. +// Guaranteed to be non-nil after Engine.Ready is closed. +func (g *GrpcServer) GRPCAddress() net.Addr { + g.addrLock.RLock() + defer g.addrLock.RUnlock() + return g.grpcAddress +} + +// shutdownWorker is a worker routine which shuts down server when the context is cancelled. +func (g *GrpcServer) shutdownWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + <-ctx.Done() + g.Server.GracefulStop() +} diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go new file mode 100644 index 00000000000..d42196cdf12 --- /dev/null +++ b/module/grpcserver/server_builder.go @@ -0,0 +1,107 @@ +package grpcserver + +import ( + "github.com/rs/zerolog" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + + "github.com/onflow/flow-go/engine/common/rpc" +) + +type Option func(*GrpcServerBuilder) + +// WithTransportCredentials sets the transport credentials parameters for a grpc server builder. +func WithTransportCredentials(transportCredentials credentials.TransportCredentials) Option { + return func(c *GrpcServerBuilder) { + c.transportCredentials = transportCredentials + } +} + +// WithStreamInterceptor sets the StreamInterceptor option to grpc server. +func WithStreamInterceptor() Option { + return func(c *GrpcServerBuilder) { + c.stateStreamInterceptorEnable = true + } +} + +// GrpcServerBuilder created for separating the creation and starting GrpcServer, +// cause services need to be registered before the server starts. +type GrpcServerBuilder struct { + log zerolog.Logger + gRPCListenAddr string + server *grpc.Server + + transportCredentials credentials.TransportCredentials // the GRPC credentials + stateStreamInterceptorEnable bool +} + +// NewGrpcServerBuilder helps to build a new grpc server. +func NewGrpcServerBuilder(log zerolog.Logger, + gRPCListenAddr string, + maxMsgSize uint, + rpcMetricsEnabled bool, + apiRateLimits map[string]int, // the api rate limit (max calls per second) for each of the Access API e.g. Ping->100, GetTransaction->300 + apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the Access API e.g. Ping->50, GetTransaction->10 + opts ...Option, +) *GrpcServerBuilder { + log = log.With().Str("component", "grpc_server").Logger() + + grpcServerBuilder := &GrpcServerBuilder{ + gRPCListenAddr: gRPCListenAddr, + } + + for _, applyOption := range opts { + applyOption(grpcServerBuilder) + } + + // create a GRPC server to serve GRPC clients + grpcOpts := []grpc.ServerOption{ + grpc.MaxRecvMsgSize(int(maxMsgSize)), + grpc.MaxSendMsgSize(int(maxMsgSize)), + } + var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors + // if rpc metrics is enabled, first create the grpc metrics interceptor + if rpcMetricsEnabled { + interceptors = append(interceptors, grpc_prometheus.UnaryServerInterceptor) + + if grpcServerBuilder.stateStreamInterceptorEnable { + // note: intentionally not adding logging or rate limit interceptors for streams. + // rate limiting is done in the handler, and we don't need log events for every message as + // that would be too noisy. + log.Info().Msg("stateStreamInterceptorEnable true") + grpcOpts = append(grpcOpts, grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor)) + } else { + log.Info().Msg("stateStreamInterceptorEnable false") + } + } + if len(apiRateLimits) > 0 { + // create a rate limit interceptor + rateLimitInterceptor := rpc.NewRateLimiterInterceptor(log, apiRateLimits, apiBurstLimits).UnaryServerInterceptor + // append the rate limit interceptor to the list of interceptors + interceptors = append(interceptors, rateLimitInterceptor) + } + // add the logging interceptor, ensure it is innermost wrapper + interceptors = append(interceptors, rpc.LoggingInterceptor(log)...) + // create a chained unary interceptor + // create an unsecured grpc server + grpcOpts = append(grpcOpts, grpc.ChainUnaryInterceptor(interceptors...)) + + if grpcServerBuilder.transportCredentials != nil { + log = log.With().Str("endpoint", "secure").Logger() + // create a secure server by using the secure grpc credentials that are passed in as part of config + grpcOpts = append(grpcOpts, grpc.Creds(grpcServerBuilder.transportCredentials)) + } else { + log = log.With().Str("endpoint", "unsecure").Logger() + } + grpcServerBuilder.log = log + grpcServerBuilder.server = grpc.NewServer(grpcOpts...) + + return grpcServerBuilder +} + +func (b *GrpcServerBuilder) Build() *GrpcServer { + return NewGrpcServer(b.log, b.gRPCListenAddr, b.server) +} diff --git a/module/hotstuff.go b/module/hotstuff.go index 47a7f758b6a..8610ce0bce1 100644 --- a/module/hotstuff.go +++ b/module/hotstuff.go @@ -4,9 +4,15 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" ) -// HotStuff defines the interface to the core HotStuff algorithm. It includes +// HotStuff defines the interface for the core HotStuff algorithm. It includes // a method to start the event loop, and utilities to submit block proposals // received from other replicas. +// +// TODO: +// +// HotStuff interface could extend HotStuffFollower. Thereby, we can +// utilize the optimized catchup mode from the follower also for the +// consensus participant. type HotStuff interface { ReadyDoneAware Startable @@ -21,32 +27,49 @@ type HotStuff interface { // HotStuffFollower is run by non-consensus nodes to observe the block chain // and make local determination about block finalization. While the process of -// reaching consensus (while guaranteeing its safety and liveness) is very intricate, +// reaching consensus (incl. guaranteeing its safety and liveness) is very intricate, // the criteria to confirm that consensus has been reached are relatively straight // forward. Each non-consensus node can simply observe the blockchain and determine // locally which blocks have been finalized without requiring additional information // from the consensus nodes. // -// Specifically, the HotStuffFollower informs other components within the node -// about finalization of blocks. It consumes block proposals broadcasted -// by the consensus node, verifies the block header and locally evaluates -// the finalization rules. +// In contrast to an active HotStuff participant, the HotStuffFollower does not validate +// block payloads. This greatly reduces the amount of CPU and memory that it consumes. +// Essentially, the consensus participants exhaustively verify the entire block including +// the payload and only vote for the block if it is valid. The consensus committee +// aggregates votes from a supermajority of participants to a Quorum Certificate [QC]. +// Thereby, it is guaranteed that only valid blocks get certified (receive a QC). +// By only consuming certified blocks, the HotStuffFollower can be sure of their +// correctness and omit the heavy payload verification. +// There is no disbenefit for nodes to wait for a QC (included in child blocks), because +// all nodes other than consensus generally require the Source Of Randomness included in +// QCs to process the block in the first place. +// +// The central purpose of the HotStuffFollower is to inform other components within the +// node about finalization of blocks. // // Notes: -// - HotStuffFollower does not handle disconnected blocks. Each block's parent must -// have been previously processed by the HotStuffFollower. // - HotStuffFollower internally prunes blocks below the last finalized view. -// When receiving a block proposal, it might not have the proposal's parent anymore. -// Nevertheless, HotStuffFollower needs the parent's view, which must be supplied -// in addition to the proposal. +// - HotStuffFollower does not handle disconnected blocks. For each input block, +// we require that the parent was previously added (unless the parent's view +// is _below_ the latest finalized view). type HotStuffFollower interface { ReadyDoneAware Startable - // SubmitProposal feeds a new block proposal into the HotStuffFollower. - // This method blocks until the proposal is accepted to the event queue. + // AddCertifiedBlock appends the given certified block to the tree of pending + // blocks and updates the latest finalized block (if finalization progressed). + // Unless the parent is below the pruning threshold (latest finalized view), we + // require that the parent has previously been added. // - // Block proposals must be submitted in order, i.e. a proposal's parent must - // have been previously processed by the HotStuffFollower. - SubmitProposal(proposal *model.Proposal) + // Notes: + // - Under normal operations, this method is non-blocking. The follower internally + // queues incoming blocks and processes them in its own worker routine. However, + // when the inbound queue is full, we block until there is space in the queue. This + // behaviour is intentional, because we cannot drop blocks (otherwise, we would + // cause disconnected blocks). Instead we simply block the compliance layer to + // avoid any pathological edge cases. + // - Blocks whose views are below the latest finalized view are dropped. + // - Inputs are idempotent (repetitions are no-ops). + AddCertifiedBlock(certifiedBlock *model.CertifiedBlock) } diff --git a/module/irrecoverable/unittest.go b/module/irrecoverable/unittest.go index 16ab422ffd2..b3e252c905b 100644 --- a/module/irrecoverable/unittest.go +++ b/module/irrecoverable/unittest.go @@ -5,6 +5,7 @@ import ( "testing" ) +// MockSignalerContext is a SignalerContext which will immediately fail a test if an error is thrown. type MockSignalerContext struct { context.Context t *testing.T @@ -24,3 +25,8 @@ func NewMockSignalerContext(t *testing.T, ctx context.Context) *MockSignalerCont t: t, } } + +func NewMockSignalerContextWithCancel(t *testing.T, parent context.Context) (*MockSignalerContext, context.CancelFunc) { + ctx, cancel := context.WithCancel(parent) + return NewMockSignalerContext(t, ctx), cancel +} diff --git a/module/jobqueue/finalized_block_reader_test.go b/module/jobqueue/finalized_block_reader_test.go index 41c5f403b97..8349828d272 100644 --- a/module/jobqueue/finalized_block_reader_test.go +++ b/module/jobqueue/finalized_block_reader_test.go @@ -62,10 +62,11 @@ func withReader( // blocks (i.e., containing guarantees), and Cs are container blocks for their preceding reference block, // Container blocks only contain receipts of their preceding reference blocks. But they do not // hold any guarantees. - root, err := s.State.Params().Root() + root, err := s.State.Params().FinalizedRoot() require.NoError(t, err) clusterCommittee := participants.Filter(filter.HasRole(flow.RoleCollection)) - results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, vertestutils.WithClusterCommittee(clusterCommittee)) + sources := unittest.RandomSourcesFixture(10) + results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, sources, vertestutils.WithClusterCommittee(clusterCommittee)) blocks := vertestutils.ExtendStateWithFinalizedBlocks(t, results, s.State) withBlockReader(reader, blocks) diff --git a/module/mempool/entity/executableblock.go b/module/mempool/entity/executableblock.go index 29300f44aef..3c80e801d3c 100644 --- a/module/mempool/entity/executableblock.go +++ b/module/mempool/entity/executableblock.go @@ -86,15 +86,25 @@ func (b *ExecutableBlock) Collections() []*CompleteCollection { return collections } -// CollectionAt returns an address to a collection at the given index, +// CompleteCollectionAt returns a complete collection at the given index, // if index out of range, nil will be returned -func (b *ExecutableBlock) CollectionAt(index int) *CompleteCollection { - if index < 0 && index > len(b.Block.Payload.Guarantees) { +func (b *ExecutableBlock) CompleteCollectionAt(index int) *CompleteCollection { + if index < 0 || index >= len(b.Block.Payload.Guarantees) { return nil } return b.CompleteCollections[b.Block.Payload.Guarantees[index].ID()] } +// CollectionAt returns a collection at the given index, +// if index out of range, nil will be returned +func (b *ExecutableBlock) CollectionAt(index int) *flow.Collection { + cc := b.CompleteCollectionAt(index) + if cc == nil { + return nil + } + return &flow.Collection{Transactions: cc.Transactions} +} + // HasAllTransactions returns whether all the transactions for all collections // in the block have been received. func (b *ExecutableBlock) HasAllTransactions() bool { diff --git a/module/mempool/execution_data.go b/module/mempool/execution_data.go new file mode 100644 index 00000000000..88d466c146b --- /dev/null +++ b/module/mempool/execution_data.go @@ -0,0 +1,36 @@ +package mempool + +import ( + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" +) + +// ExecutionData represents a concurrency-safe memory pool for BlockExecutionData. +type ExecutionData interface { + + // Has checks whether the block execution data for the given block ID is currently in + // the memory pool. + Has(flow.Identifier) bool + + // Add adds a block execution data to the mempool, keyed by block ID. + // It returns false if the execution data was already in the mempool. + Add(*execution_data.BlockExecutionDataEntity) bool + + // Remove removes block execution data from mempool by block ID. + // It returns true if the execution data was known and removed. + Remove(flow.Identifier) bool + + // ByID returns the block execution data for the given block ID from the mempool. + // It returns false if the execution data was not found in the mempool. + ByID(flow.Identifier) (*execution_data.BlockExecutionDataEntity, bool) + + // Size return the current size of the memory pool. + Size() uint + + // All retrieves all execution data that are currently in the memory pool + // as a slice. + All() []*execution_data.BlockExecutionDataEntity + + // Clear removes all execution data from the mempool. + Clear() +} diff --git a/module/mempool/herocache/backdata/cache.go b/module/mempool/herocache/backdata/cache.go index 1c7956fd578..2ac93e38957 100644 --- a/module/mempool/herocache/backdata/cache.go +++ b/module/mempool/herocache/backdata/cache.go @@ -118,6 +118,11 @@ func NewCache(sizeLimit uint32, // total buckets. capacity := uint64(sizeLimit * oversizeFactor) bucketNum := capacity / slotsPerBucket + if bucketNum == 0 { + // we panic here because we don't want to continue with a zero bucketNum (it can cause a DoS attack). + panic("bucketNum cannot be zero, choose a bigger sizeLimit or a smaller oversizeFactor") + } + if capacity%slotsPerBucket != 0 { // accounting for remainder. bucketNum++ @@ -130,7 +135,7 @@ func NewCache(sizeLimit uint32, sizeLimit: sizeLimit, buckets: make([]slotBucket, bucketNum), ejectionMode: ejectionMode, - entities: heropool.NewHeroPool(sizeLimit, ejectionMode), + entities: heropool.NewHeroPool(sizeLimit, ejectionMode, logger), availableSlotHistogram: make([]uint64, slotsPerBucket+1), // +1 is to account for empty buckets as well. interactionCounter: atomic.NewUint64(0), lastTelemetryDump: atomic.NewInt64(0), @@ -205,7 +210,7 @@ func (c *Cache) ByID(entityID flow.Identifier) (flow.Entity, bool) { } // Size returns the size of the backdata, i.e., total number of stored (entityId, entity) pairs. -func (c Cache) Size() uint { +func (c *Cache) Size() uint { defer c.logTelemetry() return uint(c.entities.Size()) @@ -213,12 +218,12 @@ func (c Cache) Size() uint { // Head returns the head of queue. // Boolean return value determines whether there is a head available. -func (c Cache) Head() (flow.Entity, bool) { +func (c *Cache) Head() (flow.Entity, bool) { return c.entities.Head() } // All returns all entities stored in the backdata. -func (c Cache) All() map[flow.Identifier]flow.Entity { +func (c *Cache) All() map[flow.Identifier]flow.Entity { defer c.logTelemetry() entitiesList := c.entities.All() @@ -234,7 +239,7 @@ func (c Cache) All() map[flow.Identifier]flow.Entity { } // Identifiers returns the list of identifiers of entities stored in the backdata. -func (c Cache) Identifiers() flow.IdentifierList { +func (c *Cache) Identifiers() flow.IdentifierList { defer c.logTelemetry() ids := make(flow.IdentifierList, c.entities.Size()) @@ -246,7 +251,7 @@ func (c Cache) Identifiers() flow.IdentifierList { } // Entities returns the list of entities stored in the backdata. -func (c Cache) Entities() []flow.Entity { +func (c *Cache) Entities() []flow.Entity { defer c.logTelemetry() entities := make([]flow.Entity, c.entities.Size()) @@ -262,7 +267,7 @@ func (c *Cache) Clear() { defer c.logTelemetry() c.buckets = make([]slotBucket, c.bucketNum) - c.entities = heropool.NewHeroPool(c.sizeLimit, c.ejectionMode) + c.entities = heropool.NewHeroPool(c.sizeLimit, c.ejectionMode, c.logger) c.availableSlotHistogram = make([]uint64, slotsPerBucket+1) c.interactionCounter = atomic.NewUint64(0) c.lastTelemetryDump = atomic.NewInt64(0) @@ -350,7 +355,7 @@ func (c *Cache) get(entityID flow.Identifier) (flow.Entity, bucketIndex, slotInd // entityId32of256AndBucketIndex determines the id prefix as well as the bucket index corresponding to the // given identifier. -func (c Cache) entityId32of256AndBucketIndex(id flow.Identifier) (sha32of256, bucketIndex) { +func (c *Cache) entityId32of256AndBucketIndex(id flow.Identifier) (sha32of256, bucketIndex) { // uint64(id[0:8]) used to compute bucket index for which this identifier belongs to b := binary.LittleEndian.Uint64(id[0:8]) % c.bucketNum @@ -361,7 +366,7 @@ func (c Cache) entityId32of256AndBucketIndex(id flow.Identifier) (sha32of256, bu } // expiryThreshold returns the threshold for which all slots with index below threshold are considered old enough for eviction. -func (c Cache) expiryThreshold() uint64 { +func (c *Cache) expiryThreshold() uint64 { var expiryThreshold uint64 = 0 if c.slotCount > uint64(c.sizeLimit) { // total number of slots written are above the predefined limit @@ -425,7 +430,7 @@ func (c *Cache) slotIndexInBucket(b bucketIndex, slotId sha32of256, entityId flo // ownerIndexOf maps the (bucketIndex, slotIndex) pair to a canonical unique (scalar) index. // This scalar index is used to represent this (bucketIndex, slotIndex) pair in the underlying // entities list. -func (c Cache) ownerIndexOf(b bucketIndex, s slotIndex) uint64 { +func (c *Cache) ownerIndexOf(b bucketIndex, s slotIndex) uint64 { return (uint64(b) * slotsPerBucket) + uint64(s) } diff --git a/module/mempool/herocache/backdata/cache_test.go b/module/mempool/herocache/backdata/cache_test.go index 7c9864786d8..bf1d7b3c60e 100644 --- a/module/mempool/herocache/backdata/cache_test.go +++ b/module/mempool/herocache/backdata/cache_test.go @@ -17,7 +17,7 @@ import ( // TestArrayBackData_SingleBucket evaluates health of state transition for storing 10 entities in a Cache with only // a single bucket (of 16). It also evaluates all stored items are retrievable. func TestArrayBackData_SingleBucket(t *testing.T) { - limit := 10 + limit := 16 bd := NewCache(uint32(limit), 1, diff --git a/module/mempool/herocache/backdata/heropool/linkedlist.go b/module/mempool/herocache/backdata/heropool/linkedlist.go index 4c72931c630..99823070d69 100644 --- a/module/mempool/herocache/backdata/heropool/linkedlist.go +++ b/module/mempool/herocache/backdata/heropool/linkedlist.go @@ -2,13 +2,16 @@ package heropool // link represents a slice-based doubly linked-list node that // consists of a next and previous poolIndex. +// if a link doesn't belong to any state it's next and prev should hold InvalidIndex. type link struct { - next poolIndex - prev poolIndex + next EIndex + prev EIndex } // state represents a doubly linked-list by its head and tail pool indices. +// If state has 0 size, its tail's and head's prev and next are treated as invalid and should hold InvalidIndex values. type state struct { - head poolIndex - tail poolIndex + head EIndex + tail EIndex + size uint32 } diff --git a/module/mempool/herocache/backdata/heropool/pool.go b/module/mempool/herocache/backdata/heropool/pool.go index 39dabcef07c..87e21abb33e 100644 --- a/module/mempool/herocache/backdata/heropool/pool.go +++ b/module/mempool/herocache/backdata/heropool/pool.go @@ -1,9 +1,13 @@ package heropool import ( - "math/rand" + "fmt" + "math" + + "github.com/rs/zerolog" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/rand" ) type EjectionMode string @@ -17,6 +21,18 @@ const ( // EIndex is data type representing an entity index in Pool. type EIndex uint32 +// InvalidIndex is used when a link doesnt point anywhere, in other words it is an equivalent of a nil address. +const InvalidIndex EIndex = math.MaxUint32 + +// A type dedicated to describe possible states of placeholders for entities in the pool. +type StateType string + +// A placeholder in a free state can be used to store an entity. +const stateFree StateType = "free-state" + +// A placeholder in a used state stores currently an entity. +const stateUsed StateType = "used-state" + // poolEntity represents the data type that is maintained by type poolEntity struct { PoolEntity @@ -47,42 +63,50 @@ func (p PoolEntity) Entity() flow.Entity { } type Pool struct { - size uint32 + logger zerolog.Logger free state // keeps track of free slots. used state // keeps track of allocated slots to cachedEntities. poolEntities []poolEntity ejectionMode EjectionMode } -func NewHeroPool(sizeLimit uint32, ejectionMode EjectionMode) *Pool { +// NewHeroPool returns a pointer to a new hero pool constructed based on a provided EjectionMode, +// logger and a provided fixed size. +func NewHeroPool(sizeLimit uint32, ejectionMode EjectionMode, logger zerolog.Logger) *Pool { l := &Pool{ free: state{ - head: poolIndex{index: 0}, - tail: poolIndex{index: 0}, + head: InvalidIndex, + tail: InvalidIndex, + size: 0, }, used: state{ - head: poolIndex{index: 0}, - tail: poolIndex{index: 0}, + head: InvalidIndex, + tail: InvalidIndex, + size: 0, }, poolEntities: make([]poolEntity, sizeLimit), ejectionMode: ejectionMode, + logger: logger, } + l.setDefaultNodeLinkValues() l.initFreeEntities() return l } +// setDefaultNodeLinkValues sets nodes prev and next to InvalidIndex for all cached entities in poolEntities. +func (p *Pool) setDefaultNodeLinkValues() { + for i := 0; i < len(p.poolEntities); i++ { + p.poolEntities[i].node.next = InvalidIndex + p.poolEntities[i].node.prev = InvalidIndex + } +} + // initFreeEntities initializes the free double linked-list with the indices of all cached entity poolEntities. func (p *Pool) initFreeEntities() { - p.free.head.setPoolIndex(0) - p.free.tail.setPoolIndex(0) - - for i := 1; i < len(p.poolEntities); i++ { - // appends slice index i to tail of free linked list - p.connect(p.free.tail, EIndex(i)) - // and updates its tail - p.free.tail.setPoolIndex(EIndex(i)) + for i := 0; i < len(p.poolEntities); i++ { + p.appendEntity(stateFree, EIndex(i)) } } @@ -93,47 +117,31 @@ func (p *Pool) initFreeEntities() { // // If the pool has no available slots and an ejection is set, ejection occurs when adding a new entity. // If an ejection occurred, ejectedEntity holds the ejected entity. -func (p *Pool) Add(entityId flow.Identifier, entity flow.Entity, owner uint64) (entityIndex EIndex, slotAvailable bool, ejectedEntity flow.Entity) { +func (p *Pool) Add(entityId flow.Identifier, entity flow.Entity, owner uint64) ( + entityIndex EIndex, slotAvailable bool, ejectedEntity flow.Entity) { entityIndex, slotAvailable, ejectedEntity = p.sliceIndexForEntity() if slotAvailable { p.poolEntities[entityIndex].entity = entity p.poolEntities[entityIndex].id = entityId p.poolEntities[entityIndex].owner = owner - p.poolEntities[entityIndex].node.next.setUndefined() - p.poolEntities[entityIndex].node.prev.setUndefined() - - if p.used.head.isUndefined() { - // used list is empty, hence setting head of used list to current entityIndex. - p.used.head.setPoolIndex(entityIndex) - p.poolEntities[p.used.head.getSliceIndex()].node.prev.setUndefined() - } - - if !p.used.tail.isUndefined() { - // links new entity to the tail - p.connect(p.used.tail, entityIndex) - } - - // since we are appending to the used list, entityIndex also acts as tail of the list. - p.used.tail.setPoolIndex(entityIndex) - - p.size++ + p.appendEntity(stateUsed, entityIndex) } return entityIndex, slotAvailable, ejectedEntity } // Get returns entity corresponding to the entity index from the underlying list. -func (p Pool) Get(entityIndex EIndex) (flow.Identifier, flow.Entity, uint64) { +func (p *Pool) Get(entityIndex EIndex) (flow.Identifier, flow.Entity, uint64) { return p.poolEntities[entityIndex].id, p.poolEntities[entityIndex].entity, p.poolEntities[entityIndex].owner } // All returns all stored entities in this pool. func (p Pool) All() []PoolEntity { - all := make([]PoolEntity, p.size) + all := make([]PoolEntity, p.used.size) next := p.used.head - for i := uint32(0); i < p.size; i++ { - e := p.poolEntities[next.getSliceIndex()] + for i := uint32(0); i < p.used.size; i++ { + e := p.poolEntities[next] all[i] = e.PoolEntity next = e.node.next } @@ -144,10 +152,10 @@ func (p Pool) All() []PoolEntity { // Head returns the head of used items. Assuming no ejection happened and pool never goes beyond limit, Head returns // the first inserted element. func (p Pool) Head() (flow.Entity, bool) { - if p.used.head.isUndefined() { + if p.used.size == 0 { return nil, false } - e := p.poolEntities[p.used.head.getSliceIndex()] + e := p.poolEntities[p.used.head] return e.Entity(), true } @@ -159,22 +167,34 @@ func (p Pool) Head() (flow.Entity, bool) { // Ejection happens if there is no available slot, and there is an ejection mode set. // If an ejection occurred, ejectedEntity holds the ejected entity. func (p *Pool) sliceIndexForEntity() (i EIndex, hasAvailableSlot bool, ejectedEntity flow.Entity) { - if p.free.head.isUndefined() { + lruEject := func() (EIndex, bool, flow.Entity) { + // LRU ejection + // the used head is the oldest entity, so we turn the used head to a free head here. + invalidatedEntity := p.invalidateUsedHead() + return p.claimFreeHead(), true, invalidatedEntity + } + + if p.free.size == 0 { // the free list is empty, so we are out of space, and we need to eject. switch p.ejectionMode { case NoEjection: // pool is set for no ejection, hence, no slice index is selected, abort immediately. - return 0, false, nil - case LRUEjection: - // LRU ejection - // the used head is the oldest entity, so we turn the used head to a free head here. - invalidatedEntity := p.invalidateUsedHead() - return p.claimFreeHead(), true, invalidatedEntity + return InvalidIndex, false, nil case RandomEjection: // we only eject randomly when the pool is full and random ejection is on. - randomIndex := EIndex(rand.Uint32() % p.size) + random, err := rand.Uint32n(p.used.size) + if err != nil { + p.logger.Fatal().Err(err). + Msg("hero pool random ejection failed - falling back to LRU ejection") + // fall back to LRU ejection only for this instance + return lruEject() + } + randomIndex := EIndex(random) invalidatedEntity := p.invalidateEntityAtIndex(randomIndex) return p.claimFreeHead(), true, invalidatedEntity + case LRUEjection: + // LRU ejection + return lruEject() } } @@ -184,40 +204,40 @@ func (p *Pool) sliceIndexForEntity() (i EIndex, hasAvailableSlot bool, ejectedEn // Size returns total number of entities that this list maintains. func (p Pool) Size() uint32 { - return p.size + return p.used.size } // getHeads returns entities corresponding to the used and free heads. -func (p Pool) getHeads() (*poolEntity, *poolEntity) { +func (p *Pool) getHeads() (*poolEntity, *poolEntity) { var usedHead, freeHead *poolEntity - if !p.used.head.isUndefined() { - usedHead = &p.poolEntities[p.used.head.getSliceIndex()] + if p.used.size != 0 { + usedHead = &p.poolEntities[p.used.head] } - if !p.free.head.isUndefined() { - freeHead = &p.poolEntities[p.free.head.getSliceIndex()] + if p.free.size != 0 { + freeHead = &p.poolEntities[p.free.head] } return usedHead, freeHead } // getTails returns entities corresponding to the used and free tails. -func (p Pool) getTails() (*poolEntity, *poolEntity) { +func (p *Pool) getTails() (*poolEntity, *poolEntity) { var usedTail, freeTail *poolEntity - if !p.used.tail.isUndefined() { - usedTail = &p.poolEntities[p.used.tail.getSliceIndex()] + if p.used.size != 0 { + usedTail = &p.poolEntities[p.used.tail] } - if !p.free.tail.isUndefined() { - freeTail = &p.poolEntities[p.free.tail.getSliceIndex()] + if p.free.size != 0 { + freeTail = &p.poolEntities[p.free.tail] } return usedTail, freeTail } // connect links the prev and next nodes as the adjacent nodes in the double-linked list. -func (p *Pool) connect(prev poolIndex, next EIndex) { - p.poolEntities[prev.getSliceIndex()].node.next.setPoolIndex(next) +func (p *Pool) connect(prev EIndex, next EIndex) { + p.poolEntities[prev].node.next = next p.poolEntities[next].node.prev = prev } @@ -225,34 +245,15 @@ func (p *Pool) connect(prev poolIndex, next EIndex) { // also removes the entity the invalidated head is presenting and appends the // node represented by the used head to the tail of the free list. func (p *Pool) invalidateUsedHead() flow.Entity { - headSliceIndex := p.used.head.getSliceIndex() + headSliceIndex := p.used.head return p.invalidateEntityAtIndex(headSliceIndex) } // claimFreeHead moves the free head forward, and returns the slice index of the // old free head to host a new entity. func (p *Pool) claimFreeHead() EIndex { - oldFreeHeadIndex := p.free.head.getSliceIndex() - // moves head forward - p.free.head = p.poolEntities[oldFreeHeadIndex].node.next - // new head should point to an undefined prev, - // but we first check if list is not empty, i.e., - // head itself is not undefined. - if !p.free.head.isUndefined() { - p.poolEntities[p.free.head.getSliceIndex()].node.prev.setUndefined() - } - - // also, we check if the old head and tail are aligned and, if so, update the - // tail as well. This happens when we claim the only existing - // node of the free list. - if p.free.tail.getSliceIndex() == oldFreeHeadIndex { - p.free.tail.setUndefined() - } - - // clears pointers of claimed head - p.poolEntities[oldFreeHeadIndex].node.next.setUndefined() - p.poolEntities[oldFreeHeadIndex].node.prev.setUndefined() - + oldFreeHeadIndex := p.free.head + p.removeEntity(stateFree, oldFreeHeadIndex) return oldFreeHeadIndex } @@ -265,75 +266,21 @@ func (p *Pool) Remove(sliceIndex EIndex) flow.Entity { // removing its corresponding linked-list node from the used linked list, and appending // it to the tail of the free list. It also removes the entity that the invalidated node is presenting. func (p *Pool) invalidateEntityAtIndex(sliceIndex EIndex) flow.Entity { - poolEntity := p.poolEntities[sliceIndex] - prev := poolEntity.node.prev - next := poolEntity.node.next - invalidatedEntity := poolEntity.entity - - if sliceIndex != p.used.head.getSliceIndex() && sliceIndex != p.used.tail.getSliceIndex() { - // links next and prev elements for non-head and non-tail element - p.connect(prev, next.getSliceIndex()) - } - - if sliceIndex == p.used.head.getSliceIndex() { - // invalidating used head - // moves head forward - oldUsedHead, _ := p.getHeads() - p.used.head = oldUsedHead.node.next - // new head should point to an undefined prev, - // but we first check if list is not empty, i.e., - // head itself is not undefined. - if !p.used.head.isUndefined() { - usedHead, _ := p.getHeads() - usedHead.node.prev.setUndefined() - } + invalidatedEntity := p.poolEntities[sliceIndex].entity + if invalidatedEntity == nil { + panic(fmt.Sprintf("removing an entity from an empty slot with an index : %d", sliceIndex)) } - - if sliceIndex == p.used.tail.getSliceIndex() { - // invalidating used tail - // moves tail backward - oldUsedTail, _ := p.getTails() - p.used.tail = oldUsedTail.node.prev - // new head should point tail to an undefined next, - // but we first check if list is not empty, i.e., - // tail itself is not undefined. - if !p.used.tail.isUndefined() { - usedTail, _ := p.getTails() - usedTail.node.next.setUndefined() - } - } - - // invalidates entity and adds it to free entities. + p.removeEntity(stateUsed, sliceIndex) p.poolEntities[sliceIndex].id = flow.ZeroID p.poolEntities[sliceIndex].entity = nil - p.poolEntities[sliceIndex].node.next.setUndefined() - p.poolEntities[sliceIndex].node.prev.setUndefined() - - p.appendToFreeList(sliceIndex) - - // decrements Size - p.size-- + p.appendEntity(stateFree, EIndex(sliceIndex)) return invalidatedEntity } -// appendToFreeList appends linked-list node represented by getSliceIndex to tail of free list. -func (p *Pool) appendToFreeList(sliceIndex EIndex) { - if p.free.head.isUndefined() { - // free list is empty - p.free.head.setPoolIndex(sliceIndex) - p.free.tail.setPoolIndex(sliceIndex) - return - } - - // appends to the tail, and updates the tail - p.connect(p.free.tail, sliceIndex) - p.free.tail.setPoolIndex(sliceIndex) -} - // isInvalidated returns true if linked-list node represented by getSliceIndex does not contain // a valid entity. -func (p Pool) isInvalidated(sliceIndex EIndex) bool { +func (p *Pool) isInvalidated(sliceIndex EIndex) bool { if p.poolEntities[sliceIndex].id != flow.ZeroID { return false } @@ -344,3 +291,78 @@ func (p Pool) isInvalidated(sliceIndex EIndex) bool { return true } + +// utility method that removes an entity from one of the states. +// NOTE: a removed entity has to be added to another state. +func (p *Pool) removeEntity(stateType StateType, entityIndex EIndex) { + var s *state = nil + switch stateType { + case stateFree: + s = &p.free + case stateUsed: + s = &p.used + default: + panic(fmt.Sprintf("unknown state type: %s", stateType)) + } + + if s.size == 0 { + panic("Removing an entity from an empty list") + } + if s.size == 1 { + // here set to InvalidIndex + s.head = InvalidIndex + s.tail = InvalidIndex + s.size-- + p.poolEntities[entityIndex].node.next = InvalidIndex + p.poolEntities[entityIndex].node.prev = InvalidIndex + return + } + node := p.poolEntities[entityIndex].node + + if entityIndex != s.head && entityIndex != s.tail { + // links next and prev elements for non-head and non-tail element + p.connect(node.prev, node.next) + } + + if entityIndex == s.head { + // moves head forward + s.head = node.next + p.poolEntities[s.head].node.prev = InvalidIndex + } + + if entityIndex == s.tail { + // moves tail backwards + s.tail = node.prev + p.poolEntities[s.tail].node.next = InvalidIndex + } + s.size-- + p.poolEntities[entityIndex].node.next = InvalidIndex + p.poolEntities[entityIndex].node.prev = InvalidIndex +} + +// appends an entity to the tail of the state or creates a first element. +// NOTE: entity should not be in any list before this method is applied +func (p *Pool) appendEntity(stateType StateType, entityIndex EIndex) { + var s *state = nil + switch stateType { + case stateFree: + s = &p.free + case stateUsed: + s = &p.used + default: + panic(fmt.Sprintf("unknown state type: %s", stateType)) + } + + if s.size == 0 { + s.head = entityIndex + s.tail = entityIndex + p.poolEntities[s.head].node.prev = InvalidIndex + p.poolEntities[s.tail].node.next = InvalidIndex + s.size = 1 + return + } + p.connect(s.tail, entityIndex) + s.size++ + s.tail = entityIndex + p.poolEntities[s.tail].node.next = InvalidIndex +} diff --git a/module/mempool/herocache/backdata/heropool/poolIndex.go b/module/mempool/herocache/backdata/heropool/poolIndex.go deleted file mode 100644 index 46d356faeea..00000000000 --- a/module/mempool/herocache/backdata/heropool/poolIndex.go +++ /dev/null @@ -1,34 +0,0 @@ -package heropool - -// poolIndex represents a slice-based linked list pointer. Instead of pointing -// to a memory address, this pointer points to a slice index. -// -// Note: an "undefined" (i.e., nil) notion for this poolIndex corresponds to the -// value of uint32(0). Hence, legit "index" poolEntities start from uint32(1). -// poolIndex also furnished with methods to convert a "poolIndex" value to -// a slice index, and vice versa. -type poolIndex struct { - index uint32 -} - -// isUndefined returns true if this poolIndex is set to zero. An undefined -// poolIndex is equivalent to a nil address-based one. -func (p poolIndex) isUndefined() bool { - return p.index == uint32(0) -} - -// setUndefined sets poolIndex to its undefined (i.e., nil equivalent) value. -func (p *poolIndex) setUndefined() { - p.index = uint32(0) -} - -// getSliceIndex returns the slice-index equivalent of the poolIndex. -func (p poolIndex) getSliceIndex() EIndex { - return EIndex(p.index) - 1 -} - -// setPoolIndex converts the input slice-based index into a pool index and -// sets the underlying poolIndex. -func (p *poolIndex) setPoolIndex(sliceIndex EIndex) { - p.index = uint32(sliceIndex + 1) -} diff --git a/module/mempool/herocache/backdata/heropool/pool_test.go b/module/mempool/herocache/backdata/heropool/pool_test.go index 8f3a83db681..5cc19609753 100644 --- a/module/mempool/herocache/backdata/heropool/pool_test.go +++ b/module/mempool/herocache/backdata/heropool/pool_test.go @@ -2,8 +2,11 @@ package heropool import ( "fmt" + "math" "testing" + "github.com/onflow/flow-go/utils/rand" + "github.com/stretchr/testify/require" "github.com/onflow/flow-go/model/flow" @@ -215,6 +218,151 @@ func TestInvalidateEntity(t *testing.T) { } } +// TestAddAndRemoveEntities checks health of heroPool for scenario where entitites are stored and removed in a predetermined order. +// LRUEjection, NoEjection and RandomEjection are tested. RandomEjection doesn't allow to provide a final state of the pool to check. +func TestAddAndRemoveEntities(t *testing.T) { + for _, tc := range []struct { + limit uint32 // capacity of the pool + entityCount uint32 // total entities to be stored + ejectionMode EjectionMode // ejection mode + numberOfOperations int + probabilityOfAdding float32 + }{ + { + limit: 500, + entityCount: 1000, + ejectionMode: LRUEjection, + numberOfOperations: 1000, + probabilityOfAdding: 0.8, + }, + { + limit: 500, + entityCount: 1000, + ejectionMode: NoEjection, + numberOfOperations: 1000, + probabilityOfAdding: 0.8, + }, + { + limit: 500, + entityCount: 1000, + ejectionMode: RandomEjection, + numberOfOperations: 1000, + probabilityOfAdding: 0.8, + }, + } { + t.Run(fmt.Sprintf("%d-limit-%d-entities", tc.limit, tc.entityCount), func(t *testing.T) { + testAddRemoveEntities(t, tc.limit, tc.entityCount, tc.ejectionMode, tc.numberOfOperations, tc.probabilityOfAdding) + }) + } +} + +// testAddRemoveEntities adds and removes randomly elements in the pool, probabilityOfAdding and its counterpart 1-probabilityOfAdding are probabilities +// for an operation to be add or remove. Current timestamp is taken as a seed for the random number generator. +func testAddRemoveEntities(t *testing.T, limit uint32, entityCount uint32, ejectionMode EjectionMode, numberOfOperations int, probabilityOfAdding float32) { + + require.GreaterOrEqual(t, entityCount, 2*limit, "entityCount must be greater or equal to 2*limit to test add/remove operations") + + randomIntN := func(length int) int { + random, err := rand.Uintn(uint(length)) + require.NoError(t, err) + return int(random) + } + + pool := NewHeroPool(limit, ejectionMode, unittest.Logger()) + entities := unittest.EntityListFixture(uint(entityCount)) + // retryLimit is the max number of retries to find an entity that is not already in the pool to add it. + // The test fails if it reaches this limit. + retryLimit := 100 + // an array of random owner Ids. + ownerIds := make([]uint64, entityCount) + // generate ownerId to index in the entities array. + for i := 0; i < int(entityCount); i++ { + randomOwnerId, err := rand.Uint64() + require.Nil(t, err) + ownerIds[i] = randomOwnerId + } + // this map maintains entities currently stored in the pool. + addedEntities := make(map[flow.Identifier]int) + addedEntitiesInPool := make(map[flow.Identifier]EIndex) + for i := 0; i < numberOfOperations; i++ { + // choose between Add and Remove with a probability of probabilityOfAdding and 1-probabilityOfAdding respectively. + if float32(randomIntN(math.MaxInt32))/math.MaxInt32 < probabilityOfAdding || len(addedEntities) == 0 { + // keeps finding an entity to add until it finds one that is not already in the pool. + found := false + for retryTime := 0; retryTime < retryLimit; retryTime++ { + toAddIndex := randomIntN(int(entityCount)) + _, found = addedEntities[entities[toAddIndex].ID()] + if !found { + // found an entity that is not in the pool, add it. + indexInThePool, _, ejectedEntity := pool.Add(entities[toAddIndex].ID(), entities[toAddIndex], ownerIds[toAddIndex]) + if ejectionMode != NoEjection || len(addedEntities) < int(limit) { + // when there is an ejection mode in place, or the pool is not full, the index should be valid. + require.NotEqual(t, InvalidIndex, indexInThePool) + } + require.LessOrEqual(t, len(addedEntities), int(limit), "pool should not contain more elements than its limit") + if ejectionMode != NoEjection && len(addedEntities) == int(limit) { + // when there is an ejection mode in place, the ejected entity should be valid. + require.NotNil(t, ejectedEntity) + } + if ejectionMode != NoEjection && len(addedEntities) >= int(limit) { + // when there is an ejection mode in place, the ejected entity should be valid. + require.NotNil(t, ejectedEntity) + } + if indexInThePool != InvalidIndex { + entityId := entities[toAddIndex].ID() + // tracks the index of the entity in the pool and the index of the entity in the entities array. + addedEntities[entityId] = int(toAddIndex) + addedEntitiesInPool[entityId] = indexInThePool + // any entity added to the pool should be in the pool, and must be retrievable. + actualFlowId, actualEntity, actualOwnerId := pool.Get(indexInThePool) + require.Equal(t, entityId, actualFlowId) + require.Equal(t, entities[toAddIndex], actualEntity, "pool returned a different entity than the one added") + require.Equal(t, ownerIds[toAddIndex], actualOwnerId, "pool returned a different owner than the one added") + } + if ejectedEntity != nil { + require.Contains(t, addedEntities, ejectedEntity.ID(), "pool ejected an entity that was not added before") + delete(addedEntities, ejectedEntity.ID()) + delete(addedEntitiesInPool, ejectedEntity.ID()) + } + break + } + } + require.Falsef(t, found, "could not find an entity to add after %d retries", retryLimit) + } else { + // randomly select an index of an entity to remove. + entityToRemove := randomIntN(len(addedEntities)) + i := 0 + var indexInPoolToRemove EIndex = 0 + var indexInEntitiesArray int = 0 + for k, v := range addedEntities { + if i == entityToRemove { + indexInPoolToRemove = addedEntitiesInPool[k] + indexInEntitiesArray = v + break + } + i++ + } + // remove the selected entity from the pool. + removedEntity := pool.Remove(indexInPoolToRemove) + expectedRemovedEntityId := entities[indexInEntitiesArray].ID() + require.Equal(t, expectedRemovedEntityId, removedEntity.ID(), "removed wrong entity") + delete(addedEntities, expectedRemovedEntityId) + delete(addedEntitiesInPool, expectedRemovedEntityId) + actualFlowId, actualEntity, _ := pool.Get(indexInPoolToRemove) + require.Equal(t, flow.ZeroID, actualFlowId) + require.Equal(t, nil, actualEntity) + } + } + for k, v := range addedEntities { + indexInPool := addedEntitiesInPool[k] + actualFlowId, actualEntity, actualOwnerId := pool.Get(indexInPool) + require.Equal(t, entities[v].ID(), actualFlowId) + require.Equal(t, entities[v], actualEntity) + require.Equal(t, ownerIds[v], actualOwnerId) + } + require.Equalf(t, len(addedEntities), int(pool.Size()), "pool size is not correct, expected %d, actual %d", len(addedEntities), pool.Size()) +} + // testInvalidatingHead keeps invalidating the head and evaluates the linked-list keeps updating its head // and remains connected. func testInvalidatingHead(t *testing.T, pool *Pool, entities []*unittest.MockEntity) { @@ -232,17 +380,17 @@ func testInvalidatingHead(t *testing.T, pool *Pool, entities []*unittest.MockEnt // size of list should be decremented after each invalidation. require.Equal(t, uint32(totalEntitiesStored-i-1), pool.Size()) // invalidated head should be appended to free entities - require.Equal(t, pool.free.tail.getSliceIndex(), EIndex(i)) + require.Equal(t, pool.free.tail, EIndex(i)) if freeListInitialSize != 0 { // number of entities is below limit, hence free list is not empty. // invalidating used head must not change the free head. - require.Equal(t, EIndex(totalEntitiesStored), pool.free.head.getSliceIndex()) + require.Equal(t, EIndex(totalEntitiesStored), pool.free.head) } else { // number of entities is greater than or equal to limit, hence free list is empty. // free head must be updated to the first invalidated head (index 0), // and must be kept there for entire test (as we invalidate head not tail). - require.Equal(t, EIndex(0), pool.free.head.getSliceIndex()) + require.Equal(t, EIndex(0), pool.free.head) } // except when the list is empty, head must be updated after invalidation, @@ -251,14 +399,14 @@ func testInvalidatingHead(t *testing.T, pool *Pool, entities []*unittest.MockEnt if i != totalEntitiesStored-1 { // used linked-list tailAccessibleFromHead(t, - pool.used.head.getSliceIndex(), - pool.used.tail.getSliceIndex(), + pool.used.head, + pool.used.tail, pool, pool.Size()) headAccessibleFromTail(t, - pool.used.head.getSliceIndex(), - pool.used.tail.getSliceIndex(), + pool.used.head, + pool.used.tail, pool, pool.Size()) @@ -266,14 +414,14 @@ func testInvalidatingHead(t *testing.T, pool *Pool, entities []*unittest.MockEnt // // after invalidating each item, size of free linked-list is incremented by one. tailAccessibleFromHead(t, - pool.free.head.getSliceIndex(), - pool.free.tail.getSliceIndex(), + pool.free.head, + pool.free.tail, pool, uint32(i+1+freeListInitialSize)) headAccessibleFromTail(t, - pool.free.head.getSliceIndex(), - pool.free.tail.getSliceIndex(), + pool.free.head, + pool.free.tail, pool, uint32(i+1+freeListInitialSize)) } @@ -287,21 +435,23 @@ func testInvalidatingHead(t *testing.T, pool *Pool, entities []*unittest.MockEnt // used tail should point to the last element in pool, since we are // invalidating head. require.Equal(t, entities[totalEntitiesStored-1].ID(), usedTail.id) - require.Equal(t, EIndex(totalEntitiesStored-1), pool.used.tail.getSliceIndex()) + require.Equal(t, EIndex(totalEntitiesStored-1), pool.used.tail) // used head must point to the next element in the pool, // i.e., invalidating head moves it forward. require.Equal(t, entities[i+1].ID(), usedHead.id) - require.Equal(t, EIndex(i+1), pool.used.head.getSliceIndex()) + require.Equal(t, EIndex(i+1), pool.used.head) } else { // pool is empty // used head and tail must be nil and their corresponding // pointer indices must be undefined. require.Nil(t, usedHead) require.Nil(t, usedTail) - require.True(t, pool.used.tail.isUndefined()) - require.True(t, pool.used.head.isUndefined()) + require.True(t, pool.used.size == 0) + require.Equal(t, pool.used.tail, InvalidIndex) + require.Equal(t, pool.used.head, InvalidIndex) } + checkEachEntityIsInFreeOrUsedState(t, pool) } } @@ -311,25 +461,25 @@ func testInvalidatingTail(t *testing.T, pool *Pool, entities []*unittest.MockEnt offset := len(pool.poolEntities) - size for i := 0; i < size; i++ { // invalidates tail index - tailIndex := pool.used.tail.getSliceIndex() + tailIndex := pool.used.tail require.Equal(t, EIndex(size-1-i), tailIndex) pool.invalidateEntityAtIndex(tailIndex) // old head index must be invalidated require.True(t, pool.isInvalidated(tailIndex)) // unclaimed head should be appended to free entities - require.Equal(t, pool.free.tail.getSliceIndex(), tailIndex) + require.Equal(t, pool.free.tail, tailIndex) if offset != 0 { // number of entities is below limit // free must head keeps pointing to first empty index after // adding all entities. - require.Equal(t, EIndex(size), pool.free.head.getSliceIndex()) + require.Equal(t, EIndex(size), pool.free.head) } else { // number of entities is greater than or equal to limit // free head must be updated to last element in the pool (size - 1), // and must be kept there for entire test (as we invalidate tail not head). - require.Equal(t, EIndex(size-1), pool.free.head.getSliceIndex()) + require.Equal(t, EIndex(size-1), pool.free.head) } // size of pool should be shrunk after each invalidation. @@ -342,27 +492,27 @@ func testInvalidatingTail(t *testing.T, pool *Pool, entities []*unittest.MockEnt // used linked-list tailAccessibleFromHead(t, - pool.used.head.getSliceIndex(), - pool.used.tail.getSliceIndex(), + pool.used.head, + pool.used.tail, pool, pool.Size()) headAccessibleFromTail(t, - pool.used.head.getSliceIndex(), - pool.used.tail.getSliceIndex(), + pool.used.head, + pool.used.tail, pool, pool.Size()) // free linked-list tailAccessibleFromHead(t, - pool.free.head.getSliceIndex(), - pool.free.tail.getSliceIndex(), + pool.free.head, + pool.free.tail, pool, uint32(i+1+offset)) headAccessibleFromTail(t, - pool.free.head.getSliceIndex(), - pool.free.tail.getSliceIndex(), + pool.free.head, + pool.free.tail, pool, uint32(i+1+offset)) } @@ -374,52 +524,55 @@ func testInvalidatingTail(t *testing.T, pool *Pool, entities []*unittest.MockEnt // // used tail should move backward after each invalidation require.Equal(t, entities[size-i-2].ID(), usedTail.id) - require.Equal(t, EIndex(size-i-2), pool.used.tail.getSliceIndex()) + require.Equal(t, EIndex(size-i-2), pool.used.tail) // used head must point to the first element in the pool, require.Equal(t, entities[0].ID(), usedHead.id) - require.Equal(t, EIndex(0), pool.used.head.getSliceIndex()) + require.Equal(t, EIndex(0), pool.used.head) } else { // pool is empty // used head and tail must be nil and their corresponding // pointer indices must be undefined. require.Nil(t, usedHead) require.Nil(t, usedTail) - require.True(t, pool.used.tail.isUndefined()) - require.True(t, pool.used.head.isUndefined()) + require.True(t, pool.used.size == 0) + require.Equal(t, pool.used.head, InvalidIndex) + require.Equal(t, pool.used.tail, InvalidIndex) } + checkEachEntityIsInFreeOrUsedState(t, pool) } } // testInitialization evaluates the state of an initialized pool before adding any element to it. func testInitialization(t *testing.T, pool *Pool, _ []*unittest.MockEntity) { - // head and tail of "used" linked-list must be undefined at initialization time, since we have no elements in the list. - require.True(t, pool.used.head.isUndefined()) - require.True(t, pool.used.tail.isUndefined()) + // "used" linked-list must have a zero size, since we have no elements in the list. + require.True(t, pool.used.size == 0) + require.Equal(t, pool.used.head, InvalidIndex) + require.Equal(t, pool.used.tail, InvalidIndex) for i := 0; i < len(pool.poolEntities); i++ { if i == 0 { - // head of "free" linked-list should point to index 0 of entities slice. - require.Equal(t, EIndex(i), pool.free.head.getSliceIndex()) + // head of "free" linked-list should point to InvalidIndex of entities slice. + require.Equal(t, EIndex(i), pool.free.head) // previous element of head must be undefined (linked-list head feature). - require.True(t, pool.poolEntities[i].node.prev.isUndefined()) + require.Equal(t, pool.poolEntities[i].node.prev, InvalidIndex) } if i != 0 { // except head, any element should point back to its previous index in slice. - require.Equal(t, EIndex(i-1), pool.poolEntities[i].node.prev.getSliceIndex()) + require.Equal(t, EIndex(i-1), pool.poolEntities[i].node.prev) } if i != len(pool.poolEntities)-1 { // except tail, any element should point forward to its next index in slice. - require.Equal(t, EIndex(i+1), pool.poolEntities[i].node.next.getSliceIndex()) + require.Equal(t, EIndex(i+1), pool.poolEntities[i].node.next) } if i == len(pool.poolEntities)-1 { // tail of "free" linked-list should point to the last index in entities slice. - require.Equal(t, EIndex(i), pool.free.tail.getSliceIndex()) + require.Equal(t, EIndex(i), pool.free.tail) // next element of tail must be undefined. - require.True(t, pool.poolEntities[i].node.next.isUndefined()) + require.Equal(t, pool.poolEntities[i].node.next, InvalidIndex) } } } @@ -492,7 +645,7 @@ func testAddingEntities(t *testing.T, pool *Pool, entitiesToBeAdded []*unittest. if i >= len(pool.poolEntities) { require.False(t, slotAvailable) require.Nil(t, ejectedEntity) - require.Equal(t, entityIndex, EIndex(0)) + require.Equal(t, entityIndex, InvalidIndex) // when pool is full and with NoEjection, the head must keep pointing to the first added element. headEntity, headExists := pool.Head() @@ -515,30 +668,31 @@ func testAddingEntities(t *testing.T, pool *Pool, entitiesToBeAdded []*unittest. } require.Equal(t, pool.poolEntities[expectedUsedHead].entity, usedHead.entity) // head must be healthy and point back to undefined. - require.True(t, usedHead.node.prev.isUndefined()) + require.Equal(t, usedHead.node.prev, InvalidIndex) } if ejectionMode != NoEjection || i < len(pool.poolEntities) { // new entity must be successfully added to tail of used linked-list require.Equal(t, entitiesToBeAdded[i], usedTail.entity) // used tail must be healthy and point back to undefined. - require.True(t, usedTail.node.next.isUndefined()) + require.Equal(t, usedTail.node.next, InvalidIndex) } if ejectionMode == NoEjection && i >= len(pool.poolEntities) { // used tail must not move require.Equal(t, entitiesToBeAdded[len(pool.poolEntities)-1], usedTail.entity) // used tail must be healthy and point back to undefined. - require.True(t, usedTail.node.next.isUndefined()) + // This is not needed anymore as tail's next is now ignored + require.Equal(t, usedTail.node.next, InvalidIndex) } // free head if i < len(pool.poolEntities)-1 { // as long as we are below limit, after adding i element, free head // should move to i+1 element. - require.Equal(t, EIndex(i+1), pool.free.head.getSliceIndex()) + require.Equal(t, EIndex(i+1), pool.free.head) // head must be healthy and point back to undefined. - require.True(t, freeHead.node.prev.isUndefined()) + require.Equal(t, freeHead.node.prev, InvalidIndex) } else { // once we go beyond limit, // we run out of free slots, @@ -552,9 +706,9 @@ func testAddingEntities(t *testing.T, pool *Pool, entitiesToBeAdded []*unittest. // must keep pointing to last index of the array-based linked-list. In other // words, adding element must not change free tail (since only free head is // updated). - require.Equal(t, EIndex(len(pool.poolEntities)-1), pool.free.tail.getSliceIndex()) + require.Equal(t, EIndex(len(pool.poolEntities)-1), pool.free.tail) // head tail be healthy and point next to undefined. - require.True(t, freeTail.node.next.isUndefined()) + require.Equal(t, freeTail.node.next, InvalidIndex) } else { // once we go beyond limit, we run out of free slots, and // free tail must be kept at undefined. @@ -572,13 +726,13 @@ func testAddingEntities(t *testing.T, pool *Pool, entitiesToBeAdded []*unittest. usedTraverseStep = uint32(len(pool.poolEntities)) } tailAccessibleFromHead(t, - pool.used.head.getSliceIndex(), - pool.used.tail.getSliceIndex(), + pool.used.head, + pool.used.tail, pool, usedTraverseStep) headAccessibleFromTail(t, - pool.used.head.getSliceIndex(), - pool.used.tail.getSliceIndex(), + pool.used.head, + pool.used.tail, pool, usedTraverseStep) @@ -596,15 +750,17 @@ func testAddingEntities(t *testing.T, pool *Pool, entitiesToBeAdded []*unittest. freeTraverseStep = uint32(0) } tailAccessibleFromHead(t, - pool.free.head.getSliceIndex(), - pool.free.tail.getSliceIndex(), + pool.free.head, + pool.free.tail, pool, freeTraverseStep) headAccessibleFromTail(t, - pool.free.head.getSliceIndex(), - pool.free.tail.getSliceIndex(), + pool.free.head, + pool.free.tail, pool, freeTraverseStep) + + checkEachEntityIsInFreeOrUsedState(t, pool) } } @@ -645,10 +801,10 @@ func withTestScenario(t *testing.T, ejectionMode EjectionMode, helpers ...func(*testing.T, *Pool, []*unittest.MockEntity)) { - pool := NewHeroPool(limit, ejectionMode) + pool := NewHeroPool(limit, ejectionMode, unittest.Logger()) // head on underlying linked-list value should be uninitialized - require.True(t, pool.used.head.isUndefined()) + require.True(t, pool.used.size == 0) require.Equal(t, pool.Size(), uint32(0)) entities := unittest.EntityListFixture(uint(entityCount)) @@ -673,8 +829,8 @@ func tailAccessibleFromHead(t *testing.T, headSliceIndex EIndex, tailSliceIndex _, ok := seen[index] require.False(t, ok, "duplicate identifiers found") - require.False(t, pool.poolEntities[index].node.next.isUndefined(), "tail not found, and reached end of list") - index = pool.poolEntities[index].node.next.getSliceIndex() + require.NotEqual(t, pool.poolEntities[index].node.next, InvalidIndex, "tail not found, and reached end of list") + index = pool.poolEntities[index].node.next } } @@ -693,6 +849,40 @@ func headAccessibleFromTail(t *testing.T, headSliceIndex EIndex, tailSliceIndex _, ok := seen[index] require.False(t, ok, "duplicate identifiers found") - index = pool.poolEntities[index].node.prev.getSliceIndex() + index = pool.poolEntities[index].node.prev + } +} + +// checkEachEntityIsInFreeOrUsedState checks if each entity in the pool belongs exactly to one of the state lists. +func checkEachEntityIsInFreeOrUsedState(t *testing.T, pool *Pool) { + pool_capacity := len(pool.poolEntities) + // check size + require.Equal(t, int(pool.free.size+pool.used.size), pool_capacity, "Pool capacity is not equal to the sum of used and free sizes") + // check elelments + nodesInFree := discoverEntitiesBelongingToStateList(t, pool, stateFree) + nodesInUsed := discoverEntitiesBelongingToStateList(t, pool, stateUsed) + for i := 0; i < pool_capacity; i++ { + require.False(t, !nodesInFree[i] && !nodesInUsed[i], "Node is not in any state list") + require.False(t, nodesInFree[i] && nodesInUsed[i], "Node is in two state lists at the same time") + } +} + +// discoverEntitiesBelongingToStateList discovers all entities in the pool that belong to the given list. +func discoverEntitiesBelongingToStateList(t *testing.T, pool *Pool, stateType StateType) []bool { + var s *state = nil + switch stateType { + case stateFree: + s = &pool.free + case stateUsed: + s = &pool.used + default: + panic(fmt.Sprintf("unknown state type: %s", stateType)) + } + result := make([]bool, len(pool.poolEntities)) + for node_index := s.head; node_index != InvalidIndex; { + require.False(t, result[node_index], "A node is present two times in the same state list") + result[node_index] = true + node_index = pool.poolEntities[node_index].node.next } + return result } diff --git a/module/mempool/herocache/dns_cache.go b/module/mempool/herocache/dns_cache.go index db4c9a9b67b..9af171c39ae 100644 --- a/module/mempool/herocache/dns_cache.go +++ b/module/mempool/herocache/dns_cache.go @@ -19,7 +19,8 @@ type DNSCache struct { txtCache *stdmap.Backend } -func NewDNSCache(sizeLimit uint32, logger zerolog.Logger, ipCollector module.HeroCacheMetrics, txtCollector module.HeroCacheMetrics) *DNSCache { +func NewDNSCache(sizeLimit uint32, logger zerolog.Logger, ipCollector module.HeroCacheMetrics, txtCollector module.HeroCacheMetrics, +) *DNSCache { return &DNSCache{ txtCache: stdmap.NewBackend( stdmap.WithBackData( diff --git a/module/mempool/herocache/execution_data.go b/module/mempool/herocache/execution_data.go new file mode 100644 index 00000000000..9a075692578 --- /dev/null +++ b/module/mempool/herocache/execution_data.go @@ -0,0 +1,98 @@ +package herocache + +import ( + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" + "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" + "github.com/onflow/flow-go/module/mempool/herocache/internal" + "github.com/onflow/flow-go/module/mempool/stdmap" +) + +type BlockExecutionData struct { + c *stdmap.Backend +} + +// NewBlockExecutionData implements a block execution data mempool based on hero cache. +func NewBlockExecutionData(limit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *BlockExecutionData { + return &BlockExecutionData{ + c: stdmap.NewBackend( + stdmap.WithBackData( + herocache.NewCache(limit, + herocache.DefaultOversizeFactor, + heropool.LRUEjection, + logger.With().Str("mempool", "block_execution_data").Logger(), + collector))), + } +} + +// Has checks whether the block execution data for the given block ID is currently in +// the memory pool. +func (t *BlockExecutionData) Has(blockID flow.Identifier) bool { + return t.c.Has(blockID) +} + +// Add adds a block execution data to the mempool, keyed by block ID. +// It returns false if the execution data was already in the mempool. +func (t *BlockExecutionData) Add(ed *execution_data.BlockExecutionDataEntity) bool { + entity := internal.NewWrappedEntity(ed.BlockID, ed) + return t.c.Add(*entity) +} + +// ByID returns the block execution data for the given block ID from the mempool. +// It returns false if the execution data was not found in the mempool. +func (t *BlockExecutionData) ByID(blockID flow.Identifier) (*execution_data.BlockExecutionDataEntity, bool) { + entity, exists := t.c.ByID(blockID) + if !exists { + return nil, false + } + + return unwrap(entity), true +} + +// All returns all block execution data from the mempool. Since it is using the HeroCache, All guarantees returning +// all block execution data in the same order as they are added. +func (t *BlockExecutionData) All() []*execution_data.BlockExecutionDataEntity { + entities := t.c.All() + eds := make([]*execution_data.BlockExecutionDataEntity, 0, len(entities)) + for _, entity := range entities { + eds = append(eds, unwrap(entity)) + } + return eds +} + +// Clear removes all block execution data stored in this mempool. +func (t *BlockExecutionData) Clear() { + t.c.Clear() +} + +// Size returns total number of stored block execution data. +func (t *BlockExecutionData) Size() uint { + return t.c.Size() +} + +// Remove removes block execution data from mempool by block ID. +// It returns true if the execution data was known and removed. +func (t *BlockExecutionData) Remove(blockID flow.Identifier) bool { + return t.c.Remove(blockID) +} + +// unwrap converts an internal.WrappedEntity to a BlockExecutionDataEntity. +func unwrap(entity flow.Entity) *execution_data.BlockExecutionDataEntity { + wrappedEntity, ok := entity.(internal.WrappedEntity) + if !ok { + panic(fmt.Sprintf("invalid wrapped entity in block execution data pool (%T)", entity)) + } + + ed, ok := wrappedEntity.Entity.(*execution_data.BlockExecutionDataEntity) + if !ok { + panic(fmt.Sprintf("invalid entity in block execution data pool (%T)", wrappedEntity.Entity)) + } + + return ed +} diff --git a/module/mempool/herocache/execution_data_test.go b/module/mempool/herocache/execution_data_test.go new file mode 100644 index 00000000000..46c0d302956 --- /dev/null +++ b/module/mempool/herocache/execution_data_test.go @@ -0,0 +1,117 @@ +package herocache_test + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/mempool/herocache" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestBlockExecutionDataPool(t *testing.T) { + ed1 := unittest.BlockExecutionDatEntityFixture() + ed2 := unittest.BlockExecutionDatEntityFixture() + + cache := herocache.NewBlockExecutionData(1000, unittest.Logger(), metrics.NewNoopCollector()) + + t.Run("should be able to add first", func(t *testing.T) { + added := cache.Add(ed1) + assert.True(t, added) + }) + + t.Run("should be able to add second", func(t *testing.T) { + added := cache.Add(ed2) + assert.True(t, added) + }) + + t.Run("should be able to get size", func(t *testing.T) { + size := cache.Size() + assert.EqualValues(t, 2, size) + }) + + t.Run("should be able to get first by blockID", func(t *testing.T) { + actual, exists := cache.ByID(ed1.BlockID) + assert.True(t, exists) + assert.Equal(t, ed1, actual) + }) + + t.Run("should be able to remove second by blockID", func(t *testing.T) { + ok := cache.Remove(ed2.BlockID) + assert.True(t, ok) + }) + + t.Run("should be able to retrieve all", func(t *testing.T) { + items := cache.All() + assert.Len(t, items, 1) + assert.Equal(t, ed1, items[0]) + }) + + t.Run("should be able to clear", func(t *testing.T) { + assert.True(t, cache.Size() > 0) + cache.Clear() + assert.Equal(t, uint(0), cache.Size()) + }) +} + +// TestConcurrentWriteAndRead checks correctness of cache mempool under concurrent read and write. +func TestBlockExecutionDataConcurrentWriteAndRead(t *testing.T) { + total := 100 + execDatas := unittest.BlockExecutionDatEntityListFixture(total) + cache := herocache.NewBlockExecutionData(uint32(total), unittest.Logger(), metrics.NewNoopCollector()) + + wg := sync.WaitGroup{} + wg.Add(total) + + // storing all cache + for i := 0; i < total; i++ { + go func(ed *execution_data.BlockExecutionDataEntity) { + require.True(t, cache.Add(ed)) + + wg.Done() + }(execDatas[i]) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "could not write all cache on time") + require.Equal(t, cache.Size(), uint(total)) + + wg.Add(total) + // reading all cache + for i := 0; i < total; i++ { + go func(ed *execution_data.BlockExecutionDataEntity) { + actual, ok := cache.ByID(ed.BlockID) + require.True(t, ok) + require.Equal(t, ed, actual) + + wg.Done() + }(execDatas[i]) + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "could not read all cache on time") +} + +// TestAllReturnsInOrder checks All method of the HeroCache-based cache mempool returns all +// cache in the same order as they are returned. +func TestBlockExecutionDataAllReturnsInOrder(t *testing.T) { + total := 100 + execDatas := unittest.BlockExecutionDatEntityListFixture(total) + cache := herocache.NewBlockExecutionData(uint32(total), unittest.Logger(), metrics.NewNoopCollector()) + + // storing all cache + for i := 0; i < total; i++ { + require.True(t, cache.Add(execDatas[i])) + ed, ok := cache.ByID(execDatas[i].BlockID) + require.True(t, ok) + require.Equal(t, execDatas[i], ed) + } + + // all cache must be retrieved in the same order as they are added + all := cache.All() + for i := 0; i < total; i++ { + require.Equal(t, execDatas[i], all[i]) + } +} diff --git a/module/mempool/herocache/internal/wrapped_entity.go b/module/mempool/herocache/internal/wrapped_entity.go new file mode 100644 index 00000000000..342f9094f3c --- /dev/null +++ b/module/mempool/herocache/internal/wrapped_entity.go @@ -0,0 +1,33 @@ +package internal + +import "github.com/onflow/flow-go/model/flow" + +// WrappedEntity is a wrapper around a flow.Entity that allows overriding the ID. +// The has 2 main use cases: +// - when the ID is expensive to compute, we can pre-compute it and use it for the cache +// - when caching an entity using a different ID than what's returned by ID(). For example, if there +// is a 1:1 mapping between a block and an entity, we can use the block ID as the cache key. +type WrappedEntity struct { + flow.Entity + id flow.Identifier +} + +var _ flow.Entity = (*WrappedEntity)(nil) + +// NewWrappedEntity creates a new WrappedEntity +func NewWrappedEntity(id flow.Identifier, entity flow.Entity) *WrappedEntity { + return &WrappedEntity{ + Entity: entity, + id: id, + } +} + +// ID returns the cached ID of the wrapped entity +func (w WrappedEntity) ID() flow.Identifier { + return w.id +} + +// Checksum returns th cached ID of the wrapped entity +func (w WrappedEntity) Checksum() flow.Identifier { + return w.id +} diff --git a/module/mempool/mock/execution_data.go b/module/mempool/mock/execution_data.go new file mode 100644 index 00000000000..9a9b1669daf --- /dev/null +++ b/module/mempool/mock/execution_data.go @@ -0,0 +1,133 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mempool + +import ( + flow "github.com/onflow/flow-go/model/flow" + execution_data "github.com/onflow/flow-go/module/executiondatasync/execution_data" + + mock "github.com/stretchr/testify/mock" +) + +// ExecutionData is an autogenerated mock type for the ExecutionData type +type ExecutionData struct { + mock.Mock +} + +// Add provides a mock function with given fields: _a0 +func (_m *ExecutionData) Add(_a0 *execution_data.BlockExecutionDataEntity) bool { + ret := _m.Called(_a0) + + var r0 bool + if rf, ok := ret.Get(0).(func(*execution_data.BlockExecutionDataEntity) bool); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// All provides a mock function with given fields: +func (_m *ExecutionData) All() []*execution_data.BlockExecutionDataEntity { + ret := _m.Called() + + var r0 []*execution_data.BlockExecutionDataEntity + if rf, ok := ret.Get(0).(func() []*execution_data.BlockExecutionDataEntity); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*execution_data.BlockExecutionDataEntity) + } + } + + return r0 +} + +// ByID provides a mock function with given fields: _a0 +func (_m *ExecutionData) ByID(_a0 flow.Identifier) (*execution_data.BlockExecutionDataEntity, bool) { + ret := _m.Called(_a0) + + var r0 *execution_data.BlockExecutionDataEntity + var r1 bool + if rf, ok := ret.Get(0).(func(flow.Identifier) (*execution_data.BlockExecutionDataEntity, bool)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(flow.Identifier) *execution_data.BlockExecutionDataEntity); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*execution_data.BlockExecutionDataEntity) + } + } + + if rf, ok := ret.Get(1).(func(flow.Identifier) bool); ok { + r1 = rf(_a0) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// Clear provides a mock function with given fields: +func (_m *ExecutionData) Clear() { + _m.Called() +} + +// Has provides a mock function with given fields: _a0 +func (_m *ExecutionData) Has(_a0 flow.Identifier) bool { + ret := _m.Called(_a0) + + var r0 bool + if rf, ok := ret.Get(0).(func(flow.Identifier) bool); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// Remove provides a mock function with given fields: _a0 +func (_m *ExecutionData) Remove(_a0 flow.Identifier) bool { + ret := _m.Called(_a0) + + var r0 bool + if rf, ok := ret.Get(0).(func(flow.Identifier) bool); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// Size provides a mock function with given fields: +func (_m *ExecutionData) Size() uint { + ret := _m.Called() + + var r0 uint + if rf, ok := ret.Get(0).(func() uint); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint) + } + + return r0 +} + +type mockConstructorTestingTNewExecutionData interface { + mock.TestingT + Cleanup(func()) +} + +// NewExecutionData creates a new instance of ExecutionData. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewExecutionData(t mockConstructorTestingTNewExecutionData) *ExecutionData { + mock := &ExecutionData{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/module/mempool/queue/heroQueue.go b/module/mempool/queue/heroQueue.go index ec1269147b8..ece206fec17 100644 --- a/module/mempool/queue/heroQueue.go +++ b/module/mempool/queue/heroQueue.go @@ -19,7 +19,8 @@ type HeroQueue struct { sizeLimit uint } -func NewHeroQueue(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *HeroQueue { +func NewHeroQueue(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, +) *HeroQueue { return &HeroQueue{ cache: herocache.NewCache( sizeLimit, diff --git a/module/mempool/queue/heroQueue_test.go b/module/mempool/queue/heroQueue_test.go index 75396a9b1ed..f0775a206c5 100644 --- a/module/mempool/queue/heroQueue_test.go +++ b/module/mempool/queue/heroQueue_test.go @@ -59,10 +59,8 @@ func TestHeroQueue_Sequential(t *testing.T) { func TestHeroQueue_Concurrent(t *testing.T) { sizeLimit := 100 q := queue.NewHeroQueue(uint32(sizeLimit), unittest.Logger(), metrics.NewNoopCollector()) - // initially queue must be zero require.Zero(t, q.Size()) - // initially there should be nothing to pop entity, ok := q.Pop() require.False(t, ok) diff --git a/module/mempool/queue/heroStore.go b/module/mempool/queue/heroStore.go index 8a9e4805c63..03c478e1893 100644 --- a/module/mempool/queue/heroStore.go +++ b/module/mempool/queue/heroStore.go @@ -33,7 +33,8 @@ type HeroStore struct { q *HeroQueue } -func NewHeroStore(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *HeroStore { +func NewHeroStore(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, +) *HeroStore { return &HeroStore{ q: NewHeroQueue(sizeLimit, logger, collector), } diff --git a/module/mempool/queue/queue.go b/module/mempool/queue/queue.go index 996193b9326..23f5cf3cf62 100644 --- a/module/mempool/queue/queue.go +++ b/module/mempool/queue/queue.go @@ -1,6 +1,9 @@ package queue import ( + "fmt" + "strings" + "github.com/onflow/flow-go/model/flow" ) @@ -166,3 +169,18 @@ func (q *Queue) Dismount() (Blockify, []*Queue) { } return q.Head.Item, queues } + +func (q *Queue) String() string { + var builder strings.Builder + builder.WriteString(fmt.Sprintf("Header: %v\n", q.Head.Item.ID())) + builder.WriteString(fmt.Sprintf("Highest: %v\n", q.Highest.Item.ID())) + builder.WriteString(fmt.Sprintf("Size: %v, Height: %v\n", q.Size(), q.Height())) + for _, node := range q.Nodes { + builder.WriteString(fmt.Sprintf("Node(height: %v): %v (children: %v)\n", + node.Item.Height(), + node.Item.ID(), + len(node.Children), + )) + } + return builder.String() +} diff --git a/module/mempool/queue/queue_test.go b/module/mempool/queue/queue_test.go index 9b4a35b825d..90fce14156b 100644 --- a/module/mempool/queue/queue_test.go +++ b/module/mempool/queue/queue_test.go @@ -1,6 +1,8 @@ package queue import ( + "fmt" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -21,15 +23,15 @@ func TestQueue(t *testing.T) { */ - a := unittest.ExecutableBlockFixture(nil) - c := unittest.ExecutableBlockFixtureWithParent(nil, a.Block.Header) - b := unittest.ExecutableBlockFixtureWithParent(nil, c.Block.Header) - d := unittest.ExecutableBlockFixtureWithParent(nil, c.Block.Header) - e := unittest.ExecutableBlockFixtureWithParent(nil, d.Block.Header) - f := unittest.ExecutableBlockFixtureWithParent(nil, d.Block.Header) - g := unittest.ExecutableBlockFixtureWithParent(nil, b.Block.Header) + a := unittest.ExecutableBlockFixture(nil, nil) + c := unittest.ExecutableBlockFixtureWithParent(nil, a.Block.Header, nil) + b := unittest.ExecutableBlockFixtureWithParent(nil, c.Block.Header, nil) + d := unittest.ExecutableBlockFixtureWithParent(nil, c.Block.Header, nil) + e := unittest.ExecutableBlockFixtureWithParent(nil, d.Block.Header, nil) + f := unittest.ExecutableBlockFixtureWithParent(nil, d.Block.Header, nil) + g := unittest.ExecutableBlockFixtureWithParent(nil, b.Block.Header, nil) - dBroken := unittest.ExecutableBlockFixtureWithParent(nil, c.Block.Header) + dBroken := unittest.ExecutableBlockFixtureWithParent(nil, c.Block.Header, nil) dBroken.Block.Header.Height += 2 //change height queue := NewQueue(a) @@ -289,4 +291,23 @@ func TestQueue(t *testing.T) { // assert.Error(t, err) //}) + t.Run("String()", func(t *testing.T) { + // a <- c <- d <- f + queue := NewQueue(a) + stored, _ := queue.TryAdd(c) + require.True(t, stored) + stored, _ = queue.TryAdd(d) + require.True(t, stored) + stored, _ = queue.TryAdd(f) + require.True(t, stored) + var builder strings.Builder + builder.WriteString(fmt.Sprintf("Header: %v\n", a.ID())) + builder.WriteString(fmt.Sprintf("Highest: %v\n", f.ID())) + builder.WriteString("Size: 4, Height: 3\n") + builder.WriteString(fmt.Sprintf("Node(height: %v): %v (children: 1)\n", a.Height(), a.ID())) + builder.WriteString(fmt.Sprintf("Node(height: %v): %v (children: 1)\n", c.Height(), c.ID())) + builder.WriteString(fmt.Sprintf("Node(height: %v): %v (children: 1)\n", d.Height(), d.ID())) + builder.WriteString(fmt.Sprintf("Node(height: %v): %v (children: 0)\n", f.Height(), f.ID())) + require.Equal(t, builder.String(), queue.String()) + }) } diff --git a/module/mempool/stdmap/backend.go b/module/mempool/stdmap/backend.go index cb0dca2640d..fb42e5297d5 100644 --- a/module/mempool/stdmap/backend.go +++ b/module/mempool/stdmap/backend.go @@ -23,12 +23,12 @@ type Backend struct { } // NewBackend creates a new memory pool backend. -// This is using EjectTrueRandomFast() +// This is using EjectRandomFast() func NewBackend(options ...OptionFunc) *Backend { b := Backend{ backData: backdata.NewMapBackData(), guaranteedCapacity: uint(math.MaxUint32), - batchEject: EjectTrueRandomFast, + batchEject: EjectRandomFast, eject: nil, ejectionCallbacks: nil, } @@ -185,14 +185,14 @@ func (b *Backend) reduce() { //defer binstat.Leave(bs) // we keep reducing the cache size until we are at limit again - // this was a loop, but the loop is now in EjectTrueRandomFast() + // this was a loop, but the loop is now in EjectRandomFast() // the ejections are batched, so this call to eject() may not actually // do anything until the batch threshold is reached (currently 128) if b.backData.Size() > b.guaranteedCapacity { // get the key from the eject function // we don't do anything if there is an error if b.batchEject != nil { - _ = b.batchEject(b) + _, _ = b.batchEject(b) } else { _, _, _ = b.eject(b) } diff --git a/module/mempool/stdmap/eject.go b/module/mempool/stdmap/eject.go index 3ed2d59683a..7cea5214b3d 100644 --- a/module/mempool/stdmap/eject.go +++ b/module/mempool/stdmap/eject.go @@ -3,12 +3,13 @@ package stdmap import ( + "fmt" "math" - "math/rand" "sort" "sync" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/rand" ) // this is the threshold for how much over the guaranteed capacity the @@ -31,49 +32,33 @@ const overCapacityThreshold = 128 // concurrency (specifically, it locks the mempool during ejection). // - The implementation should be non-blocking (though, it is allowed to // take a bit of time; the mempool will just be locked during this time). -type BatchEjectFunc func(b *Backend) bool +type BatchEjectFunc func(b *Backend) (bool, error) type EjectFunc func(b *Backend) (flow.Identifier, flow.Entity, bool) -// EjectTrueRandom relies on a random generator to pick a random entity to eject from the -// entity set. It will, on average, iterate through half the entities of the set. However, -// it provides us with a truly evenly distributed random selection. -func EjectTrueRandom(b *Backend) (flow.Identifier, flow.Entity, bool) { - var entity flow.Entity - var entityID flow.Identifier - - bFound := false - i := 0 - n := rand.Intn(int(b.backData.Size())) - for entityID, entity = range b.backData.All() { - if i == n { - bFound = true - break - } - i++ - } - return entityID, entity, bFound -} - -// EjectTrueRandomFast checks if the map size is beyond the +// EjectRandomFast checks if the map size is beyond the // threshold size, and will iterate through them and eject unneeded // entries if that is the case. Return values are unused -func EjectTrueRandomFast(b *Backend) bool { +func EjectRandomFast(b *Backend) (bool, error) { currentSize := b.backData.Size() if b.guaranteedCapacity >= currentSize { - return false + return false, nil } // At this point, we know that currentSize > b.guaranteedCapacity. As // currentSize fits into an int, b.guaranteedCapacity must also fit. overcapacity := currentSize - b.guaranteedCapacity if overcapacity <= overCapacityThreshold { - return false + return false, nil } // Randomly select indices of elements to remove: mapIndices := make([]int, 0, overcapacity) for i := overcapacity; i > 0; i-- { - mapIndices = append(mapIndices, rand.Intn(int(currentSize))) + rand, err := rand.Uintn(currentSize) + if err != nil { + return false, fmt.Errorf("random generation failed: %w", err) + } + mapIndices = append(mapIndices, int(rand)) } sort.Ints(mapIndices) // inplace @@ -99,13 +84,13 @@ func EjectTrueRandomFast(b *Backend) bool { } if idx == int(overcapacity) { - return true + return true, nil } next2Remove = mapIndices[idx] } i++ } - return true + return true, nil } // EjectPanic simply panics, crashing the program. Useful when cache is not expected @@ -158,7 +143,7 @@ func (q *LRUEjector) Untrack(entityID flow.Identifier) { // Eject implements EjectFunc for LRUEjector. It finds the entity with the lowest sequence number (i.e., // the oldest entity). It also untracks. This is using a linear search -func (q *LRUEjector) Eject(b *Backend) (flow.Identifier, flow.Entity, bool) { +func (q *LRUEjector) Eject(b *Backend) flow.Identifier { q.Lock() defer q.Unlock() @@ -171,19 +156,11 @@ func (q *LRUEjector) Eject(b *Backend) (flow.Identifier, flow.Entity, bool) { oldestID = id oldestSQ = sq } - } } - // TODO: don't do a lookup if it isn't necessary - oldestEntity, ok := b.backData.ByID(oldestID) - - if !ok { - oldestID, oldestEntity, ok = EjectTrueRandom(b) - } - // untracks the oldest id as it is supposed to be ejected delete(q.table, oldestID) - return oldestID, oldestEntity, ok + return oldestID } diff --git a/module/mempool/stdmap/eject_test.go b/module/mempool/stdmap/eject_test.go index cee1974e840..398c74938aa 100644 --- a/module/mempool/stdmap/eject_test.go +++ b/module/mempool/stdmap/eject_test.go @@ -196,7 +196,7 @@ func TestLRUEjector_UntrackEject(t *testing.T) { ejector.Untrack(items[0]) // next ejectable item should be the second oldest item - id, _, _ := ejector.Eject(backEnd) + id := ejector.Eject(backEnd) assert.Equal(t, id, items[1]) } @@ -224,7 +224,7 @@ func TestLRUEjector_EjectAll(t *testing.T) { // ejects one by one for i := 0; i < size; i++ { - id, _, _ := ejector.Eject(backEnd) + id := ejector.Eject(backEnd) require.Equal(t, id, items[i]) } } diff --git a/module/metrics.go b/module/metrics.go index cd7e5746df8..2b889b98c44 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -1,10 +1,12 @@ package module import ( + "context" "time" "github.com/libp2p/go-libp2p/core/peer" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" + httpmetrics "github.com/slok/go-http-metrics/metrics" "github.com/onflow/flow-go/model/chainsync" "github.com/onflow/flow-go/model/cluster" @@ -40,6 +42,10 @@ type NetworkSecurityMetrics interface { // OnRateLimitedPeer tracks the number of rate limited unicast messages seen on the network. OnRateLimitedPeer(pid peer.ID, role, msgType, topic, reason string) + + // OnViolationReportSkipped tracks the number of slashing violations consumer violations that were not + // reported for misbehavior when the identity of the sender not known. + OnViolationReportSkipped() } // GossipSubRouterMetrics encapsulates the metrics collectors for GossipSubRouter module of the networking layer. @@ -112,6 +118,7 @@ type GossipSubMetrics interface { GossipSubScoringMetrics GossipSubRouterMetrics GossipSubLocalMeshMetrics + GossipSubRpcValidationInspectorMetrics } type LibP2PMetrics interface { @@ -148,6 +155,20 @@ type GossipSubScoringMetrics interface { SetWarningStateCount(uint) } +// GossipSubRpcValidationInspectorMetrics encapsulates the metrics collectors for the gossipsub rpc validation control message inspectors. +type GossipSubRpcValidationInspectorMetrics interface { + // BlockingPreProcessingStarted increments the metric tracking the number of messages being pre-processed by the rpc validation inspector. + BlockingPreProcessingStarted(msgType string, sampleSize uint) + // BlockingPreProcessingFinished tracks the time spent by the rpc validation inspector to pre-process a message and decrements the metric tracking + // the number of messages being pre-processed by the rpc validation inspector. + BlockingPreProcessingFinished(msgType string, sampleSize uint, duration time.Duration) + // AsyncProcessingStarted increments the metric tracking the number of inspect message request being processed by workers in the rpc validator worker pool. + AsyncProcessingStarted(msgType string) + // AsyncProcessingFinished tracks the time spent by a rpc validation inspector worker to process an inspect message request asynchronously and decrements the metric tracking + // the number of inspect message requests being processed asynchronously by the rpc validation inspector workers. + AsyncProcessingFinished(msgType string, duration time.Duration) +} + // NetworkInboundQueueMetrics encapsulates the metrics collectors for the inbound queue of the networking layer. type NetworkInboundQueueMetrics interface { @@ -164,6 +185,9 @@ type NetworkInboundQueueMetrics interface { // NetworkCoreMetrics encapsulates the metrics collectors for the core networking layer functionality. type NetworkCoreMetrics interface { NetworkInboundQueueMetrics + AlspMetrics + NetworkSecurityMetrics + // OutboundMessageSent collects metrics related to a message sent by the node. OutboundMessageSent(sizeBytes int, topic string, protocol string, messageType string) // InboundMessageReceived collects metrics related to a message received by the node. @@ -190,10 +214,21 @@ type LibP2PConnectionMetrics interface { InboundConnections(connectionCount uint) } +// AlspMetrics encapsulates the metrics collectors for the Application Layer Spam Prevention (ALSP) module, which +// is part of the networking layer. ALSP is responsible to prevent spam attacks on the application layer messages that +// appear to be valid for the networking layer but carry on a malicious intent on the application layer (i.e., Flow protocols). +type AlspMetrics interface { + // OnMisbehaviorReported is called when a misbehavior is reported by the application layer to ALSP. + // An engine detecting a spamming-related misbehavior reports it to the ALSP module. + // Args: + // - channel: the channel on which the misbehavior was reported + // - misbehaviorType: the type of misbehavior reported + OnMisbehaviorReported(channel string, misbehaviorType string) +} + // NetworkMetrics is the blanket abstraction that encapsulates the metrics collectors for the networking layer. type NetworkMetrics interface { LibP2PMetrics - NetworkSecurityMetrics NetworkCoreMetrics } @@ -311,6 +346,26 @@ type HotstuffMetrics interface { // PayloadProductionDuration measures the time which the HotStuff's core logic // spends in the module.Builder component, i.e. the with generating block payloads. PayloadProductionDuration(duration time.Duration) + + // TimeoutCollectorsRange collects information from the node's `TimeoutAggregator` component. + // Specifically, it measurers the number of views for which we are currently collecting timeouts + // (i.e. the number of `TimeoutCollector` instances we are maintaining) and their lowest/highest view. + TimeoutCollectorsRange(lowestRetainedView uint64, newestViewCreatedCollector uint64, activeCollectors int) +} + +type CruiseCtlMetrics interface { + + // PIDError measures the current error values for the proportional, integration, + // and derivative terms of the PID controller. + PIDError(p, i, d float64) + + // TargetProposalDuration measures the current value of the Block Time Controller output: + // the target duration from parent to child proposal. + TargetProposalDuration(duration time.Duration) + + // ControllerOutput measures the output of the cruise control PID controller. + // Concretely, this is the quantity to subtract from the baseline view duration. + ControllerOutput(duration time.Duration) } type CollectionMetrics interface { @@ -556,7 +611,14 @@ type ExecutionDataPrunerMetrics interface { Pruned(height uint64, duration time.Duration) } -type AccessMetrics interface { +type RestMetrics interface { + // Example recorder taken from: + // https://github.com/slok/go-http-metrics/blob/master/metrics/prometheus/prometheus.go + httpmetrics.Recorder + AddTotalRequests(ctx context.Context, method string, routeName string) +} + +type GRPCConnectionPoolMetrics interface { // TotalConnectionsInPool updates the number connections to collection/execution nodes stored in the pool, and the size of the pool TotalConnectionsInPool(connectionCount uint, connectionPoolSize uint) @@ -579,6 +641,19 @@ type AccessMetrics interface { ConnectionFromPoolEvicted() } +type AccessMetrics interface { + RestMetrics + GRPCConnectionPoolMetrics + TransactionMetrics + BackendScriptsMetrics + + // UpdateExecutionReceiptMaxHeight is called whenever we store an execution receipt from a block from a newer height + UpdateExecutionReceiptMaxHeight(height uint64) + + // UpdateLastFullBlockHeight tracks the height of the last block for which all collections were received + UpdateLastFullBlockHeight(height uint64) +} + type ExecutionResultStats struct { ComputationUsed uint64 MemoryUsed uint64 @@ -634,9 +709,13 @@ type ExecutionMetrics interface { ExecutionCollectionExecuted(dur time.Duration, stats ExecutionResultStats) // ExecutionTransactionExecuted reports stats on executing a single transaction - ExecutionTransactionExecuted(dur time.Duration, - compUsed, memoryUsed, actualMemoryUsed uint64, - eventCounts, eventSize int, + ExecutionTransactionExecuted( + dur time.Duration, + numTxnConflictRetries int, + compUsed uint64, + memoryUsed uint64, + eventCounts int, + eventSize int, failed bool) // ExecutionChunkDataPackGenerated reports stats on chunk data pack generation @@ -666,11 +745,15 @@ type ExecutionMetrics interface { type BackendScriptsMetrics interface { // Record the round trip time while executing a script ScriptExecuted(dur time.Duration, size int) + + // ScriptExecutionErrorOnExecutionNode records script execution failures on Execution Nodes + ScriptExecutionErrorOnArchiveNode() + + // ScriptExecutionErrorOnArchiveNode records script execution failures in Archive Nodes + ScriptExecutionErrorOnExecutionNode() } type TransactionMetrics interface { - BackendScriptsMetrics - // Record the round trip time while getting a transaction result TransactionResultFetched(dur time.Duration, size int) @@ -690,9 +773,6 @@ type TransactionMetrics interface { // TransactionSubmissionFailed should be called whenever we try to submit a transaction and it fails TransactionSubmissionFailed() - - // UpdateExecutionReceiptMaxHeight is called whenever we store an execution receipt from a block from a newer height - UpdateExecutionReceiptMaxHeight(height uint64) } type PingMetrics interface { diff --git a/module/metrics/access.go b/module/metrics/access.go index 4dcfc6e6f38..1116f87f433 100644 --- a/module/metrics/access.go +++ b/module/metrics/access.go @@ -3,9 +3,36 @@ package metrics import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/counters" ) +type AccessCollectorOpts func(*AccessCollector) + +func WithTransactionMetrics(m module.TransactionMetrics) AccessCollectorOpts { + return func(ac *AccessCollector) { + ac.TransactionMetrics = m + } +} + +func WithBackendScriptsMetrics(m module.BackendScriptsMetrics) AccessCollectorOpts { + return func(ac *AccessCollector) { + ac.BackendScriptsMetrics = m + } +} + +func WithRestMetrics(m module.RestMetrics) AccessCollectorOpts { + return func(ac *AccessCollector) { + ac.RestMetrics = m + } +} + type AccessCollector struct { + module.RestMetrics + module.TransactionMetrics + module.BackendScriptsMetrics + connectionReused prometheus.Counter connectionsInPool *prometheus.GaugeVec connectionAdded prometheus.Counter @@ -13,9 +40,16 @@ type AccessCollector struct { connectionInvalidated prometheus.Counter connectionUpdated prometheus.Counter connectionEvicted prometheus.Counter + lastFullBlockHeight prometheus.Gauge + maxReceiptHeight prometheus.Gauge + + // used to skip heights that are lower than the current max height + maxReceiptHeightValue counters.StrictMonotonousCounter } -func NewAccessCollector() *AccessCollector { +var _ module.AccessMetrics = (*AccessCollector)(nil) + +func NewAccessCollector(opts ...AccessCollectorOpts) *AccessCollector { ac := &AccessCollector{ connectionReused: promauto.NewCounter(prometheus.CounterOpts{ Name: "connection_reused", @@ -59,6 +93,23 @@ func NewAccessCollector() *AccessCollector { Subsystem: subsystemConnectionPool, Help: "counter for the number of times a cached connection is evicted from the connection pool", }), + lastFullBlockHeight: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "last_full_finalized_block_height", + Namespace: namespaceAccess, + Subsystem: subsystemIngestion, + Help: "gauge to track the highest consecutive finalized block height with all collections indexed", + }), + maxReceiptHeight: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "max_receipt_height", + Namespace: namespaceAccess, + Subsystem: subsystemIngestion, + Help: "gauge to track the maximum block height of execution receipts received", + }), + maxReceiptHeightValue: counters.NewMonotonousCounter(0), + } + + for _, opt := range opts { + opt(ac) } return ac @@ -92,3 +143,13 @@ func (ac *AccessCollector) ConnectionFromPoolUpdated() { func (ac *AccessCollector) ConnectionFromPoolEvicted() { ac.connectionEvicted.Inc() } + +func (ac *AccessCollector) UpdateLastFullBlockHeight(height uint64) { + ac.lastFullBlockHeight.Set(float64(height)) +} + +func (ac *AccessCollector) UpdateExecutionReceiptMaxHeight(height uint64) { + if ac.maxReceiptHeightValue.Set(height) { + ac.maxReceiptHeight.Set(float64(height)) + } +} diff --git a/module/metrics/alsp.go b/module/metrics/alsp.go new file mode 100644 index 00000000000..3d5dc2bc510 --- /dev/null +++ b/module/metrics/alsp.go @@ -0,0 +1,49 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/onflow/flow-go/module" +) + +// AlspMetrics is a struct that contains all the metrics related to the ALSP module. +// It implements the AlspMetrics interface. +// AlspMetrics encapsulates the metrics collectors for the Application Layer Spam Prevention (ALSP) module, which +// is part of the networking layer. ALSP is responsible to prevent spam attacks on the application layer messages that +// appear to be valid for the networking layer but carry on a malicious intent on the application layer (i.e., Flow protocols). +type AlspMetrics struct { + reportedMisbehaviorCount *prometheus.CounterVec +} + +var _ module.AlspMetrics = (*AlspMetrics)(nil) + +// NewAlspMetrics creates a new AlspMetrics struct. It initializes the metrics collectors for the ALSP module. +// Returns: +// - a pointer to the AlspMetrics struct. +func NewAlspMetrics() *AlspMetrics { + alsp := &AlspMetrics{} + + alsp.reportedMisbehaviorCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemAlsp, + Name: "reported_misbehavior_total", + Help: "number of reported spamming misbehavior received by alsp", + }, []string{LabelChannel, LabelMisbehavior}, + ) + + return alsp +} + +// OnMisbehaviorReported is called when a misbehavior is reported by the application layer to ALSP. +// An engine detecting a spamming-related misbehavior reports it to the ALSP module. It increases +// the counter vector of reported misbehavior. +// Args: +// - channel: the channel on which the misbehavior was reported +// - misbehaviorType: the type of misbehavior reported +func (a *AlspMetrics) OnMisbehaviorReported(channel string, misbehaviorType string) { + a.reportedMisbehaviorCount.With(prometheus.Labels{ + LabelChannel: channel, + LabelMisbehavior: misbehaviorType, + }).Inc() +} diff --git a/module/metrics/cruisectl.go b/module/metrics/cruisectl.go new file mode 100644 index 00000000000..7d56e762d50 --- /dev/null +++ b/module/metrics/cruisectl.go @@ -0,0 +1,67 @@ +package metrics + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// CruiseCtlMetrics captures metrics about the Block Rate Controller, which adjusts +// the proposal duration to attain a target epoch switchover time. +type CruiseCtlMetrics struct { + proportionalErr prometheus.Gauge + integralErr prometheus.Gauge + derivativeErr prometheus.Gauge + targetProposalDur prometheus.Gauge + controllerOutput prometheus.Gauge +} + +func NewCruiseCtlMetrics() *CruiseCtlMetrics { + return &CruiseCtlMetrics{ + proportionalErr: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "proportional_err_s", + Namespace: namespaceConsensus, + Subsystem: subsystemCruiseCtl, + Help: "The current proportional error measured by the controller", + }), + integralErr: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "integral_err_s", + Namespace: namespaceConsensus, + Subsystem: subsystemCruiseCtl, + Help: "The current integral error measured by the controller", + }), + derivativeErr: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "derivative_err_per_s", + Namespace: namespaceConsensus, + Subsystem: subsystemCruiseCtl, + Help: "The current derivative error measured by the controller", + }), + targetProposalDur: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "target_proposal_dur_s", + Namespace: namespaceConsensus, + Subsystem: subsystemCruiseCtl, + Help: "The current target duration from parent to child proposal", + }), + controllerOutput: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "controller_output_s", + Namespace: namespaceConsensus, + Subsystem: subsystemCruiseCtl, + Help: "The most recent output of the controller; the adjustment to subtract from the baseline proposal duration", + }), + } +} + +func (c *CruiseCtlMetrics) PIDError(p, i, d float64) { + c.proportionalErr.Set(p) + c.integralErr.Set(i) + c.derivativeErr.Set(d) +} + +func (c *CruiseCtlMetrics) TargetProposalDuration(duration time.Duration) { + c.targetProposalDur.Set(duration.Seconds()) +} + +func (c *CruiseCtlMetrics) ControllerOutput(duration time.Duration) { + c.controllerOutput.Set(duration.Seconds()) +} diff --git a/module/metrics/example/verification/main.go b/module/metrics/example/verification/main.go index e915f1a1e89..b0db36320b9 100644 --- a/module/metrics/example/verification/main.go +++ b/module/metrics/example/verification/main.go @@ -10,6 +10,7 @@ import ( "github.com/rs/zerolog" vertestutils "github.com/onflow/flow-go/engine/verification/utils/unittest" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/buffer" "github.com/onflow/flow-go/module/mempool/stdmap" "github.com/onflow/flow-go/module/metrics" @@ -199,7 +200,10 @@ func demo() { tryRandomCall(func() { block := unittest.BlockFixture() - pendingBlocks.Add(unittest.IdentifierFixture(), &block) + pendingBlocks.Add(flow.Slashable[*flow.Block]{ + OriginID: unittest.IdentifierFixture(), + Message: &block, + }) }) // adds a synthetic 1 s delay for verification duration diff --git a/module/metrics/execution.go b/module/metrics/execution.go index 2912e842472..8d7b155791e 100644 --- a/module/metrics/execution.go +++ b/module/metrics/execution.go @@ -60,9 +60,8 @@ type ExecutionCollector struct { transactionCheckTime prometheus.Histogram transactionInterpretTime prometheus.Histogram transactionExecutionTime prometheus.Histogram - transactionMemoryUsage prometheus.Histogram + transactionConflictRetries prometheus.Histogram transactionMemoryEstimate prometheus.Histogram - transactionMemoryDifference prometheus.Histogram transactionComputationUsed prometheus.Histogram transactionEmittedEvents prometheus.Histogram transactionEventSize prometheus.Histogram @@ -390,20 +389,20 @@ func NewExecutionCollector(tracer module.Tracer) *ExecutionCollector { Buckets: prometheus.ExponentialBuckets(2, 2, 10), }) - transactionComputationUsed := promauto.NewHistogram(prometheus.HistogramOpts{ + transactionConflictRetries := promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceExecution, Subsystem: subsystemRuntime, - Name: "transaction_computation_used", - Help: "the total amount of computation used by a transaction", - Buckets: []float64{50, 100, 500, 1000, 5000, 10000}, + Name: "transaction_conflict_retries", + Help: "the number of conflict retries needed to successfully commit a transaction. If retry count is high, consider reducing concurrency", + Buckets: []float64{0, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100}, }) - transactionMemoryUsage := promauto.NewHistogram(prometheus.HistogramOpts{ + transactionComputationUsed := promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceExecution, Subsystem: subsystemRuntime, - Name: "transaction_memory_usage", - Help: "the total amount of memory allocated by a transaction", - Buckets: []float64{100_000, 1_000_000, 10_000_000, 50_000_000, 100_000_000, 500_000_000, 1_000_000_000}, + Name: "transaction_computation_used", + Help: "the total amount of computation used by a transaction", + Buckets: []float64{50, 100, 500, 1000, 5000, 10000}, }) transactionMemoryEstimate := promauto.NewHistogram(prometheus.HistogramOpts{ @@ -414,14 +413,6 @@ func NewExecutionCollector(tracer module.Tracer) *ExecutionCollector { Buckets: []float64{1_000_000, 10_000_000, 100_000_000, 1_000_000_000, 5_000_000_000, 10_000_000_000, 50_000_000_000, 100_000_000_000}, }) - transactionMemoryDifference := promauto.NewHistogram(prometheus.HistogramOpts{ - Namespace: namespaceExecution, - Subsystem: subsystemRuntime, - Name: "transaction_memory_difference", - Help: "the difference in actual memory usage and estimate for a transaction", - Buckets: []float64{-1, 0, 10_000_000, 100_000_000, 1_000_000_000}, - }) - transactionEmittedEvents := promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceExecution, Subsystem: subsystemRuntime, @@ -573,10 +564,9 @@ func NewExecutionCollector(tracer module.Tracer) *ExecutionCollector { transactionCheckTime: transactionCheckTime, transactionInterpretTime: transactionInterpretTime, transactionExecutionTime: transactionExecutionTime, + transactionConflictRetries: transactionConflictRetries, transactionComputationUsed: transactionComputationUsed, - transactionMemoryUsage: transactionMemoryUsage, transactionMemoryEstimate: transactionMemoryEstimate, - transactionMemoryDifference: transactionMemoryDifference, transactionEmittedEvents: transactionEmittedEvents, transactionEventSize: transactionEventSize, scriptExecutionTime: scriptExecutionTime, @@ -738,16 +728,18 @@ func (ec *ExecutionCollector) ExecutionBlockCachedPrograms(programs int) { // TransactionExecuted reports stats for executing a transaction func (ec *ExecutionCollector) ExecutionTransactionExecuted( dur time.Duration, - compUsed, memoryUsed, actualMemoryUsed uint64, - eventCounts, eventSize int, + numConflictRetries int, + compUsed uint64, + memoryUsed uint64, + eventCounts int, + eventSize int, failed bool, ) { ec.totalExecutedTransactionsCounter.Inc() ec.transactionExecutionTime.Observe(float64(dur.Milliseconds())) + ec.transactionConflictRetries.Observe(float64(numConflictRetries)) ec.transactionComputationUsed.Observe(float64(compUsed)) - ec.transactionMemoryUsage.Observe(float64(actualMemoryUsed)) ec.transactionMemoryEstimate.Observe(float64(memoryUsed)) - ec.transactionMemoryDifference.Observe(float64(memoryUsed) - float64(actualMemoryUsed)) ec.transactionEmittedEvents.Observe(float64(eventCounts)) ec.transactionEventSize.Observe(float64(eventSize)) if failed { diff --git a/module/metrics/gossipsub_rpc_validation_inspector.go b/module/metrics/gossipsub_rpc_validation_inspector.go new file mode 100644 index 00000000000..f4d79d4121d --- /dev/null +++ b/module/metrics/gossipsub_rpc_validation_inspector.go @@ -0,0 +1,86 @@ +package metrics + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/onflow/flow-go/module" +) + +// GossipSubRpcValidationInspectorMetrics metrics collector for the gossipsub RPC validation inspector. +type GossipSubRpcValidationInspectorMetrics struct { + prefix string + rpcCtrlMsgInBlockingPreProcessingGauge *prometheus.GaugeVec + rpcCtrlMsgBlockingProcessingTimeHistogram *prometheus.HistogramVec + rpcCtrlMsgInAsyncPreProcessingGauge *prometheus.GaugeVec + rpcCtrlMsgAsyncProcessingTimeHistogram *prometheus.HistogramVec +} + +var _ module.GossipSubRpcValidationInspectorMetrics = (*GossipSubRpcValidationInspectorMetrics)(nil) + +// NewGossipSubRPCValidationInspectorMetrics returns a new *GossipSubRpcValidationInspectorMetrics. +func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValidationInspectorMetrics { + gc := &GossipSubRpcValidationInspectorMetrics{prefix: prefix} + gc.rpcCtrlMsgInBlockingPreProcessingGauge = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "control_message_in_blocking_preprocess_total", + Help: "the number of rpc control messages currently being pre-processed", + }, []string{LabelCtrlMsgType}, + ) + gc.rpcCtrlMsgBlockingProcessingTimeHistogram = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_control_message_validator_blocking_preprocessing_time_seconds", + Help: "duration [seconds; measured with float64 precision] of how long the rpc control message validator blocked pre-processing an rpc control message", + Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 7.5, 10, 20}, + }, []string{LabelCtrlMsgType}, + ) + gc.rpcCtrlMsgInAsyncPreProcessingGauge = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "control_messages_in_async_processing_total", + Help: "the number of rpc control messages currently being processed asynchronously by workers from the rpc validator worker pool", + }, []string{LabelCtrlMsgType}, + ) + gc.rpcCtrlMsgAsyncProcessingTimeHistogram = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_control_message_validator_async_processing_time_seconds", + Help: "duration [seconds; measured with float64 precision] of how long it takes rpc control message validator to asynchronously process a rpc message", + Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 7.5, 10, 20}, + }, []string{LabelCtrlMsgType}, + ) + + return gc +} + +// BlockingPreProcessingStarted increments the metric tracking the number of messages being pre-processed by the rpc validation inspector. +func (c *GossipSubRpcValidationInspectorMetrics) BlockingPreProcessingStarted(msgType string, sampleSize uint) { + c.rpcCtrlMsgInBlockingPreProcessingGauge.WithLabelValues(msgType).Add(float64(sampleSize)) +} + +// BlockingPreProcessingFinished tracks the time spent by the rpc validation inspector to pre-process a message and decrements the metric tracking +// the number of messages being processed by the rpc validation inspector. +func (c *GossipSubRpcValidationInspectorMetrics) BlockingPreProcessingFinished(msgType string, sampleSize uint, duration time.Duration) { + c.rpcCtrlMsgInBlockingPreProcessingGauge.WithLabelValues(msgType).Sub(float64(sampleSize)) + c.rpcCtrlMsgBlockingProcessingTimeHistogram.WithLabelValues(msgType).Observe(duration.Seconds()) +} + +// AsyncProcessingStarted increments the metric tracking the number of messages being processed asynchronously by the rpc validation inspector. +func (c *GossipSubRpcValidationInspectorMetrics) AsyncProcessingStarted(msgType string) { + c.rpcCtrlMsgInAsyncPreProcessingGauge.WithLabelValues(msgType).Inc() +} + +// AsyncProcessingFinished tracks the time spent by the rpc validation inspector to process a message asynchronously and decrements the metric tracking +// the number of messages being processed asynchronously by the rpc validation inspector. +func (c *GossipSubRpcValidationInspectorMetrics) AsyncProcessingFinished(msgType string, duration time.Duration) { + c.rpcCtrlMsgInAsyncPreProcessingGauge.WithLabelValues(msgType).Dec() + c.rpcCtrlMsgAsyncProcessingTimeHistogram.WithLabelValues(msgType).Observe(duration.Seconds()) +} diff --git a/module/metrics/herocache.go b/module/metrics/herocache.go index c5d031d6331..f82cd84bb57 100644 --- a/module/metrics/herocache.go +++ b/module/metrics/herocache.go @@ -6,6 +6,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network" ) const subsystemHeroCache = "hero_cache" @@ -32,12 +33,58 @@ type HeroCacheCollector struct { type HeroCacheMetricsRegistrationFunc func(uint64) module.HeroCacheMetrics -func NetworkReceiveCacheMetricsFactory(registrar prometheus.Registerer) *HeroCacheCollector { - return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingReceiveCache, registrar) +// HeroCacheMetricsFactory is a factory method to create a new HeroCacheCollector for a specific cache +// with a specific namespace and a specific name. +// Args: +// - namespace: the namespace of the cache +// - cacheName: the name of the cache +type HeroCacheMetricsFactory func(namespace string, cacheName string) module.HeroCacheMetrics + +// NewHeroCacheMetricsFactory creates a new HeroCacheMetricsFactory for the given registrar. It allows to defer the +// registration of the metrics to the point where the cache is created without exposing the registrar to the cache. +// Args: +// - registrar: the prometheus registrar to register the metrics with +// Returns: +// - a HeroCacheMetricsFactory that can be used to create a new HeroCacheCollector for a specific cache +func NewHeroCacheMetricsFactory(registrar prometheus.Registerer) HeroCacheMetricsFactory { + return func(namespace string, cacheName string) module.HeroCacheMetrics { + return NewHeroCacheCollector(namespace, cacheName, registrar) + } +} + +// NewNoopHeroCacheMetricsFactory creates a new HeroCacheMetricsFactory that returns a noop collector. +// This is useful for tests that don't want to register metrics. +// Args: +// - none +// Returns: +// - a HeroCacheMetricsFactory that returns a noop collector +func NewNoopHeroCacheMetricsFactory() HeroCacheMetricsFactory { + return func(string, string) module.HeroCacheMetrics { + return NewNoopCollector() + } +} + +func NetworkReceiveCacheMetricsFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + r := ResourceNetworkingReceiveCache + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) } -func PublicNetworkReceiveCacheMetricsFactory(registrar prometheus.Registerer) *HeroCacheCollector { - return NewHeroCacheCollector(namespaceNetwork, ResourcePublicNetworkingReceiveCache, registrar) +// DisallowListCacheMetricsFactory is the factory method for creating a new HeroCacheCollector for the disallow list cache. +// The disallow-list cache is used to keep track of peers that are disallow-listed and the reasons for it. +// Args: +// - f: the HeroCacheMetricsFactory to create the collector +// - networkingType: the networking type of the cache, i.e., whether it is used for the public or the private network +// Returns: +// - a HeroCacheMetrics for the disallow list cache +func DisallowListCacheMetricsFactory(f HeroCacheMetricsFactory, networkingType network.NetworkingType) module.HeroCacheMetrics { + r := ResourceNetworkingDisallowListCache + if networkingType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) } func NetworkDnsTxtCacheMetricsFactory(registrar prometheus.Registerer) *HeroCacheCollector { @@ -64,22 +111,74 @@ func DisallowListNotificationQueueMetricFactory(registrar prometheus.Registerer) return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingDisallowListNotificationQueue, registrar) } -func GossipSubRPCValidationInspectorQueueMetricFactory(publicNetwork bool, registrar prometheus.Registerer) *HeroCacheCollector { - if publicNetwork { - return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingPublicRpcValidationInspectorQueue, registrar) +func ApplicationLayerSpamRecordCacheMetricFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + r := ResourceNetworkingApplicationLayerSpamRecordCache + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) } - return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingRpcValidationInspectorQueue, registrar) + + return f(namespaceNetwork, r) } -func GossipSubRPCMetricsObserverInspectorQueueMetricFactory(publicNetwork bool, registrar prometheus.Registerer) *HeroCacheCollector { - if publicNetwork { - return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingPublicRpcMetricsObserverInspectorQueue, registrar) +func ApplicationLayerSpamRecordQueueMetricsFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + r := ResourceNetworkingApplicationLayerSpamReportQueue + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) } - return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingRpcMetricsObserverInspectorQueue, registrar) + return f(namespaceNetwork, r) } -func RpcInspectorNotificationQueueMetricFactory(registrar prometheus.Registerer) *HeroCacheCollector { - return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingRpcInspectorNotificationQueue, registrar) +func GossipSubRPCMetricsObserverInspectorQueueMetricFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + // we don't use the public prefix for the metrics here for sake of backward compatibility of metric name. + r := ResourceNetworkingRpcMetricsObserverInspectorQueue + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) +} + +func GossipSubRPCInspectorQueueMetricFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + // we don't use the public prefix for the metrics here for sake of backward compatibility of metric name. + r := ResourceNetworkingRpcValidationInspectorQueue + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) +} + +func GossipSubRPCSentTrackerMetricFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + // we don't use the public prefix for the metrics here for sake of backward compatibility of metric name. + r := ResourceNetworkingRPCSentTrackerCache + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) +} + +func GossipSubRPCSentTrackerQueueMetricFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + // we don't use the public prefix for the metrics here for sake of backward compatibility of metric name. + r := ResourceNetworkingRPCSentTrackerQueue + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) +} + +func RpcInspectorNotificationQueueMetricFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + r := ResourceNetworkingRpcInspectorNotificationQueue + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) +} + +func GossipSubRPCInspectorClusterPrefixedCacheMetricFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + // we don't use the public prefix for the metrics here for sake of backward compatibility of metric name. + r := ResourceNetworkingRpcClusterPrefixReceivedCache + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) } func CollectionNodeTransactionsCacheMetrics(registrar prometheus.Registerer, epoch uint64) *HeroCacheCollector { @@ -94,6 +193,16 @@ func AccessNodeExecutionDataCacheMetrics(registrar prometheus.Registerer) *HeroC return NewHeroCacheCollector(namespaceAccess, ResourceExecutionDataCache, registrar) } +// PrependPublicPrefix prepends the string "public" to the given string. +// This is used to distinguish between public and private metrics. +// Args: +// - str: the string to prepend, example: "my_metric" +// Returns: +// - the prepended string, example: "public_my_metric" +func PrependPublicPrefix(str string) string { + return fmt.Sprintf("%s_%s", "public", str) +} + func NewHeroCacheCollector(nameSpace string, cacheName string, registrar prometheus.Registerer) *HeroCacheCollector { histogramNormalizedBucketSlotAvailable := prometheus.NewHistogram(prometheus.HistogramOpts{ diff --git a/module/metrics/hotstuff.go b/module/metrics/hotstuff.go index 258c15ddec0..df843cdeaa8 100644 --- a/module/metrics/hotstuff.go +++ b/module/metrics/hotstuff.go @@ -43,6 +43,8 @@ type HotstuffCollector struct { signerComputationsDuration prometheus.Histogram validatorComputationsDuration prometheus.Histogram payloadProductionDuration prometheus.Histogram + timeoutCollectorsRange *prometheus.GaugeVec + numberOfActiveCollectors prometheus.Gauge } var _ module.HotstuffMetrics = (*HotstuffCollector)(nil) @@ -185,6 +187,20 @@ func NewHotstuffCollector(chain flow.ChainID) *HotstuffCollector { Buckets: []float64{0.02, 0.05, 0.1, 0.2, 0.5, 1, 2}, ConstLabels: prometheus.Labels{LabelChain: chain.String()}, }), + timeoutCollectorsRange: promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "timeout_collectors_range", + Namespace: namespaceConsensus, + Subsystem: subsystemHotstuff, + Help: "lowest and highest views that we are maintaining TimeoutCollectors for", + ConstLabels: prometheus.Labels{LabelChain: chain.String()}, + }, []string{"prefix"}), + numberOfActiveCollectors: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "active_collectors", + Namespace: namespaceConsensus, + Subsystem: subsystemHotstuff, + Help: "number of active TimeoutCollectors that the TimeoutAggregator component currently maintains", + ConstLabels: prometheus.Labels{LabelChain: chain.String()}, + }), } return hc @@ -277,3 +293,12 @@ func (hc *HotstuffCollector) ValidatorProcessingDuration(duration time.Duration) func (hc *HotstuffCollector) PayloadProductionDuration(duration time.Duration) { hc.payloadProductionDuration.Observe(duration.Seconds()) // unit: seconds; with float64 precision } + +// TimeoutCollectorsRange collects information from the node's `TimeoutAggregator` component. +// Specifically, it measurers the number of views for which we are currently collecting timeouts +// (i.e. the number of `TimeoutCollector` instances we are maintaining) and their lowest/highest view. +func (hc *HotstuffCollector) TimeoutCollectorsRange(lowestRetainedView uint64, newestViewCreatedCollector uint64, activeCollectors int) { + hc.timeoutCollectorsRange.WithLabelValues("lowest_view_of_active_timeout_collectors").Set(float64(lowestRetainedView)) + hc.timeoutCollectorsRange.WithLabelValues("newest_view_of_active_timeout_collectors").Set(float64(newestViewCreatedCollector)) + hc.numberOfActiveCollectors.Set(float64(activeCollectors)) +} diff --git a/module/metrics/labels.go b/module/metrics/labels.go index eb436a8d934..c67b9283f38 100644 --- a/module/metrics/labels.go +++ b/module/metrics/labels.go @@ -18,6 +18,12 @@ const ( LabelConnectionDirection = "direction" LabelConnectionUseFD = "usefd" // whether the connection is using a file descriptor LabelSuccess = "success" + LabelCtrlMsgType = "control_message" + LabelMisbehavior = "misbehavior" + LabelHandler = "handler" + LabelStatusCode = "code" + LabelMethod = "method" + LabelService = "service" ) const ( @@ -41,75 +47,80 @@ const ( ) const ( - ResourceUndefined = "undefined" - ResourceProposal = "proposal" - ResourceHeader = "header" - ResourceFinalizedHeight = "finalized_height" - ResourceIndex = "index" - ResourceIdentity = "identity" - ResourceGuarantee = "guarantee" - ResourceResult = "result" - ResourceResultApprovals = "result_approvals" - ResourceReceipt = "receipt" - ResourceQC = "qc" - ResourceMyReceipt = "my_receipt" - ResourceCollection = "collection" - ResourceApproval = "approval" - ResourceSeal = "seal" - ResourcePendingIncorporatedSeal = "pending_incorporated_seal" - ResourceCommit = "commit" - ResourceTransaction = "transaction" - ResourceClusterPayload = "cluster_payload" - ResourceClusterProposal = "cluster_proposal" - ResourceProcessedResultID = "processed_result_id" // verification node, finder engine // TODO: remove finder engine labels - ResourceDiscardedResultID = "discarded_result_id" // verification node, finder engine - ResourcePendingReceipt = "pending_receipt" // verification node, finder engine - ResourceReceiptIDsByResult = "receipt_ids_by_result" // verification node, finder engine - ResourcePendingReceiptIDsByBlock = "pending_receipt_ids_by_block" // verification node, finder engine - ResourcePendingResult = "pending_result" // verification node, match engine - ResourceChunkIDsByResult = "chunk_ids_by_result" // verification node, match engine - ResourcePendingChunk = "pending_chunk" // verification node, match engine - ResourcePendingBlock = "pending_block" // verification node, match engine - ResourceCachedReceipt = "cached_receipt" // verification node, finder engine - ResourceCachedBlockID = "cached_block_id" // verification node, finder engine - ResourceChunkStatus = "chunk_status" // verification node, fetcher engine - ResourceChunkRequest = "chunk_request" // verification node, requester engine - ResourceChunkConsumer = "chunk_consumer_jobs" // verification node - ResourceBlockConsumer = "block_consumer_jobs" // verification node - ResourceEpochSetup = "epoch_setup" - ResourceEpochCommit = "epoch_commit" - ResourceEpochStatus = "epoch_status" - ResourceNetworkingReceiveCache = "networking_received_message" // networking layer - ResourcePublicNetworkingReceiveCache = "public_networking_received_message" // networking layer - ResourceNetworkingDnsIpCache = "networking_dns_ip_cache" // networking layer - ResourceNetworkingDnsTxtCache = "networking_dns_txt_cache" // networking layer - ResourceNetworkingDisallowListNotificationQueue = "networking_disallow_list_notification_queue" - ResourceNetworkingRpcInspectorNotificationQueue = "networking_rpc_inspector_notification_queue" - ResourceNetworkingRpcValidationInspectorQueue = "networking_rpc_validation_inspector_queue" - ResourceNetworkingRpcMetricsObserverInspectorQueue = "networking_rpc_metrics_observer_inspector_queue" - ResourceNetworkingPublicRpcValidationInspectorQueue = "networking_public_rpc_validation_inspector_queue" - ResourceNetworkingPublicRpcMetricsObserverInspectorQueue = "networking_public_rpc_metrics_observer_inspector_queue" + ResourceUndefined = "undefined" + ResourceProposal = "proposal" + ResourceHeader = "header" + ResourceFinalizedHeight = "finalized_height" + ResourceIndex = "index" + ResourceIdentity = "identity" + ResourceGuarantee = "guarantee" + ResourceResult = "result" + ResourceResultApprovals = "result_approvals" + ResourceReceipt = "receipt" + ResourceQC = "qc" + ResourceMyReceipt = "my_receipt" + ResourceCollection = "collection" + ResourceApproval = "approval" + ResourceSeal = "seal" + ResourcePendingIncorporatedSeal = "pending_incorporated_seal" + ResourceCommit = "commit" + ResourceTransaction = "transaction" + ResourceClusterPayload = "cluster_payload" + ResourceClusterProposal = "cluster_proposal" + ResourceProcessedResultID = "processed_result_id" // verification node, finder engine // TODO: remove finder engine labels + ResourceDiscardedResultID = "discarded_result_id" // verification node, finder engine + ResourcePendingReceipt = "pending_receipt" // verification node, finder engine + ResourceReceiptIDsByResult = "receipt_ids_by_result" // verification node, finder engine + ResourcePendingReceiptIDsByBlock = "pending_receipt_ids_by_block" // verification node, finder engine + ResourcePendingResult = "pending_result" // verification node, match engine + ResourceChunkIDsByResult = "chunk_ids_by_result" // verification node, match engine + ResourcePendingChunk = "pending_chunk" // verification node, match engine + ResourcePendingBlock = "pending_block" // verification node, match engine + ResourceCachedReceipt = "cached_receipt" // verification node, finder engine + ResourceCachedBlockID = "cached_block_id" // verification node, finder engine + ResourceChunkStatus = "chunk_status" // verification node, fetcher engine + ResourceChunkRequest = "chunk_request" // verification node, requester engine + ResourceChunkConsumer = "chunk_consumer_jobs" // verification node + ResourceBlockConsumer = "block_consumer_jobs" // verification node + ResourceEpochSetup = "epoch_setup" + ResourceEpochCommit = "epoch_commit" + ResourceEpochStatus = "epoch_status" + ResourceNetworkingReceiveCache = "networking_received_message" // networking layer + ResourceNetworkingDnsIpCache = "networking_dns_ip_cache" // networking layer + ResourceNetworkingDnsTxtCache = "networking_dns_txt_cache" // networking layer + ResourceNetworkingDisallowListNotificationQueue = "networking_disallow_list_notification_queue" + ResourceNetworkingRpcInspectorNotificationQueue = "networking_rpc_inspector_notification_queue" + ResourceNetworkingRpcValidationInspectorQueue = "networking_rpc_validation_inspector_queue" + ResourceNetworkingRpcMetricsObserverInspectorQueue = "networking_rpc_metrics_observer_inspector_queue" + ResourceNetworkingApplicationLayerSpamRecordCache = "application_layer_spam_record_cache" + ResourceNetworkingApplicationLayerSpamReportQueue = "application_layer_spam_report_queue" + ResourceNetworkingRpcClusterPrefixReceivedCache = "rpc_cluster_prefixed_received_cache" + ResourceNetworkingDisallowListCache = "disallow_list_cache" + ResourceNetworkingRPCSentTrackerCache = "gossipsub_rpc_sent_tracker_cache" + ResourceNetworkingRPCSentTrackerQueue = "gossipsub_rpc_sent_tracker_queue" - ResourceFollowerPendingBlocksCache = "follower_pending_block_cache" // follower engine - ResourceClusterBlockProposalQueue = "cluster_compliance_proposal_queue" // collection node, compliance engine - ResourceTransactionIngestQueue = "ingest_transaction_queue" // collection node, ingest engine - ResourceBeaconKey = "beacon-key" // consensus node, DKG engine - ResourceApprovalQueue = "sealing_approval_queue" // consensus node, sealing engine - ResourceReceiptQueue = "sealing_receipt_queue" // consensus node, sealing engine - ResourceApprovalResponseQueue = "sealing_approval_response_queue" // consensus node, sealing engine - ResourceBlockResponseQueue = "compliance_block_response_queue" // consensus node, compliance engine - ResourceBlockProposalQueue = "compliance_proposal_queue" // consensus node, compliance engine - ResourceBlockVoteQueue = "vote_aggregator_queue" // consensus/collection node, vote aggregator - ResourceTimeoutObjectQueue = "timeout_aggregator_queue" // consensus/collection node, timeout aggregator - ResourceCollectionGuaranteesQueue = "ingestion_col_guarantee_queue" // consensus node, ingestion engine - ResourceChunkDataPack = "chunk_data_pack" // execution node - ResourceChunkDataPackRequests = "chunk_data_pack_request" // execution node - ResourceEvents = "events" // execution node - ResourceServiceEvents = "service_events" // execution node - ResourceTransactionResults = "transaction_results" // execution node - ResourceTransactionResultIndices = "transaction_result_indices" // execution node - ResourceTransactionResultByBlock = "transaction_result_by_block" // execution node - ResourceExecutionDataCache = "execution_data_cache" // access node + ResourceFollowerPendingBlocksCache = "follower_pending_block_cache" // follower engine + ResourceFollowerLoopCertifiedBlocksChannel = "follower_loop_certified_blocks_channel" // follower loop, certified blocks buffered channel + ResourceClusterBlockProposalQueue = "cluster_compliance_proposal_queue" // collection node, compliance engine + ResourceTransactionIngestQueue = "ingest_transaction_queue" // collection node, ingest engine + ResourceBeaconKey = "beacon-key" // consensus node, DKG engine + ResourceDKGMessage = "dkg_private_message" // consensus, DKG messaging engine + ResourceApprovalQueue = "sealing_approval_queue" // consensus node, sealing engine + ResourceReceiptQueue = "sealing_receipt_queue" // consensus node, sealing engine + ResourceApprovalResponseQueue = "sealing_approval_response_queue" // consensus node, sealing engine + ResourceBlockResponseQueue = "compliance_block_response_queue" // consensus node, compliance engine + ResourceBlockProposalQueue = "compliance_proposal_queue" // consensus node, compliance engine + ResourceBlockVoteQueue = "vote_aggregator_queue" // consensus/collection node, vote aggregator + ResourceTimeoutObjectQueue = "timeout_aggregator_queue" // consensus/collection node, timeout aggregator + ResourceCollectionGuaranteesQueue = "ingestion_col_guarantee_queue" // consensus node, ingestion engine + ResourceChunkDataPack = "chunk_data_pack" // execution node + ResourceChunkDataPackRequests = "chunk_data_pack_request" // execution node + ResourceEvents = "events" // execution node + ResourceServiceEvents = "service_events" // execution node + ResourceTransactionResults = "transaction_results" // execution node + ResourceTransactionResultIndices = "transaction_result_indices" // execution node + ResourceTransactionResultByBlock = "transaction_result_by_block" // execution node + ResourceExecutionDataCache = "execution_data_cache" // access node ) const ( diff --git a/module/metrics/namespaces.go b/module/metrics/namespaces.go index cca570b3474..f89f2a530ae 100644 --- a/module/metrics/namespaces.go +++ b/module/metrics/namespaces.go @@ -15,6 +15,7 @@ const ( namespaceExecutionDataSync = "execution_data_sync" namespaceChainsync = "chainsync" namespaceFollowerEngine = "follower" + namespaceRestAPI = "access_rest_api" ) // Network subsystems represent the various layers of networking. @@ -27,6 +28,8 @@ const ( subsystemBitswap = "bitswap" subsystemAuth = "authorization" subsystemRateLimiting = "ratelimit" + subsystemAlsp = "alsp" + subsystemSecurity = "security" ) // Storage subsystems represent the various components of the storage layer. @@ -41,6 +44,7 @@ const ( subsystemTransactionTiming = "transaction_timing" subsystemTransactionSubmission = "transaction_submission" subsystemConnectionPool = "connection_pool" + subsystemHTTP = "http" ) // Observer subsystem @@ -57,6 +61,7 @@ const ( const ( subsystemCompliance = "compliance" subsystemHotstuff = "hotstuff" + subsystemCruiseCtl = "cruisectl" subsystemMatchEngine = "match" ) diff --git a/module/metrics/network.go b/module/metrics/network.go index 4020ebe0f1f..311dbba9f15 100644 --- a/module/metrics/network.go +++ b/module/metrics/network.go @@ -26,6 +26,8 @@ type NetworkCollector struct { *GossipSubMetrics *GossipSubScoreMetrics *GossipSubLocalMeshMetrics + *GossipSubRpcValidationInspectorMetrics + *AlspMetrics outboundMessageSize *prometheus.HistogramVec inboundMessageSize *prometheus.HistogramVec duplicateMessagesDropped *prometheus.CounterVec @@ -43,9 +45,10 @@ type NetworkCollector struct { dnsLookupRequestDroppedCount prometheus.Counter routingTableSize prometheus.Gauge - // authorization, rate limiting metrics + // security metrics unAuthorizedMessagesCount *prometheus.CounterVec rateLimitedUnicastMessagesCount *prometheus.CounterVec + violationReportSkippedCount prometheus.Counter prefix string } @@ -74,6 +77,8 @@ func NewNetworkCollector(logger zerolog.Logger, opts ...NetworkCollectorOpt) *Ne nc.GossipSubLocalMeshMetrics = NewGossipSubLocalMeshMetrics(nc.prefix) nc.GossipSubMetrics = NewGossipSubMetrics(nc.prefix) nc.GossipSubScoreMetrics = NewGossipSubScoreMetrics(nc.prefix) + nc.GossipSubRpcValidationInspectorMetrics = NewGossipSubRPCValidationInspectorMetrics(nc.prefix) + nc.AlspMetrics = NewAlspMetrics() nc.outboundMessageSize = promauto.NewHistogramVec( prometheus.HistogramOpts{ @@ -241,6 +246,15 @@ func NewNetworkCollector(logger zerolog.Logger, opts ...NetworkCollectorOpt) *Ne }, []string{LabelNodeRole, LabelMessage, LabelChannel, LabelRateLimitReason}, ) + nc.violationReportSkippedCount = promauto.NewCounter( + prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemSecurity, + Name: nc.prefix + "slashing_violation_reports_skipped_count", + Help: "number of slashing violations consumer violations that were not reported for misbehavior because the identity of the sender not known", + }, + ) + return nc } @@ -354,3 +368,9 @@ func (nc *NetworkCollector) OnRateLimitedPeer(peerID peer.ID, role, msgType, top Msg("unicast peer rate limited") nc.rateLimitedUnicastMessagesCount.WithLabelValues(role, msgType, topic, reason).Inc() } + +// OnViolationReportSkipped tracks the number of slashing violations consumer violations that were not +// reported for misbehavior when the identity of the sender not known. +func (nc *NetworkCollector) OnViolationReportSkipped() { + nc.violationReportSkippedCount.Inc() +} diff --git a/module/metrics/noop.go b/module/metrics/noop.go index 9999461d6da..9bf5be48f0d 100644 --- a/module/metrics/noop.go +++ b/module/metrics/noop.go @@ -4,17 +4,18 @@ import ( "context" "time" + "google.golang.org/grpc/codes" + "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" + httpmetrics "github.com/slok/go-http-metrics/metrics" "github.com/onflow/flow-go/model/chainsync" "github.com/onflow/flow-go/model/cluster" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/network/channels" - - httpmetrics "github.com/slok/go-http-metrics/metrics" ) type NoopCollector struct{} @@ -32,6 +33,9 @@ func (nc *NoopCollector) UnicastMessageSendingCompleted(topic string) func (nc *NoopCollector) BlockProposed(*flow.Block) {} func (nc *NoopCollector) BlockProposalDuration(duration time.Duration) {} +// interface check +var _ module.BackendScriptsMetrics = (*NoopCollector)(nil) +var _ module.TransactionMetrics = (*NoopCollector)(nil) var _ module.HotstuffMetrics = (*NoopCollector)(nil) var _ module.EngineMetrics = (*NoopCollector)(nil) var _ module.HeroCacheMetrics = (*NoopCollector)(nil) @@ -116,6 +120,7 @@ func (nc *NoopCollector) CommitteeProcessingDuration(duration time.Duration) func (nc *NoopCollector) SignerProcessingDuration(duration time.Duration) {} func (nc *NoopCollector) ValidatorProcessingDuration(duration time.Duration) {} func (nc *NoopCollector) PayloadProductionDuration(duration time.Duration) {} +func (nc *NoopCollector) TimeoutCollectorsRange(uint64, uint64, int) {} func (nc *NoopCollector) TransactionIngested(txID flow.Identifier) {} func (nc *NoopCollector) ClusterBlockProposed(*cluster.Block) {} func (nc *NoopCollector) ClusterBlockFinalized(*cluster.Block) {} @@ -162,7 +167,7 @@ func (nc *NoopCollector) ExecutionCollectionExecuted(_ time.Duration, _ module.E } func (nc *NoopCollector) ExecutionBlockExecutionEffortVectorComponent(_ string, _ uint) {} func (nc *NoopCollector) ExecutionBlockCachedPrograms(programs int) {} -func (nc *NoopCollector) ExecutionTransactionExecuted(_ time.Duration, _, _, _ uint64, _, _ int, _ bool) { +func (nc *NoopCollector) ExecutionTransactionExecuted(_ time.Duration, _ int, _, _ uint64, _, _ int, _ bool) { } func (nc *NoopCollector) ExecutionChunkDataPackGenerated(_, _ int) {} func (nc *NoopCollector) ExecutionScriptExecuted(dur time.Duration, compUsed, _, _ uint64) {} @@ -192,6 +197,8 @@ func (nc *NoopCollector) RuntimeSetNumberOfAccounts(count uint64) func (nc *NoopCollector) RuntimeTransactionProgramsCacheMiss() {} func (nc *NoopCollector) RuntimeTransactionProgramsCacheHit() {} func (nc *NoopCollector) ScriptExecuted(dur time.Duration, size int) {} +func (nc *NoopCollector) ScriptExecutionErrorOnArchiveNode() {} +func (nc *NoopCollector) ScriptExecutionErrorOnExecutionNode() {} func (nc *NoopCollector) TransactionResultFetched(dur time.Duration, size int) {} func (nc *NoopCollector) TransactionReceived(txID flow.Identifier, when time.Time) {} func (nc *NoopCollector) TransactionFinalized(txID flow.Identifier, when time.Time) {} @@ -199,6 +206,7 @@ func (nc *NoopCollector) TransactionExecuted(txID flow.Identifier, when time.Tim func (nc *NoopCollector) TransactionExpired(txID flow.Identifier) {} func (nc *NoopCollector) TransactionSubmissionFailed() {} func (nc *NoopCollector) UpdateExecutionReceiptMaxHeight(height uint64) {} +func (nc *NoopCollector) UpdateLastFullBlockHeight(height uint64) {} func (nc *NoopCollector) ChunkDataPackRequestProcessed() {} func (nc *NoopCollector) ExecutionSync(syncing bool) {} func (nc *NoopCollector) ExecutionBlockDataUploadStarted() {} @@ -290,3 +298,15 @@ func (nc *NoopCollector) OnBehaviourPenaltyUpdated(f float64) func (nc *NoopCollector) OnIPColocationFactorUpdated(f float64) {} func (nc *NoopCollector) OnAppSpecificScoreUpdated(f float64) {} func (nc *NoopCollector) OnOverallPeerScoreUpdated(f float64) {} + +func (nc *NoopCollector) BlockingPreProcessingStarted(string, uint) {} +func (nc *NoopCollector) BlockingPreProcessingFinished(string, uint, time.Duration) {} +func (nc *NoopCollector) AsyncProcessingStarted(string) {} +func (nc *NoopCollector) AsyncProcessingFinished(string, time.Duration) {} + +func (nc *NoopCollector) OnMisbehaviorReported(string, string) {} +func (nc *NoopCollector) OnViolationReportSkipped() {} + +var _ ObserverMetrics = (*NoopCollector)(nil) + +func (nc *NoopCollector) RecordRPC(handler, rpc string, code codes.Code) {} diff --git a/module/metrics/observer.go b/module/metrics/observer.go index 4e885c9bf4c..375aa66a2ac 100644 --- a/module/metrics/observer.go +++ b/module/metrics/observer.go @@ -6,10 +6,16 @@ import ( "google.golang.org/grpc/codes" ) +type ObserverMetrics interface { + RecordRPC(handler, rpc string, code codes.Code) +} + type ObserverCollector struct { rpcs *prometheus.CounterVec } +var _ ObserverMetrics = (*ObserverCollector)(nil) + func NewObserverCollector() *ObserverCollector { return &ObserverCollector{ rpcs: promauto.NewCounterVec(prometheus.CounterOpts{ diff --git a/module/metrics/rest_api.go b/module/metrics/rest_api.go index 36c3d1b8b1a..e9132f243c6 100644 --- a/module/metrics/rest_api.go +++ b/module/metrics/rest_api.go @@ -2,108 +2,112 @@ package metrics import ( "context" + "fmt" "time" "github.com/prometheus/client_golang/prometheus" - httpmetrics "github.com/slok/go-http-metrics/metrics" - metricsProm "github.com/slok/go-http-metrics/metrics/prometheus" -) -// Example recorder taken from: -// https://github.com/slok/go-http-metrics/blob/master/metrics/prometheus/prometheus.go -type RestCollector interface { - httpmetrics.Recorder - AddTotalRequests(ctx context.Context, service string, id string) -} + "github.com/onflow/flow-go/module" +) -type recorder struct { +type RestCollector struct { httpRequestDurHistogram *prometheus.HistogramVec httpResponseSizeHistogram *prometheus.HistogramVec httpRequestsInflight *prometheus.GaugeVec httpRequestsTotal *prometheus.GaugeVec -} - -// NewRestCollector returns a new metrics recorder that implements the recorder -// using Prometheus as the backend. -func NewRestCollector(cfg metricsProm.Config) RestCollector { - if len(cfg.DurationBuckets) == 0 { - cfg.DurationBuckets = prometheus.DefBuckets - } - - if len(cfg.SizeBuckets) == 0 { - cfg.SizeBuckets = prometheus.ExponentialBuckets(100, 10, 8) - } - - if cfg.Registry == nil { - cfg.Registry = prometheus.DefaultRegisterer - } - if cfg.HandlerIDLabel == "" { - cfg.HandlerIDLabel = "handler" - } - - if cfg.StatusCodeLabel == "" { - cfg.StatusCodeLabel = "code" - } + // urlToRouteMapper is a callback that converts a URL to a route name + urlToRouteMapper func(string) (string, error) +} - if cfg.MethodLabel == "" { - cfg.MethodLabel = "method" - } +var _ module.RestMetrics = (*RestCollector)(nil) - if cfg.ServiceLabel == "" { - cfg.ServiceLabel = "service" +// NewRestCollector returns a new metrics RestCollector that implements the RestCollector +// using Prometheus as the backend. +func NewRestCollector(urlToRouteMapper func(string) (string, error), registerer prometheus.Registerer) (*RestCollector, error) { + if urlToRouteMapper == nil { + return nil, fmt.Errorf("urlToRouteMapper cannot be nil") } - r := &recorder{ + r := &RestCollector{ + urlToRouteMapper: urlToRouteMapper, httpRequestDurHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: cfg.Prefix, - Subsystem: "http", + Namespace: namespaceRestAPI, + Subsystem: subsystemHTTP, Name: "request_duration_seconds", Help: "The latency of the HTTP requests.", - Buckets: cfg.DurationBuckets, - }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel, cfg.MethodLabel, cfg.StatusCodeLabel}), + Buckets: prometheus.DefBuckets, + }, []string{LabelService, LabelHandler, LabelMethod, LabelStatusCode}), httpResponseSizeHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: cfg.Prefix, - Subsystem: "http", + Namespace: namespaceRestAPI, + Subsystem: subsystemHTTP, Name: "response_size_bytes", Help: "The size of the HTTP responses.", - Buckets: cfg.SizeBuckets, - }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel, cfg.MethodLabel, cfg.StatusCodeLabel}), + Buckets: prometheus.ExponentialBuckets(100, 10, 8), + }, []string{LabelService, LabelHandler, LabelMethod, LabelStatusCode}), httpRequestsInflight: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: cfg.Prefix, - Subsystem: "http", + Namespace: namespaceRestAPI, + Subsystem: subsystemHTTP, Name: "requests_inflight", Help: "The number of inflight requests being handled at the same time.", - }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel}), + }, []string{LabelService, LabelHandler}), httpRequestsTotal: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: cfg.Prefix, - Subsystem: "http", + Namespace: namespaceRestAPI, + Subsystem: subsystemHTTP, Name: "requests_total", Help: "The number of requests handled over time.", - }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel}), + }, []string{LabelMethod, LabelHandler}), } - return r + registerer.MustRegister( + r.httpRequestDurHistogram, + r.httpResponseSizeHistogram, + r.httpRequestsInflight, + r.httpRequestsTotal, + ) + + return r, nil +} + +// ObserveHTTPRequestDuration records the duration of the REST request. +// This method is called automatically by go-http-metrics/middleware +func (r *RestCollector) ObserveHTTPRequestDuration(_ context.Context, p httpmetrics.HTTPReqProperties, duration time.Duration) { + handler := r.mapURLToRoute(p.ID) + r.httpRequestDurHistogram.WithLabelValues(p.Service, handler, p.Method, p.Code).Observe(duration.Seconds()) } -// These methods are called automatically by go-http-metrics/middleware -func (r recorder) ObserveHTTPRequestDuration(_ context.Context, p httpmetrics.HTTPReqProperties, duration time.Duration) { - r.httpRequestDurHistogram.WithLabelValues(p.Service, p.ID, p.Method, p.Code).Observe(duration.Seconds()) +// ObserveHTTPResponseSize records the response size of the REST request. +// This method is called automatically by go-http-metrics/middleware +func (r *RestCollector) ObserveHTTPResponseSize(_ context.Context, p httpmetrics.HTTPReqProperties, sizeBytes int64) { + handler := r.mapURLToRoute(p.ID) + r.httpResponseSizeHistogram.WithLabelValues(p.Service, handler, p.Method, p.Code).Observe(float64(sizeBytes)) } -func (r recorder) ObserveHTTPResponseSize(_ context.Context, p httpmetrics.HTTPReqProperties, sizeBytes int64) { - r.httpResponseSizeHistogram.WithLabelValues(p.Service, p.ID, p.Method, p.Code).Observe(float64(sizeBytes)) +// AddInflightRequests increments and decrements the number of inflight request being processed. +// This method is called automatically by go-http-metrics/middleware +func (r *RestCollector) AddInflightRequests(_ context.Context, p httpmetrics.HTTPProperties, quantity int) { + handler := r.mapURLToRoute(p.ID) + r.httpRequestsInflight.WithLabelValues(p.Service, handler).Add(float64(quantity)) } -func (r recorder) AddInflightRequests(_ context.Context, p httpmetrics.HTTPProperties, quantity int) { - r.httpRequestsInflight.WithLabelValues(p.Service, p.ID).Add(float64(quantity)) +// AddTotalRequests records all REST requests +// This is a custom method called by the REST handler +func (r *RestCollector) AddTotalRequests(_ context.Context, method, path string) { + handler := r.mapURLToRoute(path) + r.httpRequestsTotal.WithLabelValues(method, handler).Inc() } -// New custom method to track all requests made for every REST API request -func (r recorder) AddTotalRequests(_ context.Context, method string, id string) { - r.httpRequestsTotal.WithLabelValues(method, id).Inc() +// mapURLToRoute uses the urlToRouteMapper callback to convert a URL to a route name +// This normalizes the URL, removing dynamic information converting it to a static string +func (r *RestCollector) mapURLToRoute(url string) string { + route, err := r.urlToRouteMapper(url) + if err != nil { + return "unknown" + } + + return route } diff --git a/module/metrics/transaction.go b/module/metrics/transaction.go index 94b19e4f219..bbcb50414bd 100644 --- a/module/metrics/transaction.go +++ b/module/metrics/transaction.go @@ -7,33 +7,39 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/rs/zerolog" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/mempool" ) type TransactionCollector struct { - transactionTimings mempool.TransactionTimings - log zerolog.Logger - logTimeToFinalized bool - logTimeToExecuted bool - logTimeToFinalizedExecuted bool - timeToFinalized prometheus.Summary - timeToExecuted prometheus.Summary - timeToFinalizedExecuted prometheus.Summary - transactionSubmission *prometheus.CounterVec - scriptExecutedDuration *prometheus.HistogramVec - transactionResultDuration *prometheus.HistogramVec - scriptSize prometheus.Histogram - transactionSize prometheus.Histogram - maxReceiptHeight prometheus.Gauge - - // used to skip heights that are lower than the current max height - maxReceiptHeightValue counters.StrictMonotonousCounter + transactionTimings mempool.TransactionTimings + log zerolog.Logger + logTimeToFinalized bool + logTimeToExecuted bool + logTimeToFinalizedExecuted bool + timeToFinalized prometheus.Summary + timeToExecuted prometheus.Summary + timeToFinalizedExecuted prometheus.Summary + transactionSubmission *prometheus.CounterVec + transactionSize prometheus.Histogram + scriptExecutedDuration *prometheus.HistogramVec + scriptExecutionErrorOnExecutor *prometheus.CounterVec + scriptSize prometheus.Histogram + transactionResultDuration *prometheus.HistogramVec } -func NewTransactionCollector(transactionTimings mempool.TransactionTimings, log zerolog.Logger, - logTimeToFinalized bool, logTimeToExecuted bool, logTimeToFinalizedExecuted bool) *TransactionCollector { +// interface check +var _ module.BackendScriptsMetrics = (*TransactionCollector)(nil) +var _ module.TransactionMetrics = (*TransactionCollector)(nil) + +func NewTransactionCollector( + log zerolog.Logger, + transactionTimings mempool.TransactionTimings, + logTimeToFinalized bool, + logTimeToExecuted bool, + logTimeToFinalizedExecuted bool, +) *TransactionCollector { tc := &TransactionCollector{ transactionTimings: transactionTimings, @@ -97,6 +103,12 @@ func NewTransactionCollector(transactionTimings mempool.TransactionTimings, log Help: "histogram for the duration in ms of the round trip time for executing a script", Buckets: []float64{1, 100, 500, 1000, 2000, 5000}, }, []string{"script_size"}), + scriptExecutionErrorOnExecutor: promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "script_execution_error_archive", + Namespace: namespaceAccess, + Subsystem: subsystemTransactionSubmission, + Help: "histogram for the internal errors for executing a script for a block on the archive node", + }, []string{"source"}), transactionResultDuration: promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "transaction_result_fetched_duration", Namespace: namespaceAccess, @@ -116,18 +128,13 @@ func NewTransactionCollector(transactionTimings mempool.TransactionTimings, log Subsystem: subsystemTransactionSubmission, Help: "histogram for the transaction size in kb of scripts used in GetTransactionResult", }), - maxReceiptHeight: promauto.NewGauge(prometheus.GaugeOpts{ - Name: "max_receipt_height", - Namespace: namespaceAccess, - Subsystem: subsystemIngestion, - Help: "gauge to track the maximum block height of execution receipts received", - }), - maxReceiptHeightValue: counters.NewMonotonousCounter(0), } return tc } +// Script exec metrics + func (tc *TransactionCollector) ScriptExecuted(dur time.Duration, size int) { // record the execute script duration and script size tc.scriptSize.Observe(float64(size / 1024)) @@ -136,6 +143,18 @@ func (tc *TransactionCollector) ScriptExecuted(dur time.Duration, size int) { }).Observe(float64(dur.Milliseconds())) } +func (tc *TransactionCollector) ScriptExecutionErrorOnArchiveNode() { + // record the execution error along with blockID and scriptHash for Archive node + tc.scriptExecutionErrorOnExecutor.WithLabelValues("archive").Inc() +} + +func (tc *TransactionCollector) ScriptExecutionErrorOnExecutionNode() { + // record the execution error along with blockID and scriptHash for Execution node + tc.scriptExecutionErrorOnExecutor.WithLabelValues("execution").Inc() +} + +// TransactionResult metrics + func (tc *TransactionCollector) TransactionResultFetched(dur time.Duration, size int) { // record the transaction result duration and transaction script/payload size tc.transactionSize.Observe(float64(size / 1024)) @@ -280,9 +299,3 @@ func (tc *TransactionCollector) TransactionExpired(txID flow.Identifier) { tc.transactionSubmission.WithLabelValues("expired").Inc() tc.transactionTimings.Remove(txID) } - -func (tc *TransactionCollector) UpdateExecutionReceiptMaxHeight(height uint64) { - if tc.maxReceiptHeightValue.Set(height) { - tc.maxReceiptHeight.Set(float64(height)) - } -} diff --git a/module/mock/access_metrics.go b/module/mock/access_metrics.go index c6e25585e6a..83690a36625 100644 --- a/module/mock/access_metrics.go +++ b/module/mock/access_metrics.go @@ -2,13 +2,32 @@ package mock -import mock "github.com/stretchr/testify/mock" +import ( + context "context" + + flow "github.com/onflow/flow-go/model/flow" + metrics "github.com/slok/go-http-metrics/metrics" + + mock "github.com/stretchr/testify/mock" + + time "time" +) // AccessMetrics is an autogenerated mock type for the AccessMetrics type type AccessMetrics struct { mock.Mock } +// AddInflightRequests provides a mock function with given fields: ctx, props, quantity +func (_m *AccessMetrics) AddInflightRequests(ctx context.Context, props metrics.HTTPProperties, quantity int) { + _m.Called(ctx, props, quantity) +} + +// AddTotalRequests provides a mock function with given fields: ctx, method, routeName +func (_m *AccessMetrics) AddTotalRequests(ctx context.Context, method string, routeName string) { + _m.Called(ctx, method, routeName) +} + // ConnectionAddedToPool provides a mock function with given fields: func (_m *AccessMetrics) ConnectionAddedToPool() { _m.Called() @@ -39,11 +58,76 @@ func (_m *AccessMetrics) NewConnectionEstablished() { _m.Called() } +// ObserveHTTPRequestDuration provides a mock function with given fields: ctx, props, duration +func (_m *AccessMetrics) ObserveHTTPRequestDuration(ctx context.Context, props metrics.HTTPReqProperties, duration time.Duration) { + _m.Called(ctx, props, duration) +} + +// ObserveHTTPResponseSize provides a mock function with given fields: ctx, props, sizeBytes +func (_m *AccessMetrics) ObserveHTTPResponseSize(ctx context.Context, props metrics.HTTPReqProperties, sizeBytes int64) { + _m.Called(ctx, props, sizeBytes) +} + +// ScriptExecuted provides a mock function with given fields: dur, size +func (_m *AccessMetrics) ScriptExecuted(dur time.Duration, size int) { + _m.Called(dur, size) +} + +// ScriptExecutionErrorOnArchiveNode provides a mock function with given fields: +func (_m *AccessMetrics) ScriptExecutionErrorOnArchiveNode() { + _m.Called() +} + +// ScriptExecutionErrorOnExecutionNode provides a mock function with given fields: +func (_m *AccessMetrics) ScriptExecutionErrorOnExecutionNode() { + _m.Called() +} + // TotalConnectionsInPool provides a mock function with given fields: connectionCount, connectionPoolSize func (_m *AccessMetrics) TotalConnectionsInPool(connectionCount uint, connectionPoolSize uint) { _m.Called(connectionCount, connectionPoolSize) } +// TransactionExecuted provides a mock function with given fields: txID, when +func (_m *AccessMetrics) TransactionExecuted(txID flow.Identifier, when time.Time) { + _m.Called(txID, when) +} + +// TransactionExpired provides a mock function with given fields: txID +func (_m *AccessMetrics) TransactionExpired(txID flow.Identifier) { + _m.Called(txID) +} + +// TransactionFinalized provides a mock function with given fields: txID, when +func (_m *AccessMetrics) TransactionFinalized(txID flow.Identifier, when time.Time) { + _m.Called(txID, when) +} + +// TransactionReceived provides a mock function with given fields: txID, when +func (_m *AccessMetrics) TransactionReceived(txID flow.Identifier, when time.Time) { + _m.Called(txID, when) +} + +// TransactionResultFetched provides a mock function with given fields: dur, size +func (_m *AccessMetrics) TransactionResultFetched(dur time.Duration, size int) { + _m.Called(dur, size) +} + +// TransactionSubmissionFailed provides a mock function with given fields: +func (_m *AccessMetrics) TransactionSubmissionFailed() { + _m.Called() +} + +// UpdateExecutionReceiptMaxHeight provides a mock function with given fields: height +func (_m *AccessMetrics) UpdateExecutionReceiptMaxHeight(height uint64) { + _m.Called(height) +} + +// UpdateLastFullBlockHeight provides a mock function with given fields: height +func (_m *AccessMetrics) UpdateLastFullBlockHeight(height uint64) { + _m.Called(height) +} + type mockConstructorTestingTNewAccessMetrics interface { mock.TestingT Cleanup(func()) diff --git a/module/mock/alsp_metrics.go b/module/mock/alsp_metrics.go new file mode 100644 index 00000000000..937a210d61a --- /dev/null +++ b/module/mock/alsp_metrics.go @@ -0,0 +1,30 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import mock "github.com/stretchr/testify/mock" + +// AlspMetrics is an autogenerated mock type for the AlspMetrics type +type AlspMetrics struct { + mock.Mock +} + +// OnMisbehaviorReported provides a mock function with given fields: channel, misbehaviorType +func (_m *AlspMetrics) OnMisbehaviorReported(channel string, misbehaviorType string) { + _m.Called(channel, misbehaviorType) +} + +type mockConstructorTestingTNewAlspMetrics interface { + mock.TestingT + Cleanup(func()) +} + +// NewAlspMetrics creates a new instance of AlspMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewAlspMetrics(t mockConstructorTestingTNewAlspMetrics) *AlspMetrics { + mock := &AlspMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/module/mock/backend_scripts_metrics.go b/module/mock/backend_scripts_metrics.go index c2d30cea955..8a4eaf0ab33 100644 --- a/module/mock/backend_scripts_metrics.go +++ b/module/mock/backend_scripts_metrics.go @@ -18,6 +18,16 @@ func (_m *BackendScriptsMetrics) ScriptExecuted(dur time.Duration, size int) { _m.Called(dur, size) } +// ScriptExecutionErrorOnArchiveNode provides a mock function with given fields: +func (_m *BackendScriptsMetrics) ScriptExecutionErrorOnArchiveNode() { + _m.Called() +} + +// ScriptExecutionErrorOnExecutionNode provides a mock function with given fields: +func (_m *BackendScriptsMetrics) ScriptExecutionErrorOnExecutionNode() { + _m.Called() +} + type mockConstructorTestingTNewBackendScriptsMetrics interface { mock.TestingT Cleanup(func()) diff --git a/module/mock/cruise_ctl_metrics.go b/module/mock/cruise_ctl_metrics.go new file mode 100644 index 00000000000..137c6e1d78c --- /dev/null +++ b/module/mock/cruise_ctl_metrics.go @@ -0,0 +1,44 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + mock "github.com/stretchr/testify/mock" + + time "time" +) + +// CruiseCtlMetrics is an autogenerated mock type for the CruiseCtlMetrics type +type CruiseCtlMetrics struct { + mock.Mock +} + +// ControllerOutput provides a mock function with given fields: duration +func (_m *CruiseCtlMetrics) ControllerOutput(duration time.Duration) { + _m.Called(duration) +} + +// PIDError provides a mock function with given fields: p, i, d +func (_m *CruiseCtlMetrics) PIDError(p float64, i float64, d float64) { + _m.Called(p, i, d) +} + +// TargetProposalDuration provides a mock function with given fields: duration +func (_m *CruiseCtlMetrics) TargetProposalDuration(duration time.Duration) { + _m.Called(duration) +} + +type mockConstructorTestingTNewCruiseCtlMetrics interface { + mock.TestingT + Cleanup(func()) +} + +// NewCruiseCtlMetrics creates a new instance of CruiseCtlMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCruiseCtlMetrics(t mockConstructorTestingTNewCruiseCtlMetrics) *CruiseCtlMetrics { + mock := &CruiseCtlMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/module/mock/execution_metrics.go b/module/mock/execution_metrics.go index 276c1dfe589..9abc39fdb3b 100644 --- a/module/mock/execution_metrics.go +++ b/module/mock/execution_metrics.go @@ -96,9 +96,9 @@ func (_m *ExecutionMetrics) ExecutionSync(syncing bool) { _m.Called(syncing) } -// ExecutionTransactionExecuted provides a mock function with given fields: dur, compUsed, memoryUsed, actualMemoryUsed, eventCounts, eventSize, failed -func (_m *ExecutionMetrics) ExecutionTransactionExecuted(dur time.Duration, compUsed uint64, memoryUsed uint64, actualMemoryUsed uint64, eventCounts int, eventSize int, failed bool) { - _m.Called(dur, compUsed, memoryUsed, actualMemoryUsed, eventCounts, eventSize, failed) +// ExecutionTransactionExecuted provides a mock function with given fields: dur, numTxnConflictRetries, compUsed, memoryUsed, eventCounts, eventSize, failed +func (_m *ExecutionMetrics) ExecutionTransactionExecuted(dur time.Duration, numTxnConflictRetries int, compUsed uint64, memoryUsed uint64, eventCounts int, eventSize int, failed bool) { + _m.Called(dur, numTxnConflictRetries, compUsed, memoryUsed, eventCounts, eventSize, failed) } // FinishBlockReceivedToExecuted provides a mock function with given fields: blockID diff --git a/module/mock/finalized_header_cache.go b/module/mock/finalized_header_cache.go new file mode 100644 index 00000000000..018981fb347 --- /dev/null +++ b/module/mock/finalized_header_cache.go @@ -0,0 +1,44 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// FinalizedHeaderCache is an autogenerated mock type for the FinalizedHeaderCache type +type FinalizedHeaderCache struct { + mock.Mock +} + +// Get provides a mock function with given fields: +func (_m *FinalizedHeaderCache) Get() *flow.Header { + ret := _m.Called() + + var r0 *flow.Header + if rf, ok := ret.Get(0).(func() *flow.Header); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Header) + } + } + + return r0 +} + +type mockConstructorTestingTNewFinalizedHeaderCache interface { + mock.TestingT + Cleanup(func()) +} + +// NewFinalizedHeaderCache creates a new instance of FinalizedHeaderCache. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewFinalizedHeaderCache(t mockConstructorTestingTNewFinalizedHeaderCache) *FinalizedHeaderCache { + mock := &FinalizedHeaderCache{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/module/mock/gossip_sub_metrics.go b/module/mock/gossip_sub_metrics.go index da87176c43b..3d8df1a65c5 100644 --- a/module/mock/gossip_sub_metrics.go +++ b/module/mock/gossip_sub_metrics.go @@ -14,6 +14,26 @@ type GossipSubMetrics struct { mock.Mock } +// AsyncProcessingFinished provides a mock function with given fields: msgType, duration +func (_m *GossipSubMetrics) AsyncProcessingFinished(msgType string, duration time.Duration) { + _m.Called(msgType, duration) +} + +// AsyncProcessingStarted provides a mock function with given fields: msgType +func (_m *GossipSubMetrics) AsyncProcessingStarted(msgType string) { + _m.Called(msgType) +} + +// BlockingPreProcessingFinished provides a mock function with given fields: msgType, sampleSize, duration +func (_m *GossipSubMetrics) BlockingPreProcessingFinished(msgType string, sampleSize uint, duration time.Duration) { + _m.Called(msgType, sampleSize, duration) +} + +// BlockingPreProcessingStarted provides a mock function with given fields: msgType, sampleSize +func (_m *GossipSubMetrics) BlockingPreProcessingStarted(msgType string, sampleSize uint) { + _m.Called(msgType, sampleSize) +} + // OnAppSpecificScoreUpdated provides a mock function with given fields: _a0 func (_m *GossipSubMetrics) OnAppSpecificScoreUpdated(_a0 float64) { _m.Called(_a0) diff --git a/module/mock/gossip_sub_rpc_validation_inspector_metrics.go b/module/mock/gossip_sub_rpc_validation_inspector_metrics.go new file mode 100644 index 00000000000..ff89fca4d9f --- /dev/null +++ b/module/mock/gossip_sub_rpc_validation_inspector_metrics.go @@ -0,0 +1,49 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + mock "github.com/stretchr/testify/mock" + + time "time" +) + +// GossipSubRpcValidationInspectorMetrics is an autogenerated mock type for the GossipSubRpcValidationInspectorMetrics type +type GossipSubRpcValidationInspectorMetrics struct { + mock.Mock +} + +// AsyncProcessingFinished provides a mock function with given fields: msgType, duration +func (_m *GossipSubRpcValidationInspectorMetrics) AsyncProcessingFinished(msgType string, duration time.Duration) { + _m.Called(msgType, duration) +} + +// AsyncProcessingStarted provides a mock function with given fields: msgType +func (_m *GossipSubRpcValidationInspectorMetrics) AsyncProcessingStarted(msgType string) { + _m.Called(msgType) +} + +// BlockingPreProcessingFinished provides a mock function with given fields: msgType, sampleSize, duration +func (_m *GossipSubRpcValidationInspectorMetrics) BlockingPreProcessingFinished(msgType string, sampleSize uint, duration time.Duration) { + _m.Called(msgType, sampleSize, duration) +} + +// BlockingPreProcessingStarted provides a mock function with given fields: msgType, sampleSize +func (_m *GossipSubRpcValidationInspectorMetrics) BlockingPreProcessingStarted(msgType string, sampleSize uint) { + _m.Called(msgType, sampleSize) +} + +type mockConstructorTestingTNewGossipSubRpcValidationInspectorMetrics interface { + mock.TestingT + Cleanup(func()) +} + +// NewGossipSubRpcValidationInspectorMetrics creates a new instance of GossipSubRpcValidationInspectorMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGossipSubRpcValidationInspectorMetrics(t mockConstructorTestingTNewGossipSubRpcValidationInspectorMetrics) *GossipSubRpcValidationInspectorMetrics { + mock := &GossipSubRpcValidationInspectorMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/module/mock/grpc_connection_pool_metrics.go b/module/mock/grpc_connection_pool_metrics.go new file mode 100644 index 00000000000..2eddb3cf002 --- /dev/null +++ b/module/mock/grpc_connection_pool_metrics.go @@ -0,0 +1,60 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import mock "github.com/stretchr/testify/mock" + +// GRPCConnectionPoolMetrics is an autogenerated mock type for the GRPCConnectionPoolMetrics type +type GRPCConnectionPoolMetrics struct { + mock.Mock +} + +// ConnectionAddedToPool provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) ConnectionAddedToPool() { + _m.Called() +} + +// ConnectionFromPoolEvicted provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) ConnectionFromPoolEvicted() { + _m.Called() +} + +// ConnectionFromPoolInvalidated provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) ConnectionFromPoolInvalidated() { + _m.Called() +} + +// ConnectionFromPoolReused provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) ConnectionFromPoolReused() { + _m.Called() +} + +// ConnectionFromPoolUpdated provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) ConnectionFromPoolUpdated() { + _m.Called() +} + +// NewConnectionEstablished provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) NewConnectionEstablished() { + _m.Called() +} + +// TotalConnectionsInPool provides a mock function with given fields: connectionCount, connectionPoolSize +func (_m *GRPCConnectionPoolMetrics) TotalConnectionsInPool(connectionCount uint, connectionPoolSize uint) { + _m.Called(connectionCount, connectionPoolSize) +} + +type mockConstructorTestingTNewGRPCConnectionPoolMetrics interface { + mock.TestingT + Cleanup(func()) +} + +// NewGRPCConnectionPoolMetrics creates a new instance of GRPCConnectionPoolMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGRPCConnectionPoolMetrics(t mockConstructorTestingTNewGRPCConnectionPoolMetrics) *GRPCConnectionPoolMetrics { + mock := &GRPCConnectionPoolMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/module/mock/hot_stuff_follower.go b/module/mock/hot_stuff_follower.go index 7443aabb766..23c43d387cd 100644 --- a/module/mock/hot_stuff_follower.go +++ b/module/mock/hot_stuff_follower.go @@ -14,6 +14,11 @@ type HotStuffFollower struct { mock.Mock } +// AddCertifiedBlock provides a mock function with given fields: certifiedBlock +func (_m *HotStuffFollower) AddCertifiedBlock(certifiedBlock *model.CertifiedBlock) { + _m.Called(certifiedBlock) +} + // Done provides a mock function with given fields: func (_m *HotStuffFollower) Done() <-chan struct{} { ret := _m.Called() @@ -51,11 +56,6 @@ func (_m *HotStuffFollower) Start(_a0 irrecoverable.SignalerContext) { _m.Called(_a0) } -// SubmitProposal provides a mock function with given fields: proposal -func (_m *HotStuffFollower) SubmitProposal(proposal *model.Proposal) { - _m.Called(proposal) -} - type mockConstructorTestingTNewHotStuffFollower interface { mock.TestingT Cleanup(func()) diff --git a/module/mock/hotstuff_metrics.go b/module/mock/hotstuff_metrics.go index 79760994bad..e3a82ca4040 100644 --- a/module/mock/hotstuff_metrics.go +++ b/module/mock/hotstuff_metrics.go @@ -78,6 +78,11 @@ func (_m *HotstuffMetrics) SignerProcessingDuration(duration time.Duration) { _m.Called(duration) } +// TimeoutCollectorsRange provides a mock function with given fields: lowestRetainedView, newestViewCreatedCollector, activeCollectors +func (_m *HotstuffMetrics) TimeoutCollectorsRange(lowestRetainedView uint64, newestViewCreatedCollector uint64, activeCollectors int) { + _m.Called(lowestRetainedView, newestViewCreatedCollector, activeCollectors) +} + // TimeoutObjectProcessingDuration provides a mock function with given fields: duration func (_m *HotstuffMetrics) TimeoutObjectProcessingDuration(duration time.Duration) { _m.Called(duration) diff --git a/module/mock/lib_p2_p_metrics.go b/module/mock/lib_p2_p_metrics.go index 78b39fdae55..97b79643972 100644 --- a/module/mock/lib_p2_p_metrics.go +++ b/module/mock/lib_p2_p_metrics.go @@ -50,6 +50,16 @@ func (_m *LibP2PMetrics) AllowStream(p peer.ID, dir network.Direction) { _m.Called(p, dir) } +// AsyncProcessingFinished provides a mock function with given fields: msgType, duration +func (_m *LibP2PMetrics) AsyncProcessingFinished(msgType string, duration time.Duration) { + _m.Called(msgType, duration) +} + +// AsyncProcessingStarted provides a mock function with given fields: msgType +func (_m *LibP2PMetrics) AsyncProcessingStarted(msgType string) { + _m.Called(msgType) +} + // BlockConn provides a mock function with given fields: dir, usefd func (_m *LibP2PMetrics) BlockConn(dir network.Direction, usefd bool) { _m.Called(dir, usefd) @@ -90,6 +100,16 @@ func (_m *LibP2PMetrics) BlockStream(p peer.ID, dir network.Direction) { _m.Called(p, dir) } +// BlockingPreProcessingFinished provides a mock function with given fields: msgType, sampleSize, duration +func (_m *LibP2PMetrics) BlockingPreProcessingFinished(msgType string, sampleSize uint, duration time.Duration) { + _m.Called(msgType, sampleSize, duration) +} + +// BlockingPreProcessingStarted provides a mock function with given fields: msgType, sampleSize +func (_m *LibP2PMetrics) BlockingPreProcessingStarted(msgType string, sampleSize uint) { + _m.Called(msgType, sampleSize) +} + // DNSLookupDuration provides a mock function with given fields: duration func (_m *LibP2PMetrics) DNSLookupDuration(duration time.Duration) { _m.Called(duration) diff --git a/module/mock/network_core_metrics.go b/module/mock/network_core_metrics.go index ac7d4bab7c9..d78c3355449 100644 --- a/module/mock/network_core_metrics.go +++ b/module/mock/network_core_metrics.go @@ -5,6 +5,8 @@ package mock import ( mock "github.com/stretchr/testify/mock" + peer "github.com/libp2p/go-libp2p/core/peer" + time "time" ) @@ -43,6 +45,26 @@ func (_m *NetworkCoreMetrics) MessageRemoved(priority int) { _m.Called(priority) } +// OnMisbehaviorReported provides a mock function with given fields: channel, misbehaviorType +func (_m *NetworkCoreMetrics) OnMisbehaviorReported(channel string, misbehaviorType string) { + _m.Called(channel, misbehaviorType) +} + +// OnRateLimitedPeer provides a mock function with given fields: pid, role, msgType, topic, reason +func (_m *NetworkCoreMetrics) OnRateLimitedPeer(pid peer.ID, role string, msgType string, topic string, reason string) { + _m.Called(pid, role, msgType, topic, reason) +} + +// OnUnauthorizedMessage provides a mock function with given fields: role, msgType, topic, offense +func (_m *NetworkCoreMetrics) OnUnauthorizedMessage(role string, msgType string, topic string, offense string) { + _m.Called(role, msgType, topic, offense) +} + +// OnViolationReportSkipped provides a mock function with given fields: +func (_m *NetworkCoreMetrics) OnViolationReportSkipped() { + _m.Called() +} + // OutboundMessageSent provides a mock function with given fields: sizeBytes, topic, protocol, messageType func (_m *NetworkCoreMetrics) OutboundMessageSent(sizeBytes int, topic string, protocol string, messageType string) { _m.Called(sizeBytes, topic, protocol, messageType) diff --git a/module/mock/network_metrics.go b/module/mock/network_metrics.go index 17e7db0409a..2909f7d677f 100644 --- a/module/mock/network_metrics.go +++ b/module/mock/network_metrics.go @@ -50,6 +50,16 @@ func (_m *NetworkMetrics) AllowStream(p peer.ID, dir network.Direction) { _m.Called(p, dir) } +// AsyncProcessingFinished provides a mock function with given fields: msgType, duration +func (_m *NetworkMetrics) AsyncProcessingFinished(msgType string, duration time.Duration) { + _m.Called(msgType, duration) +} + +// AsyncProcessingStarted provides a mock function with given fields: msgType +func (_m *NetworkMetrics) AsyncProcessingStarted(msgType string) { + _m.Called(msgType) +} + // BlockConn provides a mock function with given fields: dir, usefd func (_m *NetworkMetrics) BlockConn(dir network.Direction, usefd bool) { _m.Called(dir, usefd) @@ -90,6 +100,16 @@ func (_m *NetworkMetrics) BlockStream(p peer.ID, dir network.Direction) { _m.Called(p, dir) } +// BlockingPreProcessingFinished provides a mock function with given fields: msgType, sampleSize, duration +func (_m *NetworkMetrics) BlockingPreProcessingFinished(msgType string, sampleSize uint, duration time.Duration) { + _m.Called(msgType, sampleSize, duration) +} + +// BlockingPreProcessingStarted provides a mock function with given fields: msgType, sampleSize +func (_m *NetworkMetrics) BlockingPreProcessingStarted(msgType string, sampleSize uint) { + _m.Called(msgType, sampleSize) +} + // DNSLookupDuration provides a mock function with given fields: duration func (_m *NetworkMetrics) DNSLookupDuration(duration time.Duration) { _m.Called(duration) @@ -220,6 +240,11 @@ func (_m *NetworkMetrics) OnMeshMessageDeliveredUpdated(_a0 channels.Topic, _a1 _m.Called(_a0, _a1) } +// OnMisbehaviorReported provides a mock function with given fields: channel, misbehaviorType +func (_m *NetworkMetrics) OnMisbehaviorReported(channel string, misbehaviorType string) { + _m.Called(channel, misbehaviorType) +} + // OnOverallPeerScoreUpdated provides a mock function with given fields: _a0 func (_m *NetworkMetrics) OnOverallPeerScoreUpdated(_a0 float64) { _m.Called(_a0) @@ -275,6 +300,11 @@ func (_m *NetworkMetrics) OnUnauthorizedMessage(role string, msgType string, top _m.Called(role, msgType, topic, offense) } +// OnViolationReportSkipped provides a mock function with given fields: +func (_m *NetworkMetrics) OnViolationReportSkipped() { + _m.Called() +} + // OutboundConnections provides a mock function with given fields: connectionCount func (_m *NetworkMetrics) OutboundConnections(connectionCount uint) { _m.Called(connectionCount) diff --git a/module/mock/network_security_metrics.go b/module/mock/network_security_metrics.go index 51d045c2a12..a48a693c0ab 100644 --- a/module/mock/network_security_metrics.go +++ b/module/mock/network_security_metrics.go @@ -23,6 +23,11 @@ func (_m *NetworkSecurityMetrics) OnUnauthorizedMessage(role string, msgType str _m.Called(role, msgType, topic, offense) } +// OnViolationReportSkipped provides a mock function with given fields: +func (_m *NetworkSecurityMetrics) OnViolationReportSkipped() { + _m.Called() +} + type mockConstructorTestingTNewNetworkSecurityMetrics interface { mock.TestingT Cleanup(func()) diff --git a/module/mock/pending_block_buffer.go b/module/mock/pending_block_buffer.go index b94869f7a04..a49acafce8d 100644 --- a/module/mock/pending_block_buffer.go +++ b/module/mock/pending_block_buffer.go @@ -12,13 +12,13 @@ type PendingBlockBuffer struct { mock.Mock } -// Add provides a mock function with given fields: originID, block -func (_m *PendingBlockBuffer) Add(originID flow.Identifier, block *flow.Block) bool { - ret := _m.Called(originID, block) +// Add provides a mock function with given fields: block +func (_m *PendingBlockBuffer) Add(block flow.Slashable[*flow.Block]) bool { + ret := _m.Called(block) var r0 bool - if rf, ok := ret.Get(0).(func(flow.Identifier, *flow.Block) bool); ok { - r0 = rf(originID, block) + if rf, ok := ret.Get(0).(func(flow.Slashable[*flow.Block]) bool); ok { + r0 = rf(block) } else { r0 = ret.Get(0).(bool) } diff --git a/module/mock/pending_cluster_block_buffer.go b/module/mock/pending_cluster_block_buffer.go index a1f30da90c0..686c0d9cbf0 100644 --- a/module/mock/pending_cluster_block_buffer.go +++ b/module/mock/pending_cluster_block_buffer.go @@ -14,13 +14,13 @@ type PendingClusterBlockBuffer struct { mock.Mock } -// Add provides a mock function with given fields: originID, block -func (_m *PendingClusterBlockBuffer) Add(originID flow.Identifier, block *cluster.Block) bool { - ret := _m.Called(originID, block) +// Add provides a mock function with given fields: block +func (_m *PendingClusterBlockBuffer) Add(block flow.Slashable[*cluster.Block]) bool { + ret := _m.Called(block) var r0 bool - if rf, ok := ret.Get(0).(func(flow.Identifier, *cluster.Block) bool); ok { - r0 = rf(originID, block) + if rf, ok := ret.Get(0).(func(flow.Slashable[*cluster.Block]) bool); ok { + r0 = rf(block) } else { r0 = ret.Get(0).(bool) } diff --git a/module/mock/rest_metrics.go b/module/mock/rest_metrics.go new file mode 100644 index 00000000000..b5fbd8bc50a --- /dev/null +++ b/module/mock/rest_metrics.go @@ -0,0 +1,52 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + context "context" + + metrics "github.com/slok/go-http-metrics/metrics" + mock "github.com/stretchr/testify/mock" + + time "time" +) + +// RestMetrics is an autogenerated mock type for the RestMetrics type +type RestMetrics struct { + mock.Mock +} + +// AddInflightRequests provides a mock function with given fields: ctx, props, quantity +func (_m *RestMetrics) AddInflightRequests(ctx context.Context, props metrics.HTTPProperties, quantity int) { + _m.Called(ctx, props, quantity) +} + +// AddTotalRequests provides a mock function with given fields: ctx, method, routeName +func (_m *RestMetrics) AddTotalRequests(ctx context.Context, method string, routeName string) { + _m.Called(ctx, method, routeName) +} + +// ObserveHTTPRequestDuration provides a mock function with given fields: ctx, props, duration +func (_m *RestMetrics) ObserveHTTPRequestDuration(ctx context.Context, props metrics.HTTPReqProperties, duration time.Duration) { + _m.Called(ctx, props, duration) +} + +// ObserveHTTPResponseSize provides a mock function with given fields: ctx, props, sizeBytes +func (_m *RestMetrics) ObserveHTTPResponseSize(ctx context.Context, props metrics.HTTPReqProperties, sizeBytes int64) { + _m.Called(ctx, props, sizeBytes) +} + +type mockConstructorTestingTNewRestMetrics interface { + mock.TestingT + Cleanup(func()) +} + +// NewRestMetrics creates a new instance of RestMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewRestMetrics(t mockConstructorTestingTNewRestMetrics) *RestMetrics { + mock := &RestMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/module/mock/transaction_metrics.go b/module/mock/transaction_metrics.go index 49f5f0c3958..06cfb5d8f9a 100644 --- a/module/mock/transaction_metrics.go +++ b/module/mock/transaction_metrics.go @@ -14,11 +14,6 @@ type TransactionMetrics struct { mock.Mock } -// ScriptExecuted provides a mock function with given fields: dur, size -func (_m *TransactionMetrics) ScriptExecuted(dur time.Duration, size int) { - _m.Called(dur, size) -} - // TransactionExecuted provides a mock function with given fields: txID, when func (_m *TransactionMetrics) TransactionExecuted(txID flow.Identifier, when time.Time) { _m.Called(txID, when) @@ -49,11 +44,6 @@ func (_m *TransactionMetrics) TransactionSubmissionFailed() { _m.Called() } -// UpdateExecutionReceiptMaxHeight provides a mock function with given fields: height -func (_m *TransactionMetrics) UpdateExecutionReceiptMaxHeight(height uint64) { - _m.Called(height) -} - type mockConstructorTestingTNewTransactionMetrics interface { mock.TestingT Cleanup(func()) diff --git a/module/signature/aggregation_test.go b/module/signature/aggregation_test.go index aacd0a89f06..aebc696b091 100644 --- a/module/signature/aggregation_test.go +++ b/module/signature/aggregation_test.go @@ -4,7 +4,6 @@ package signature import ( - "crypto/rand" "errors" mrand "math/rand" "sort" @@ -17,6 +16,13 @@ import ( "github.com/onflow/flow-go/crypto" ) +func getPRG(t *testing.T) *mrand.Rand { + random := time.Now().UnixNano() + t.Logf("rng seed is %d", random) + rng := mrand.New(mrand.NewSource(random)) + return rng +} + // Utility function that flips a point sign bit to negate the point // this is shortcut which works only for zcash BLS12-381 compressed serialization // that is currently supported by the flow crypto module @@ -25,7 +31,7 @@ func negatePoint(pointbytes []byte) { pointbytes[0] ^= 0x20 } -func createAggregationData(t *testing.T, signersNumber int) ( +func createAggregationData(t *testing.T, rand *mrand.Rand, signersNumber int) ( []byte, string, []crypto.Signature, []crypto.PublicKey, ) { // create message and tag @@ -54,7 +60,7 @@ func createAggregationData(t *testing.T, signersNumber int) ( } func TestAggregatorSameMessage(t *testing.T) { - + rand := getPRG(t) signersNum := 20 // constructor edge cases @@ -79,7 +85,7 @@ func TestAggregatorSameMessage(t *testing.T) { // Happy paths // all signatures are valid t.Run("happy path", func(t *testing.T) { - msg, tag, sigs, pks := createAggregationData(t, signersNum) + msg, tag, sigs, pks := createAggregationData(t, rand, signersNum) aggregator, err := NewSignatureAggregatorSameMessage(msg, tag, pks) require.NoError(t, err) @@ -150,7 +156,7 @@ func TestAggregatorSameMessage(t *testing.T) { // Unhappy paths t.Run("invalid inputs", func(t *testing.T) { - msg, tag, sigs, pks := createAggregationData(t, signersNum) + msg, tag, sigs, pks := createAggregationData(t, rand, signersNum) aggregator, err := NewSignatureAggregatorSameMessage(msg, tag, pks) require.NoError(t, err) // invalid indices for different methods @@ -184,7 +190,7 @@ func TestAggregatorSameMessage(t *testing.T) { }) t.Run("duplicate signers", func(t *testing.T) { - msg, tag, sigs, pks := createAggregationData(t, signersNum) + msg, tag, sigs, pks := createAggregationData(t, rand, signersNum) aggregator, err := NewSignatureAggregatorSameMessage(msg, tag, pks) require.NoError(t, err) @@ -223,7 +229,7 @@ func TestAggregatorSameMessage(t *testing.T) { // 1: No signature has been added. t.Run("aggregate with no signatures", func(t *testing.T) { - msg, tag, _, pks := createAggregationData(t, 1) + msg, tag, _, pks := createAggregationData(t, rand, 1) aggregator, err := NewSignatureAggregatorSameMessage(msg, tag, pks) require.NoError(t, err) // Aggregation should error with sentinel InsufficientSignaturesError @@ -239,7 +245,7 @@ func TestAggregatorSameMessage(t *testing.T) { // 2.a. aggregated public key is not identity // 2.b. aggregated public key is identity t.Run("invalid signature serialization", func(t *testing.T) { - msg, tag, sigs, pks := createAggregationData(t, 2) + msg, tag, sigs, pks := createAggregationData(t, rand, 2) invalidStructureSig := (crypto.Signature)([]byte{0, 0}) t.Run("with non-identity aggregated public key", func(t *testing.T) { @@ -305,7 +311,7 @@ func TestAggregatorSameMessage(t *testing.T) { // 3.a. aggregated public key is not identity // 3.b. aggregated public key is identity t.Run("correct serialization and invalid signature", func(t *testing.T) { - msg, tag, sigs, pks := createAggregationData(t, 2) + msg, tag, sigs, pks := createAggregationData(t, rand, 2) t.Run("with non-identity aggregated public key", func(t *testing.T) { aggregator, err := NewSignatureAggregatorSameMessage(msg, tag, pks) @@ -374,7 +380,7 @@ func TestAggregatorSameMessage(t *testing.T) { // 4. All signatures are valid but aggregated key is identity t.Run("all valid signatures and identity aggregated key", func(t *testing.T) { - msg, tag, sigs, pks := createAggregationData(t, 2) + msg, tag, sigs, pks := createAggregationData(t, rand, 2) // public key at index 1 is opposite of public key at index 0 (pks[1] = -pks[0]) // so that aggregation of pks[0] and pks[1] is identity @@ -413,9 +419,7 @@ func TestAggregatorSameMessage(t *testing.T) { } func TestKeyAggregator(t *testing.T) { - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) signersNum := 20 // create keys @@ -497,8 +501,8 @@ func TestKeyAggregator(t *testing.T) { rounds := 30 for i := 0; i < rounds; i++ { go func() { // test module concurrency - low := mrand.Intn(signersNum - 1) - high := low + 1 + mrand.Intn(signersNum-1-low) + low := rand.Intn(signersNum - 1) + high := low + 1 + rand.Intn(signersNum-1-low) var key, expectedKey crypto.PublicKey var err error key, err = aggregator.KeyAggregate(indices[low:high]) diff --git a/module/signature/signer_indices_test.go b/module/signature/signer_indices_test.go index c34daea4f37..0bd7aaee34e 100644 --- a/module/signature/signer_indices_test.go +++ b/module/signature/signer_indices_test.go @@ -112,7 +112,7 @@ func Test_EncodeSignerToIndicesAndSigType(t *testing.T) { // create committee committeeIdentities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) committee := committeeIdentities.NodeIDs() - stakingSigners, beaconSigners := sampleSigners(committee, numStakingSigners, numRandomBeaconSigners) + stakingSigners, beaconSigners := sampleSigners(t, committee, numStakingSigners, numRandomBeaconSigners) // encode prefixed, sigTypes, err := signature.EncodeSignerToIndicesAndSigType(committee, stakingSigners, beaconSigners) @@ -150,7 +150,7 @@ func Test_DecodeSigTypeToStakingAndBeaconSigners(t *testing.T) { // create committee committeeIdentities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) committee := committeeIdentities.NodeIDs() - stakingSigners, beaconSigners := sampleSigners(committee, numStakingSigners, numRandomBeaconSigners) + stakingSigners, beaconSigners := sampleSigners(t, committee, numStakingSigners, numRandomBeaconSigners) // encode signerIndices, sigTypes, err := signature.EncodeSignerToIndicesAndSigType(committee, stakingSigners, beaconSigners) @@ -276,7 +276,8 @@ func Test_EncodeSignersToIndices(t *testing.T) { // create committee identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) committee := identities.NodeIDs() - signers := committee.Sample(uint(numSigners)) + signers, err := committee.Sample(uint(numSigners)) + require.NoError(t, err) // encode prefixed, err := signature.EncodeSignersToIndices(committee, signers) @@ -305,7 +306,8 @@ func Test_DecodeSignerIndicesToIdentifiers(t *testing.T) { // create committee identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) committee := identities.NodeIDs() - signers := committee.Sample(uint(numSigners)) + signers, err := committee.Sample(uint(numSigners)) + require.NoError(t, err) sort.Sort(signers) // encode @@ -340,7 +342,8 @@ func Test_DecodeSignerIndicesToIdentities(t *testing.T) { // create committee identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) - signers := identities.Sample(uint(numSigners)) + signers, err := identities.Sample(uint(numSigners)) + require.NoError(t, err) // encode signerIndices, err := signature.EncodeSignersToIndices(identities.NodeIDs(), signers.NodeIDs()) @@ -356,6 +359,7 @@ func Test_DecodeSignerIndicesToIdentities(t *testing.T) { // sampleSigners takes `committee` and samples to _disjoint_ subsets // (`stakingSigners` and `randomBeaconSigners`) with the specified cardinality func sampleSigners( + t *rapid.T, committee flow.IdentifierList, numStakingSigners int, numRandomBeaconSigners int, @@ -364,9 +368,12 @@ func sampleSigners( panic(fmt.Sprintf("Cannot sample %d nodes out of a committee is size %d", numStakingSigners+numRandomBeaconSigners, len(committee))) } - stakingSigners = committee.Sample(uint(numStakingSigners)) + var err error + stakingSigners, err = committee.Sample(uint(numStakingSigners)) + require.NoError(t, err) remaining := committee.Filter(id.Not(id.In(stakingSigners...))) - randomBeaconSigners = remaining.Sample(uint(numRandomBeaconSigners)) + randomBeaconSigners, err = remaining.Sample(uint(numRandomBeaconSigners)) + require.NoError(t, err) return } diff --git a/module/state_synchronization/execution_data_requester.go b/module/state_synchronization/execution_data_requester.go index b0b65015a31..dd479455698 100644 --- a/module/state_synchronization/execution_data_requester.go +++ b/module/state_synchronization/execution_data_requester.go @@ -14,9 +14,14 @@ type OnExecutionDataReceivedConsumer func(*execution_data.BlockExecutionDataEnti type ExecutionDataRequester interface { component.Component - // OnBlockFinalized accepts block finalization notifications from the FinalizationDistributor + // OnBlockFinalized accepts block finalization notifications from the FollowerDistributor OnBlockFinalized(*model.Block) // AddOnExecutionDataReceivedConsumer adds a callback to be called when a new ExecutionData is received - AddOnExecutionDataReceivedConsumer(fn OnExecutionDataReceivedConsumer) + AddOnExecutionDataReceivedConsumer(OnExecutionDataReceivedConsumer) + + // HighestConsecutiveHeight returns the highest consecutive block height for which ExecutionData + // has been received. + // This method must only be called after the component is Ready. If it is called early, an error is returned. + HighestConsecutiveHeight() (uint64, error) } diff --git a/module/state_synchronization/mock/execution_data_requester.go b/module/state_synchronization/mock/execution_data_requester.go index 139c8102c6a..2aee152ac57 100644 --- a/module/state_synchronization/mock/execution_data_requester.go +++ b/module/state_synchronization/mock/execution_data_requester.go @@ -16,9 +16,9 @@ type ExecutionDataRequester struct { mock.Mock } -// AddOnExecutionDataReceivedConsumer provides a mock function with given fields: fn -func (_m *ExecutionDataRequester) AddOnExecutionDataReceivedConsumer(fn state_synchronization.OnExecutionDataReceivedConsumer) { - _m.Called(fn) +// AddOnExecutionDataReceivedConsumer provides a mock function with given fields: _a0 +func (_m *ExecutionDataRequester) AddOnExecutionDataReceivedConsumer(_a0 state_synchronization.OnExecutionDataReceivedConsumer) { + _m.Called(_a0) } // Done provides a mock function with given fields: @@ -37,6 +37,30 @@ func (_m *ExecutionDataRequester) Done() <-chan struct{} { return r0 } +// HighestConsecutiveHeight provides a mock function with given fields: +func (_m *ExecutionDataRequester) HighestConsecutiveHeight() (uint64, error) { + ret := _m.Called() + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func() (uint64, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // OnBlockFinalized provides a mock function with given fields: _a0 func (_m *ExecutionDataRequester) OnBlockFinalized(_a0 *model.Block) { _m.Called(_a0) diff --git a/module/state_synchronization/requester/execution_data_requester.go b/module/state_synchronization/requester/execution_data_requester.go index 394f64a2889..6cc1a828e91 100644 --- a/module/state_synchronization/requester/execution_data_requester.go +++ b/module/state_synchronization/requester/execution_data_requester.go @@ -16,6 +16,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/jobqueue" "github.com/onflow/flow-go/module/state_synchronization" @@ -23,6 +24,7 @@ import ( "github.com/onflow/flow-go/module/util" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/logging" ) // The ExecutionDataRequester downloads ExecutionData for sealed blocks from other participants in @@ -115,7 +117,6 @@ type ExecutionDataConfig struct { type executionDataRequester struct { component.Component - cm *component.ComponentManager downloader execution_data.Downloader metrics module.ExecutionDataRequesterMetrics config ExecutionDataConfig @@ -123,8 +124,6 @@ type executionDataRequester struct { // Local db objects headers storage.Headers - results storage.ExecutionResults - seals storage.Seals executionDataReader *jobs.ExecutionDataReader @@ -135,6 +134,8 @@ type executionDataRequester struct { blockConsumer *jobqueue.ComponentConsumer notificationConsumer *jobqueue.ComponentConsumer + execDataCache *cache.ExecutionDataCache + // List of callbacks to call when ExecutionData is successfully fetched for a block consumers []state_synchronization.OnExecutionDataReceivedConsumer @@ -148,21 +149,19 @@ func New( log zerolog.Logger, edrMetrics module.ExecutionDataRequesterMetrics, downloader execution_data.Downloader, + execDataCache *cache.ExecutionDataCache, processedHeight storage.ConsumerProgress, processedNotifications storage.ConsumerProgress, state protocol.State, headers storage.Headers, - results storage.ExecutionResults, - seals storage.Seals, cfg ExecutionDataConfig, ) state_synchronization.ExecutionDataRequester { e := &executionDataRequester{ log: log.With().Str("component", "execution_data_requester").Logger(), downloader: downloader, + execDataCache: execDataCache, metrics: edrMetrics, headers: headers, - results: results, - seals: seals, config: cfg, finalizationNotifier: engine.NewNotifier(), } @@ -193,20 +192,18 @@ func New( fetchWorkers, // the number of concurrent workers e.config.MaxSearchAhead, // max number of unsent notifications to allow before pausing new fetches ) + // notifies notificationConsumer when new ExecutionData blobs are available // SetPostNotifier will notify executionDataNotifier AFTER e.blockConsumer.LastProcessedIndex is updated. // Even though it doesn't guarantee to notify for every height at least once, the notificationConsumer is - // able to guarantee to process every height at least once, because the notificationConsumer finds new job - // using executionDataReader which finds new height using e.blockConsumer.LastProcessedIndex + // able to guarantee to process every height at least once, because the notificationConsumer finds new jobs + // using executionDataReader which finds new heights using e.blockConsumer.LastProcessedIndex e.blockConsumer.SetPostNotifier(func(module.JobID) { executionDataNotifier.Notify() }) // jobqueue Jobs object tracks downloaded execution data by height. This is used by the // notificationConsumer to get downloaded execution data from storage. e.executionDataReader = jobs.NewExecutionDataReader( - e.downloader, - e.headers, - e.results, - e.seals, + e.execDataCache, e.config.FetchTimeout, // method to get highest consecutive height that has downloaded execution data. it is used // here by the notification job consumer to discover new jobs. @@ -237,21 +234,33 @@ func New( 0, // search ahead limit controlled by worker count ) - builder := component.NewComponentManagerBuilder(). + e.Component = component.NewComponentManagerBuilder(). AddWorker(e.runBlockConsumer). - AddWorker(e.runNotificationConsumer) - - e.cm = builder.Build() - e.Component = e.cm + AddWorker(e.runNotificationConsumer). + Build() return e } -// OnBlockFinalized accepts block finalization notifications from the FinalizationDistributor +// OnBlockFinalized accepts block finalization notifications from the FollowerDistributor func (e *executionDataRequester) OnBlockFinalized(*model.Block) { e.finalizationNotifier.Notify() } +// HighestConsecutiveHeight returns the highest consecutive block height for which ExecutionData +// has been received. +// This method must only be called after the component is Ready. If it is called early, an error is returned. +func (e *executionDataRequester) HighestConsecutiveHeight() (uint64, error) { + select { + case <-e.blockConsumer.Ready(): + default: + // LastProcessedIndex is not meaningful until the component has completed startup + return 0, fmt.Errorf("HighestConsecutiveHeight must not be called before the component is ready") + } + + return e.blockConsumer.LastProcessedIndex(), nil +} + // AddOnExecutionDataReceivedConsumer adds a callback to be called when a new ExecutionData is received // Callback Implementations must: // - be concurrency safe @@ -361,7 +370,7 @@ func (e *executionDataRequester) processSealedHeight(ctx irrecoverable.SignalerC }) } -func (e *executionDataRequester) processFetchRequest(ctx irrecoverable.SignalerContext, blockID flow.Identifier, height uint64, fetchTimeout time.Duration) error { +func (e *executionDataRequester) processFetchRequest(parentCtx irrecoverable.SignalerContext, blockID flow.Identifier, height uint64, fetchTimeout time.Duration) error { logger := e.log.With(). Str("block_id", blockID.String()). Uint64("height", height). @@ -369,24 +378,15 @@ func (e *executionDataRequester) processFetchRequest(ctx irrecoverable.SignalerC logger.Debug().Msg("processing fetch request") - seal, err := e.seals.FinalizedSealForBlock(blockID) - if err != nil { - ctx.Throw(fmt.Errorf("failed to get seal for block %s: %w", blockID, err)) - } - - result, err := e.results.ByID(seal.ResultID) - if err != nil { - ctx.Throw(fmt.Errorf("failed to lookup execution result for block %s: %w", blockID, err)) - } - - logger = logger.With().Str("execution_data_id", result.ExecutionDataID.String()).Logger() - start := time.Now() e.metrics.ExecutionDataFetchStarted() logger.Debug().Msg("downloading execution data") - _, err = e.fetchExecutionData(ctx, result.ExecutionDataID, fetchTimeout) + ctx, cancel := context.WithTimeout(parentCtx, fetchTimeout) + defer cancel() + + execData, err := e.execDataCache.ByBlockID(ctx, blockID) e.metrics.ExecutionDataFetchFinished(time.Since(start), err == nil, height) @@ -409,31 +409,16 @@ func (e *executionDataRequester) processFetchRequest(ctx irrecoverable.SignalerC if err != nil { logger.Error().Err(err).Msg("unexpected error fetching execution data") - ctx.Throw(err) + parentCtx.Throw(err) } - logger.Info().Msg("execution data fetched") + logger.Info(). + Hex("execution_data_id", logging.ID(execData.ID())). + Msg("execution data fetched") return nil } -// fetchExecutionData fetches the ExecutionData by its ID, and times out if fetchTimeout is exceeded -func (e *executionDataRequester) fetchExecutionData(signalerCtx irrecoverable.SignalerContext, executionDataID flow.Identifier, fetchTimeout time.Duration) (*execution_data.BlockExecutionData, error) { - ctx, cancel := context.WithTimeout(signalerCtx, fetchTimeout) - defer cancel() - - // Get the data from the network - // this is a blocking call, won't be unblocked until either hitting error (including timeout) or - // the data is received - executionData, err := e.downloader.Download(ctx, executionDataID) - - if err != nil { - return nil, err - } - - return executionData, nil -} - // Notification Worker Methods func (e *executionDataRequester) processNotificationJob(ctx irrecoverable.SignalerContext, job module.Job, jobComplete func()) { @@ -443,17 +428,16 @@ func (e *executionDataRequester) processNotificationJob(ctx irrecoverable.Signal ctx.Throw(fmt.Errorf("failed to convert job to entry: %w", err)) } - e.processNotification(ctx, entry.Height, entry.ExecutionData) - jobComplete() -} - -func (e *executionDataRequester) processNotification(ctx irrecoverable.SignalerContext, height uint64, executionData *execution_data.BlockExecutionDataEntity) { - e.log.Debug().Msgf("notifying for block %d", height) + e.log.Debug(). + Hex("block_id", logging.ID(entry.BlockID)). + Uint64("height", entry.Height). + Msgf("notifying for block") // send notifications - e.notifyConsumers(executionData) + e.notifyConsumers(entry.ExecutionData) + jobComplete() - e.metrics.NotificationSent(height) + e.metrics.NotificationSent(entry.Height) } func (e *executionDataRequester) notifyConsumers(executionData *execution_data.BlockExecutionDataEntity) { diff --git a/module/state_synchronization/requester/execution_data_requester_test.go b/module/state_synchronization/requester/execution_data_requester_test.go index 295aadb4ae2..5ac29329094 100644 --- a/module/state_synchronization/requester/execution_data_requester_test.go +++ b/module/state_synchronization/requester/execution_data_requester_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/rand" - "os" "sync" "testing" "time" @@ -12,7 +11,6 @@ import ( "github.com/dgraph-io/badger/v2" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" - "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -20,12 +18,15 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" + "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/blobs" "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" exedatamock "github.com/onflow/flow-go/module/executiondatasync/execution_data/mock" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mempool/herocache" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/state_synchronization" "github.com/onflow/flow-go/module/state_synchronization/requester" @@ -51,7 +52,6 @@ type ExecutionDataRequesterSuite struct { func TestExecutionDataRequesterSuite(t *testing.T) { t.Parallel() - rand.Seed(time.Now().UnixMilli()) suite.Run(t, new(ExecutionDataRequesterSuite)) } @@ -112,7 +112,7 @@ func mockDownloader(edStore map[flow.Identifier]*testExecutionDataServiceEntry) return ed.ExecutionData, nil } - downloader.On("Download", mock.Anything, mock.AnythingOfType("flow.Identifier")). + downloader.On("Get", mock.Anything, mock.AnythingOfType("flow.Identifier")). Return( func(ctx context.Context, id flow.Identifier) *execution_data.BlockExecutionData { ed, _ := get(id) @@ -275,8 +275,10 @@ func (suite *ExecutionDataRequesterSuite) TestRequesterPausesAndResumes() { testData.maxSearchAhead = maxSearchAhead testData.waitTimeout = time.Second * 10 - // calculate the expected number of blocks that should be downloaded before resuming - expectedDownloads := maxSearchAhead + (pauseHeight-1)*2 + // calculate the expected number of blocks that should be downloaded before resuming. + // the test should download all blocks up to pauseHeight, then maxSearchAhead blocks beyond. + // the pause block itself is excluded. + expectedDownloads := pauseHeight + maxSearchAhead - 1 edr, fd := suite.prepareRequesterTest(testData) fetchedExecutionData := suite.runRequesterTestPauseResume(edr, fd, testData, int(expectedDownloads), resume) @@ -302,10 +304,10 @@ func (suite *ExecutionDataRequesterSuite) TestRequesterHalts() { testData := suite.generateTestData(suite.run.blockCount, generate(suite.run.blockCount)) // start processing with all seals available - edr, finalizationDistributor := suite.prepareRequesterTest(testData) + edr, followerDistributor := suite.prepareRequesterTest(testData) testData.resumeHeight = testData.endHeight testData.expectedIrrecoverable = expectedErr - fetchedExecutionData := suite.runRequesterTestHalts(edr, finalizationDistributor, testData) + fetchedExecutionData := suite.runRequesterTestHalts(edr, followerDistributor, testData) assert.Less(suite.T(), len(fetchedExecutionData), testData.sealedCount) suite.T().Log("Shutting down test") @@ -385,10 +387,14 @@ func generatePauseResume(pauseHeight uint64) (specialBlockGenerator, func()) { return generate, resume } -func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun) (state_synchronization.ExecutionDataRequester, *pubsub.FinalizationDistributor) { +func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun) (state_synchronization.ExecutionDataRequester, *pubsub.FollowerDistributor) { + logger := unittest.Logger() + metrics := metrics.NewNoopCollector() + headers := synctest.MockBlockHeaderStorage( synctest.WithByID(cfg.blocksByID), synctest.WithByHeight(cfg.blocksByHeight), + synctest.WithBlockIDByHeight(cfg.blocksByHeight), ) results := synctest.MockResultsStorage( synctest.WithResultByID(cfg.resultsByID), @@ -400,20 +406,22 @@ func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun suite.downloader = mockDownloader(cfg.executionDataEntries) - finalizationDistributor := pubsub.NewFinalizationDistributor() + heroCache := herocache.NewBlockExecutionData(state_stream.DefaultCacheSize, logger, metrics) + cache := cache.NewExecutionDataCache(suite.downloader, headers, seals, results, heroCache) + + followerDistributor := pubsub.NewFollowerDistributor() processedHeight := bstorage.NewConsumerProgress(suite.db, module.ConsumeProgressExecutionDataRequesterBlockHeight) processedNotification := bstorage.NewConsumerProgress(suite.db, module.ConsumeProgressExecutionDataRequesterNotification) edr := requester.New( - zerolog.New(os.Stdout).With().Timestamp().Logger(), - metrics.NewNoopCollector(), + logger, + metrics, suite.downloader, + cache, processedHeight, processedNotification, state, headers, - results, - seals, requester.ExecutionDataConfig{ InitialBlockHeight: cfg.startHeight - 1, MaxSearchAhead: cfg.maxSearchAhead, @@ -423,12 +431,12 @@ func (suite *ExecutionDataRequesterSuite) prepareRequesterTest(cfg *fetchTestRun }, ) - finalizationDistributor.AddOnBlockFinalizedConsumer(edr.OnBlockFinalized) + followerDistributor.AddOnBlockFinalizedConsumer(edr.OnBlockFinalized) - return edr, finalizationDistributor + return edr, followerDistributor } -func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FinalizationDistributor, cfg *fetchTestRun) receivedExecutionData { +func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchronization.ExecutionDataRequester, followerDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun) receivedExecutionData { // make sure test helper goroutines are cleaned up ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -444,8 +452,8 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchr edr.Start(signalerCtx) unittest.RequireCloseBefore(suite.T(), edr.Ready(), cfg.waitTimeout, "timed out waiting for requester to be ready") - // Send blocks through finalizationDistributor - suite.finalizeBlocks(cfg, finalizationDistributor) + // Send blocks through followerDistributor + suite.finalizeBlocks(cfg, followerDistributor) // testDone should never close because the requester paused unittest.RequireNeverClosedWithin(suite.T(), testDone, 100*time.Millisecond, "finished sending notifications unexpectedly") @@ -457,7 +465,7 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTestHalts(edr state_synchr return fetchedExecutionData } -func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FinalizationDistributor, cfg *fetchTestRun, expectedDownloads int, resume func()) receivedExecutionData { +func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_synchronization.ExecutionDataRequester, followerDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun, expectedDownloads int, resume func()) receivedExecutionData { // make sure test helper goroutines are cleaned up ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) @@ -471,14 +479,14 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_ edr.Start(signalerCtx) unittest.RequireCloseBefore(suite.T(), edr.Ready(), cfg.waitTimeout, "timed out waiting for requester to be ready") - // Send all blocks through finalizationDistributor - suite.finalizeBlocks(cfg, finalizationDistributor) + // Send all blocks through followerDistributor + suite.finalizeBlocks(cfg, followerDistributor) // requester should pause downloads until resume is called, so testDone should not be closed unittest.RequireNeverClosedWithin(suite.T(), testDone, 500*time.Millisecond, "finished unexpectedly") // confirm the expected number of downloads were attempted - suite.downloader.AssertNumberOfCalls(suite.T(), "Download", expectedDownloads) + suite.downloader.AssertNumberOfCalls(suite.T(), "Get", expectedDownloads) suite.T().Log("Resuming") resume() @@ -493,7 +501,7 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTestPauseResume(edr state_ return fetchedExecutionData } -func (suite *ExecutionDataRequesterSuite) runRequesterTest(edr state_synchronization.ExecutionDataRequester, finalizationDistributor *pubsub.FinalizationDistributor, cfg *fetchTestRun) receivedExecutionData { +func (suite *ExecutionDataRequesterSuite) runRequesterTest(edr state_synchronization.ExecutionDataRequester, followerDistributor *pubsub.FollowerDistributor, cfg *fetchTestRun) receivedExecutionData { // make sure test helper goroutines are cleaned up ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) @@ -509,8 +517,8 @@ func (suite *ExecutionDataRequesterSuite) runRequesterTest(edr state_synchroniza edr.Start(signalerCtx) unittest.RequireCloseBefore(suite.T(), edr.Ready(), cfg.waitTimeout, "timed out waiting for requester to be ready") - // Send blocks through finalizationDistributor - suite.finalizeBlocks(cfg, finalizationDistributor) + // Send blocks through followerDistributor + suite.finalizeBlocks(cfg, followerDistributor) // Pause until we've received all of the expected notifications unittest.RequireCloseBefore(suite.T(), testDone, cfg.waitTimeout, "timed out waiting for notifications") @@ -538,7 +546,7 @@ func (suite *ExecutionDataRequesterSuite) consumeExecutionDataNotifications(cfg } } -func (suite *ExecutionDataRequesterSuite) finalizeBlocks(cfg *fetchTestRun, finalizationDistributor *pubsub.FinalizationDistributor) { +func (suite *ExecutionDataRequesterSuite) finalizeBlocks(cfg *fetchTestRun, followerDistributor *pubsub.FollowerDistributor) { for i := cfg.StartHeight(); i <= cfg.endHeight; i++ { b := cfg.blocksByHeight[i] @@ -552,7 +560,7 @@ func (suite *ExecutionDataRequesterSuite) finalizeBlocks(cfg *fetchTestRun, fina suite.T().Log(">>>> Sealing block", sealedHeader.ID(), sealedHeader.Height) } - finalizationDistributor.OnFinalizedBlock(&model.Block{}) // actual block is unused + followerDistributor.OnFinalizedBlock(&model.Block{}) // actual block is unused if cfg.stopHeight == i { break @@ -656,9 +664,9 @@ func (suite *ExecutionDataRequesterSuite) generateTestData(blockCount int, speci height := uint64(i) block := buildBlock(height, previousBlock, seals) - ed := synctest.ExecutionDataFixture(block.ID()) + ed := unittest.BlockExecutionDataFixture(unittest.WithBlockExecutionDataBlockID(block.ID())) - cid, err := eds.AddExecutionData(context.Background(), ed) + cid, err := eds.Add(context.Background(), ed) require.NoError(suite.T(), err) result := buildResult(block, cid, previousResult) @@ -753,6 +761,8 @@ type mockSnapshot struct { mu sync.Mutex } +var _ protocol.Snapshot = &mockSnapshot{} + func (m *mockSnapshot) set(header *flow.Header, err error) { m.mu.Lock() defer m.mu.Unlock() @@ -777,10 +787,11 @@ func (m *mockSnapshot) Identity(nodeID flow.Identifier) (*flow.Identity, error) func (m *mockSnapshot) SealedResult() (*flow.ExecutionResult, *flow.Seal, error) { return nil, nil, nil } -func (m *mockSnapshot) Commit() (flow.StateCommitment, error) { return flow.DummyStateCommitment, nil } -func (m *mockSnapshot) SealingSegment() (*flow.SealingSegment, error) { return nil, nil } -func (m *mockSnapshot) Descendants() ([]flow.Identifier, error) { return nil, nil } -func (m *mockSnapshot) RandomSource() ([]byte, error) { return nil, nil } -func (m *mockSnapshot) Phase() (flow.EpochPhase, error) { return flow.EpochPhaseUndefined, nil } -func (m *mockSnapshot) Epochs() protocol.EpochQuery { return nil } -func (m *mockSnapshot) Params() protocol.GlobalParams { return nil } +func (m *mockSnapshot) Commit() (flow.StateCommitment, error) { return flow.DummyStateCommitment, nil } +func (m *mockSnapshot) SealingSegment() (*flow.SealingSegment, error) { return nil, nil } +func (m *mockSnapshot) Descendants() ([]flow.Identifier, error) { return nil, nil } +func (m *mockSnapshot) RandomSource() ([]byte, error) { return nil, nil } +func (m *mockSnapshot) Phase() (flow.EpochPhase, error) { return flow.EpochPhaseUndefined, nil } +func (m *mockSnapshot) Epochs() protocol.EpochQuery { return nil } +func (m *mockSnapshot) Params() protocol.GlobalParams { return nil } +func (m *mockSnapshot) VersionBeacon() (*flow.SealedVersionBeacon, error) { return nil, nil } diff --git a/module/state_synchronization/requester/jobs/execution_data_reader.go b/module/state_synchronization/requester/jobs/execution_data_reader.go index eabd7178b21..bd5f7adbeae 100644 --- a/module/state_synchronization/requester/jobs/execution_data_reader.go +++ b/module/state_synchronization/requester/jobs/execution_data_reader.go @@ -8,6 +8,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/storage" ) @@ -19,15 +20,14 @@ type BlockEntry struct { ExecutionData *execution_data.BlockExecutionDataEntity } +var _ module.Jobs = (*ExecutionDataReader)(nil) + // ExecutionDataReader provides an abstraction for consumers to read blocks as job. type ExecutionDataReader struct { - downloader execution_data.Downloader - headers storage.Headers - results storage.ExecutionResults - seals storage.Seals + store *cache.ExecutionDataCache - fetchTimeout time.Duration - highestAvailableHeight func() uint64 + fetchTimeout time.Duration + highestConsecutiveHeight func() uint64 // TODO: refactor this to accept a context in AtIndex instead of storing it on the struct. // This requires also refactoring jobqueue.Consumer @@ -36,20 +36,14 @@ type ExecutionDataReader struct { // NewExecutionDataReader creates and returns a ExecutionDataReader. func NewExecutionDataReader( - downloader execution_data.Downloader, - headers storage.Headers, - results storage.ExecutionResults, - seals storage.Seals, + store *cache.ExecutionDataCache, fetchTimeout time.Duration, - highestAvailableHeight func() uint64, + highestConsecutiveHeight func() uint64, ) *ExecutionDataReader { return &ExecutionDataReader{ - downloader: downloader, - headers: headers, - results: results, - seals: seals, - fetchTimeout: fetchTimeout, - highestAvailableHeight: highestAvailableHeight, + store: store, + fetchTimeout: fetchTimeout, + highestConsecutiveHeight: highestConsecutiveHeight, } } @@ -67,14 +61,17 @@ func (r *ExecutionDataReader) AtIndex(height uint64) (module.Job, error) { return nil, fmt.Errorf("execution data reader is not initialized") } - // height has not been downloaded, so height is not available yet - if height > r.highestAvailableHeight() { + // data for the requested height or a lower height, has not been downloaded yet. + if height > r.highestConsecutiveHeight() { return nil, storage.ErrNotFound } - executionData, err := r.getExecutionData(r.ctx, height) + ctx, cancel := context.WithTimeout(r.ctx, r.fetchTimeout) + defer cancel() + + executionData, err := r.store.ByHeight(ctx, height) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get execution data for height %d: %w", height, err) } return BlockEntryToJob(&BlockEntry{ @@ -86,36 +83,5 @@ func (r *ExecutionDataReader) AtIndex(height uint64) (module.Job, error) { // Head returns the highest consecutive block height with downloaded execution data func (r *ExecutionDataReader) Head() (uint64, error) { - return r.highestAvailableHeight(), nil -} - -// getExecutionData returns the ExecutionData for the given block height. -// This is used by the execution data reader to get the ExecutionData for a block. -func (r *ExecutionDataReader) getExecutionData(signalCtx irrecoverable.SignalerContext, height uint64) (*execution_data.BlockExecutionDataEntity, error) { - header, err := r.headers.ByHeight(height) - if err != nil { - return nil, fmt.Errorf("failed to lookup header for height %d: %w", height, err) - } - - // get the ExecutionResultID for the block from the block's seal - seal, err := r.seals.FinalizedSealForBlock(header.ID()) - if err != nil { - return nil, fmt.Errorf("failed to lookup seal for block %s: %w", header.ID(), err) - } - - result, err := r.results.ByID(seal.ResultID) - if err != nil { - return nil, fmt.Errorf("failed to lookup execution result for block %s: %w", header.ID(), err) - } - - ctx, cancel := context.WithTimeout(signalCtx, r.fetchTimeout) - defer cancel() - - executionData, err := r.downloader.Download(ctx, result.ExecutionDataID) - - if err != nil { - return nil, fmt.Errorf("failed to get execution data for block %s: %w", header.ID(), err) - } - - return execution_data.NewBlockExecutionDataEntity(result.ExecutionDataID, executionData), nil + return r.highestConsecutiveHeight(), nil } diff --git a/module/state_synchronization/requester/jobs/execution_data_reader_test.go b/module/state_synchronization/requester/jobs/execution_data_reader_test.go index 63c22042605..90240c83dd8 100644 --- a/module/state_synchronization/requester/jobs/execution_data_reader_test.go +++ b/module/state_synchronization/requester/jobs/execution_data_reader_test.go @@ -3,7 +3,6 @@ package jobs import ( "context" "errors" - "math/rand" "testing" "time" @@ -12,10 +11,14 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/onflow/flow-go/engine/access/state_stream" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" exedatamock "github.com/onflow/flow-go/module/executiondatasync/execution_data/mock" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mempool/herocache" + "github.com/onflow/flow-go/module/metrics" synctest "github.com/onflow/flow-go/module/state_synchronization/requester/unittest" "github.com/onflow/flow-go/storage" storagemock "github.com/onflow/flow-go/storage/mock" @@ -42,7 +45,6 @@ type ExecutionDataReaderSuite struct { func TestExecutionDataReaderSuite(t *testing.T) { t.Parallel() - rand.Seed(time.Now().UnixMilli()) suite.Run(t, new(ExecutionDataReaderSuite)) } @@ -56,7 +58,7 @@ func (suite *ExecutionDataReaderSuite) SetupTest() { suite.block.Header.Height: suite.block, } - suite.executionData = synctest.ExecutionDataFixture(suite.block.ID()) + suite.executionData = unittest.BlockExecutionDataFixture(unittest.WithBlockExecutionDataBlockID(suite.block.ID())) suite.highestAvailableHeight = func() uint64 { return suite.block.Header.Height + 1 } @@ -74,7 +76,10 @@ func (suite *ExecutionDataReaderSuite) reset() { unittest.Seal.WithResult(result), ) - suite.headers = synctest.MockBlockHeaderStorage(synctest.WithByHeight(suite.blocksByHeight)) + suite.headers = synctest.MockBlockHeaderStorage( + synctest.WithByHeight(suite.blocksByHeight), + synctest.WithBlockIDByHeight(suite.blocksByHeight), + ) suite.results = synctest.MockResultsStorage( synctest.WithResultByID(map[flow.Identifier]*flow.ExecutionResult{ result.ID(): result, @@ -87,11 +92,12 @@ func (suite *ExecutionDataReaderSuite) reset() { ) suite.downloader = new(exedatamock.Downloader) + + heroCache := herocache.NewBlockExecutionData(state_stream.DefaultCacheSize, unittest.Logger(), metrics.NewNoopCollector()) + cache := cache.NewExecutionDataCache(suite.downloader, suite.headers, suite.seals, suite.results, heroCache) + suite.reader = NewExecutionDataReader( - suite.downloader, - suite.headers, - suite.results, - suite.seals, + cache, suite.fetchTimeout, func() uint64 { return suite.highestAvailableHeight() @@ -101,7 +107,7 @@ func (suite *ExecutionDataReaderSuite) reset() { func (suite *ExecutionDataReaderSuite) TestAtIndex() { setExecutionDataGet := func(executionData *execution_data.BlockExecutionData, err error) { - suite.downloader.On("Download", mock.Anything, suite.executionDataID).Return( + suite.downloader.On("Get", mock.Anything, suite.executionDataID).Return( func(ctx context.Context, id flow.Identifier) *execution_data.BlockExecutionData { return executionData }, @@ -130,7 +136,7 @@ func (suite *ExecutionDataReaderSuite) TestAtIndex() { suite.Run("returns successfully", func() { suite.reset() suite.runTest(func() { - ed := synctest.ExecutionDataFixture(unittest.IdentifierFixture()) + ed := unittest.BlockExecutionDataFixture() setExecutionDataGet(ed, nil) edEntity := execution_data.NewBlockExecutionDataEntity(suite.executionDataID, ed) diff --git a/module/state_synchronization/requester/unittest/unittest.go b/module/state_synchronization/requester/unittest/unittest.go index bd4af6c8a7a..9da4ad91995 100644 --- a/module/state_synchronization/requester/unittest/unittest.go +++ b/module/state_synchronization/requester/unittest/unittest.go @@ -12,20 +12,12 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/blobs" - "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/network/mocknetwork" statemock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/storage" storagemock "github.com/onflow/flow-go/storage/mock" ) -func ExecutionDataFixture(blockID flow.Identifier) *execution_data.BlockExecutionData { - return &execution_data.BlockExecutionData{ - BlockID: blockID, - ChunkExecutionDatas: []*execution_data.ChunkExecutionData{}, - } -} - func MockBlobService(bs blockstore.Blockstore) *mocknetwork.BlobService { bex := new(mocknetwork.BlobService) @@ -157,6 +149,25 @@ func WithByID(blocksByID map[flow.Identifier]*flow.Block) BlockHeaderMockOptions } } +func WithBlockIDByHeight(blocksByHeight map[uint64]*flow.Block) BlockHeaderMockOptions { + return func(blocks *storagemock.Headers) { + blocks.On("BlockIDByHeight", mock.AnythingOfType("uint64")).Return( + func(height uint64) flow.Identifier { + if _, has := blocksByHeight[height]; !has { + return flow.ZeroID + } + return blocksByHeight[height].Header.ID() + }, + func(height uint64) error { + if _, has := blocksByHeight[height]; !has { + return fmt.Errorf("block %d not found: %w", height, storage.ErrNotFound) + } + return nil + }, + ) + } +} + func MockBlockHeaderStorage(opts ...BlockHeaderMockOptions) *storagemock.Headers { headers := new(storagemock.Headers) diff --git a/module/trace/constants.go b/module/trace/constants.go index 308f9173473..5dada818038 100644 --- a/module/trace/constants.go +++ b/module/trace/constants.go @@ -63,7 +63,7 @@ const ( // Builder COLBuildOn SpanName = "col.builder" - COLBuildOnSetup SpanName = "col.builder.setup" + COLBuildOnGetBuildCtx SpanName = "col.builder.getBuildCtx" COLBuildOnUnfinalizedLookup SpanName = "col.builder.unfinalizedLookup" COLBuildOnFinalizedLookup SpanName = "col.builder.finalizedLookup" COLBuildOnCreatePayload SpanName = "col.builder.createPayload" @@ -72,10 +72,11 @@ const ( // Cluster State COLClusterStateMutatorExtend SpanName = "col.state.mutator.extend" - COLClusterStateMutatorExtendSetup SpanName = "col.state.mutator.extend.setup" - COLClusterStateMutatorExtendCheckAncestry SpanName = "col.state.mutator.extend.ancestry" - COLClusterStateMutatorExtendCheckTransactionsValid SpanName = "col.state.mutator.extend.transactions.validity" - COLClusterStateMutatorExtendCheckTransactionsDupes SpanName = "col.state.mutator.extend.transactions.dupes" + COLClusterStateMutatorExtendCheckHeader SpanName = "col.state.mutator.extend.checkHeader" + COLClusterStateMutatorExtendGetExtendCtx SpanName = "col.state.mutator.extend.getExtendCtx" + COLClusterStateMutatorExtendCheckAncestry SpanName = "col.state.mutator.extend.checkAncestry" + COLClusterStateMutatorExtendCheckReferenceBlock SpanName = "col.state.mutator.extend.checkRefBlock" + COLClusterStateMutatorExtendCheckTransactionsValid SpanName = "col.state.mutator.extend.checkTransactionsValid" COLClusterStateMutatorExtendDBInsert SpanName = "col.state.mutator.extend.dbInsert" // Execution Node @@ -91,9 +92,8 @@ const ( EXEBroadcastExecutionReceipt SpanName = "exe.provider.broadcastExecutionReceipt" - EXEComputeBlock SpanName = "exe.computer.computeBlock" - EXEComputeTransaction SpanName = "exe.computer.computeTransaction" - EXEPostProcessTransaction SpanName = "exe.computer.postProcessTransaction" + EXEComputeBlock SpanName = "exe.computer.computeBlock" + EXEComputeTransaction SpanName = "exe.computer.computeTransaction" EXEStateSaveExecutionResults SpanName = "exe.state.saveExecutionResults" EXECommitDelta SpanName = "exe.state.commitDelta" @@ -168,6 +168,7 @@ const ( FVMEnvProgramLog SpanName = "fvm.env.programLog" FVMEnvEmitEvent SpanName = "fvm.env.emitEvent" FVMEnvGenerateUUID SpanName = "fvm.env.generateUUID" + FVMEnvGenerateAccountLocalID SpanName = "fvm.env.generateAccountLocalID" FVMEnvDecodeArgument SpanName = "fvm.env.decodeArgument" FVMEnvHash SpanName = "fvm.env.Hash" FVMEnvVerifySignature SpanName = "fvm.env.verifySignature" @@ -177,7 +178,7 @@ const ( FVMEnvBLSAggregatePublicKeys SpanName = "fvm.env.blsAggregatePublicKeys" FVMEnvGetCurrentBlockHeight SpanName = "fvm.env.getCurrentBlockHeight" FVMEnvGetBlockAtHeight SpanName = "fvm.env.getBlockAtHeight" - FVMEnvUnsafeRandom SpanName = "fvm.env.unsafeRandom" + FVMEnvRandom SpanName = "fvm.env.unsafeRandom" FVMEnvCreateAccount SpanName = "fvm.env.createAccount" FVMEnvAddAccountKey SpanName = "fvm.env.addAccountKey" FVMEnvAddEncodedAccountKey SpanName = "fvm.env.addEncodedAccountKey" diff --git a/module/trace/trace_test.go b/module/trace/trace_test.go index c98a632d4a9..f1011589930 100644 --- a/module/trace/trace_test.go +++ b/module/trace/trace_test.go @@ -2,7 +2,7 @@ package trace import ( "context" - "math/rand" + "crypto/rand" "testing" "github.com/rs/zerolog" diff --git a/module/util/util.go b/module/util/util.go index 1be65b3d9da..55a24fc19d1 100644 --- a/module/util/util.go +++ b/module/util/util.go @@ -2,6 +2,7 @@ package util import ( "context" + "math" "reflect" "github.com/onflow/flow-go/module" @@ -185,3 +186,25 @@ func DetypeSlice[T any](typedSlice []T) []any { } return untypedSlice } + +// SampleN computes a percentage of the given number 'n', and returns the result as an unsigned integer. +// If the calculated sample is greater than the provided 'max' value, it returns the ceil of 'max'. +// If 'n' is less than or equal to 0, it returns 0. +// +// Parameters: +// - n: The input number, used as the base to compute the percentage. +// - max: The maximum value that the computed sample should not exceed. +// - percentage: The percentage (in range 0.0 to 1.0) to be applied to 'n'. +// +// Returns: +// - The computed sample as an unsigned integer, with consideration to the given constraints. +func SampleN(n int, max, percentage float64) uint { + if n <= 0 { + return 0 + } + sample := float64(n) * percentage + if sample > max { + sample = max + } + return uint(math.Ceil(sample)) +} diff --git a/module/util/util_test.go b/module/util/util_test.go index 7d3069573e3..8d0f42ed1ed 100644 --- a/module/util/util_test.go +++ b/module/util/util_test.go @@ -303,3 +303,38 @@ func TestDetypeSlice(t *testing.T) { assert.Equal(t, slice[i], detyped[i].(int)) } } + +// TestSampleN contains a series of test cases to validate the behavior of the util.SampleN function. +// The test cases cover different scenarios: +// 1. "returns expected sample": Checks if the function returns the expected sample value when +// given a valid input. +// 2. "returns max value when sample greater than max": Verifies that the function returns the +// maximum allowed value when the calculated sample exceeds the maximum limit. +// 3. "returns 0 when n is less than or equal to 0": Asserts that the function returns 0 when +// the input 'n' is less than or equal to 0, which represents an invalid input. +func TestSampleN(t *testing.T) { + t.Run("returns expected sample", func(t *testing.T) { + n := 8 + max := 5.0 + percentage := .5 + sample := util.SampleN(n, max, percentage) + assert.Equal(t, uint(4), sample) + }) + t.Run("returns max value when sample greater than max", func(t *testing.T) { + n := 20 + max := 5.0 + percentage := .5 + sample := util.SampleN(n, max, percentage) + assert.Equal(t, uint(max), sample) + }) + t.Run("returns 0 when n is less than or equal to 0", func(t *testing.T) { + n := 0 + max := 5.0 + percentage := .5 + sample := util.SampleN(n, max, percentage) + assert.Equal(t, uint(0), sample, "sample returned should be 0 when n == 0") + n = -1 + sample = util.SampleN(n, max, percentage) + assert.Equal(t, uint(0), sample, "sample returned should be 0 when n < 0") + }) +} diff --git a/network/alsp.go b/network/alsp.go new file mode 100644 index 00000000000..2ed3fd938ca --- /dev/null +++ b/network/alsp.go @@ -0,0 +1,53 @@ +package network + +import ( + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/network/channels" +) + +// Misbehavior is the type of malicious action concerning a message dissemination that can be reported by the engines. +// The misbehavior is used to penalize the misbehaving node at the protocol level concerning the messages that the current +// node has received from the misbehaving node. +type Misbehavior string + +func (m Misbehavior) String() string { + return string(m) +} + +// MisbehaviorReporter is an interface that is used to report misbehavior of a remote node. +// The misbehavior is reported to the networking layer to penalize the misbehaving node. +type MisbehaviorReporter interface { + // ReportMisbehavior reports the misbehavior of a node on sending a message to the current node that appears valid + // based on the networking layer but is considered invalid by the current node based on the Flow protocol. + // The misbehavior is reported to the networking layer to penalize the misbehaving node. + // Implementation must be thread-safe and non-blocking. + ReportMisbehavior(MisbehaviorReport) +} + +// MisbehaviorReport abstracts the semantics of a misbehavior report. +// The misbehavior report is generated by the engine that detects a misbehavior on a delivered message to it. The +// engine crafts a misbehavior report and sends it to the networking layer to penalize the misbehaving node. +type MisbehaviorReport interface { + // OriginId returns the ID of the misbehaving node. + OriginId() flow.Identifier + + // Reason returns the reason of the misbehavior. + Reason() Misbehavior + + // Penalty returns the penalty value of the misbehavior. + Penalty() float64 +} + +// MisbehaviorReportManager abstracts the semantics of handling misbehavior reports. +// The misbehavior report manager is responsible for handling misbehavior reports that are sent by the engines. +// The misbehavior report manager is responsible for penalizing the misbehaving node and disallow-listing the node +// if the overall penalty of the misbehaving node drops below the disallow-listing threshold. +type MisbehaviorReportManager interface { + component.Component + // HandleMisbehaviorReport handles the misbehavior report that is sent by the engine. + // The implementation of this function should penalize the misbehaving node and report the node to be + // disallow-listed if the overall penalty of the misbehaving node drops below the disallow-listing threshold. + // The implementation of this function should be thread-safe and non-blocking. + HandleMisbehaviorReport(channels.Channel, MisbehaviorReport) +} diff --git a/network/alsp/cache.go b/network/alsp/cache.go new file mode 100644 index 00000000000..eeab8fee302 --- /dev/null +++ b/network/alsp/cache.go @@ -0,0 +1,35 @@ +package alsp + +import ( + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/network/alsp/model" +) + +// SpamRecordCache is a cache of spam records for the ALSP module. +// It is used to keep track of the spam records of the nodes that have been reported for spamming. +type SpamRecordCache interface { + // Adjust applies the given adjust function to the spam record of the given origin id. + // Returns the Penalty value of the record after the adjustment. + // It returns an error if the adjustFunc returns an error or if the record does not exist. + // Assuming that adjust is always called when the record exists, the error is irrecoverable and indicates a bug. + Adjust(originId flow.Identifier, adjustFunc model.RecordAdjustFunc) (float64, error) + + // Identities returns the list of identities of the nodes that have a spam record in the cache. + Identities() []flow.Identifier + + // Remove removes the spam record of the given origin id from the cache. + // Returns true if the record is removed, false otherwise (i.e., the record does not exist). + Remove(originId flow.Identifier) bool + + // Get returns the spam record of the given origin id. + // Returns the record and true if the record exists, nil and false otherwise. + // Args: + // - originId: the origin id of the spam record. + // Returns: + // - the record and true if the record exists, nil and false otherwise. + // Note that the returned record is a copy of the record in the cache (we do not want the caller to modify the record). + Get(originId flow.Identifier) (*model.ProtocolSpamRecord, bool) + + // Size returns the number of records in the cache. + Size() uint +} diff --git a/network/alsp/internal/cache.go b/network/alsp/internal/cache.go new file mode 100644 index 00000000000..c29ae4bd988 --- /dev/null +++ b/network/alsp/internal/cache.go @@ -0,0 +1,195 @@ +package internal + +import ( + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" + "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" + "github.com/onflow/flow-go/module/mempool/stdmap" + "github.com/onflow/flow-go/network/alsp" + "github.com/onflow/flow-go/network/alsp/model" +) + +var ErrSpamRecordNotFound = fmt.Errorf("spam record not found") + +// SpamRecordCache is a cache that stores spam records at the protocol layer for ALSP. +type SpamRecordCache struct { + recordFactory model.SpamRecordFactoryFunc // recordFactory is a factory function that creates a new spam record. + c *stdmap.Backend // c is the underlying cache. +} + +var _ alsp.SpamRecordCache = (*SpamRecordCache)(nil) + +// NewSpamRecordCache creates a new SpamRecordCache. +// Args: +// - sizeLimit: the maximum number of records that the cache can hold. +// - logger: the logger used by the cache. +// - collector: the metrics collector used by the cache. +// - recordFactory: a factory function that creates a new spam record. +// Returns: +// - *SpamRecordCache, the created cache. +// Note that the cache is supposed to keep the spam record for the authorized (staked) nodes. Since the number of such nodes is +// expected to be small, we do not eject any records from the cache. The cache size must be large enough to hold all +// the spam records of the authorized nodes. Also, this cache is keeping at most one record per origin id, so the +// size of the cache must be at least the number of authorized nodes. +func NewSpamRecordCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, recordFactory model.SpamRecordFactoryFunc) *SpamRecordCache { + backData := herocache.NewCache(sizeLimit, + herocache.DefaultOversizeFactor, + // this cache is supposed to keep the spam record for the authorized (staked) nodes. Since the number of such nodes is + // expected to be small, we do not eject any records from the cache. The cache size must be large enough to hold all + // the spam records of the authorized nodes. Also, this cache is keeping at most one record per origin id, so the + // size of the cache must be at least the number of authorized nodes. + heropool.NoEjection, + logger.With().Str("mempool", "aslp-spam-records").Logger(), + collector) + + return &SpamRecordCache{ + recordFactory: recordFactory, + c: stdmap.NewBackend(stdmap.WithBackData(backData)), + } +} + +// init initializes the spam record cache for the given origin id if it does not exist. +// Returns true if the record is initialized, false otherwise (i.e., the record already exists). +// Args: +// - originId: the origin id of the spam record. +// Returns: +// - true if the record is initialized, false otherwise (i.e., the record already exists). +// Note that if Init is called multiple times for the same origin id, the record is initialized only once, and the +// subsequent calls return false and do not change the record (i.e., the record is not re-initialized). +func (s *SpamRecordCache) init(originId flow.Identifier) bool { + return s.c.Add(ProtocolSpamRecordEntity{s.recordFactory(originId)}) +} + +// Adjust applies the given adjust function to the spam record of the given origin id. +// Returns the Penalty value of the record after the adjustment. +// It returns an error if the adjustFunc returns an error or if the record does not exist. +// Note that if the record the Adjust is called when the record does not exist, the record is initialized and the +// adjust function is applied to the initialized record again. In this case, the adjust function should not return an error. +// Args: +// - originId: the origin id of the spam record. +// - adjustFunc: the function that adjusts the spam record. +// Returns: +// - Penalty value of the record after the adjustment. +// - error any returned error should be considered as an irrecoverable error and indicates a bug. +func (s *SpamRecordCache) Adjust(originId flow.Identifier, adjustFunc model.RecordAdjustFunc) (float64, error) { + // first, we try to optimistically adjust the record assuming that the record already exists. + penalty, err := s.adjust(originId, adjustFunc) + + switch { + + case err == ErrSpamRecordNotFound: + // if the record does not exist, we initialize the record and try to adjust it again. + // Note: there is an edge case where the record is initialized by another goroutine between the two calls. + // In this case, the init function is invoked twice, but it is not a problem because the underlying + // cache is thread-safe. Hence, we do not need to synchronize the two calls. In such cases, one of the + // two calls returns false, and the other call returns true. We do not care which call returns false, hence, + // we ignore the return value of the init function. + _ = s.init(originId) + // as the record is initialized, the adjust function should not return an error, and any returned error + // is an irrecoverable error and indicates a bug. + return s.adjust(originId, adjustFunc) + case err != nil: + // if the adjust function returns an unexpected error on the first attempt, we return the error directly. + return 0, err + default: + // if the adjust function returns no error, we return the penalty value. + return penalty, nil + } +} + +// adjust applies the given adjust function to the spam record of the given origin id. +// Returns the Penalty value of the record after the adjustment. +// It returns an error if the adjustFunc returns an error or if the record does not exist. +// Args: +// - originId: the origin id of the spam record. +// - adjustFunc: the function that adjusts the spam record. +// Returns: +// - Penalty value of the record after the adjustment. +// - error if the adjustFunc returns an error or if the record does not exist (ErrSpamRecordNotFound). Except the ErrSpamRecordNotFound, +// any other error should be treated as an irrecoverable error and indicates a bug. +func (s *SpamRecordCache) adjust(originId flow.Identifier, adjustFunc model.RecordAdjustFunc) (float64, error) { + var rErr error + adjustedEntity, adjusted := s.c.Adjust(originId, func(entity flow.Entity) flow.Entity { + record, ok := entity.(ProtocolSpamRecordEntity) + if !ok { + // sanity check + // This should never happen, because the cache only contains ProtocolSpamRecordEntity entities. + panic(fmt.Sprintf("invalid entity type, expected ProtocolSpamRecordEntity type, got: %T", entity)) + } + + // Adjust the record. + adjustedRecord, err := adjustFunc(record.ProtocolSpamRecord) + if err != nil { + rErr = fmt.Errorf("adjust function failed: %w", err) + return entity // returns the original entity (reverse the adjustment). + } + + // Return the adjusted record. + return ProtocolSpamRecordEntity{adjustedRecord} + }) + + if rErr != nil { + return 0, fmt.Errorf("failed to adjust record: %w", rErr) + } + + if !adjusted { + return 0, ErrSpamRecordNotFound + } + + return adjustedEntity.(ProtocolSpamRecordEntity).Penalty, nil +} + +// Get returns the spam record of the given origin id. +// Returns the record and true if the record exists, nil and false otherwise. +// Args: +// - originId: the origin id of the spam record. +// Returns: +// - the record and true if the record exists, nil and false otherwise. +// Note that the returned record is a copy of the record in the cache (we do not want the caller to modify the record). +func (s *SpamRecordCache) Get(originId flow.Identifier) (*model.ProtocolSpamRecord, bool) { + entity, ok := s.c.ByID(originId) + if !ok { + return nil, false + } + + record, ok := entity.(ProtocolSpamRecordEntity) + if !ok { + // sanity check + // This should never happen, because the cache only contains ProtocolSpamRecordEntity entities. + panic(fmt.Sprintf("invalid entity type, expected ProtocolSpamRecordEntity type, got: %T", entity)) + } + + // return a copy of the record (we do not want the caller to modify the record). + return &model.ProtocolSpamRecord{ + OriginId: record.OriginId, + Decay: record.Decay, + CutoffCounter: record.CutoffCounter, + Penalty: record.Penalty, + DisallowListed: record.DisallowListed, + }, true +} + +// Identities returns the list of identities of the nodes that have a spam record in the cache. +func (s *SpamRecordCache) Identities() []flow.Identifier { + return flow.GetIDs(s.c.All()) +} + +// Remove removes the spam record of the given origin id from the cache. +// Returns true if the record is removed, false otherwise (i.e., the record does not exist). +// Args: +// - originId: the origin id of the spam record. +// Returns: +// - true if the record is removed, false otherwise (i.e., the record does not exist). +func (s *SpamRecordCache) Remove(originId flow.Identifier) bool { + return s.c.Remove(originId) +} + +// Size returns the number of spam records in the cache. +func (s *SpamRecordCache) Size() uint { + return s.c.Size() +} diff --git a/network/alsp/internal/cache_entity.go b/network/alsp/internal/cache_entity.go new file mode 100644 index 00000000000..939a1b7bf79 --- /dev/null +++ b/network/alsp/internal/cache_entity.go @@ -0,0 +1,28 @@ +package internal + +import ( + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/network/alsp/model" +) + +// ProtocolSpamRecordEntity is an entity that represents a spam record. It is internally +// used by the SpamRecordCache to store the spam records in the cache. +// The identifier of this entity is the origin id of the spam record. This entails that the spam records +// are deduplicated by origin id. +type ProtocolSpamRecordEntity struct { + model.ProtocolSpamRecord +} + +var _ flow.Entity = (*ProtocolSpamRecordEntity)(nil) + +// ID returns the origin id of the spam record, which is used as the unique identifier of the entity for maintenance and +// deduplication purposes in the cache. +func (p ProtocolSpamRecordEntity) ID() flow.Identifier { + return p.OriginId +} + +// Checksum returns the origin id of the spam record, it does not have any purpose in the cache. +// It is implemented to satisfy the flow.Entity interface. +func (p ProtocolSpamRecordEntity) Checksum() flow.Identifier { + return p.OriginId +} diff --git a/network/alsp/internal/cache_test.go b/network/alsp/internal/cache_test.go new file mode 100644 index 00000000000..22efd456a8e --- /dev/null +++ b/network/alsp/internal/cache_test.go @@ -0,0 +1,815 @@ +package internal_test + +import ( + "errors" + "sync" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network/alsp/internal" + "github.com/onflow/flow-go/network/alsp/model" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestNewSpamRecordCache tests the creation of a new SpamRecordCache. +// It ensures that the returned cache is not nil. It does not test the +// functionality of the cache. +func TestNewSpamRecordCache(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + require.Equalf(t, uint(0), cache.Size(), "cache size must be 0") +} + +// protocolSpamRecordFixture creates a new protocol spam record with the given origin id. +// Args: +// - id: the origin id of the spam record. +// Returns: +// - alsp.ProtocolSpamRecord, the created spam record. +// Note that the returned spam record is not a valid spam record. It is used only for testing. +func protocolSpamRecordFixture(id flow.Identifier) model.ProtocolSpamRecord { + return model.ProtocolSpamRecord{ + OriginId: id, + Decay: 1000, + CutoffCounter: 0, + Penalty: 0, + } +} + +// TestSpamRecordCache_Adjust_Init tests that when the Adjust function is called +// on a record that does not exist in the cache, the record is initialized and +// the adjust function is applied to the initialized record. +func TestSpamRecordCache_Adjust_Init(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + + recordFactoryCalled := 0 + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + require.Less(t, recordFactoryCalled, 2, "record factory must be called only twice") + return protocolSpamRecordFixture(id) + } + adjustFuncIncrement := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + record.Penalty += 1 + return record, nil + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + require.Zerof(t, cache.Size(), "expected cache to be empty") + + originID1 := unittest.IdentifierFixture() + originID2 := unittest.IdentifierFixture() + + // adjusting a spam record for an origin ID that does not exist in the cache should initialize the record. + initializedPenalty, err := cache.Adjust(originID1, adjustFuncIncrement) + require.NoError(t, err, "expected no error") + require.Equal(t, float64(1), initializedPenalty, "expected initialized penalty to be 1") + + record1, ok := cache.Get(originID1) + require.True(t, ok, "expected record to exist") + require.NotNil(t, record1, "expected non-nil record") + require.Equal(t, originID1, record1.OriginId, "expected record to have correct origin ID") + require.False(t, record1.DisallowListed, "expected record to not be disallow listed") + require.Equal(t, cache.Size(), uint(1), "expected cache to have one record") + + // adjusting a spam record for an origin ID that already exists in the cache should not initialize the record, + // but should apply the adjust function to the existing record. + initializedPenalty, err = cache.Adjust(originID1, adjustFuncIncrement) + require.NoError(t, err, "expected no error") + require.Equal(t, float64(2), initializedPenalty, "expected initialized penalty to be 2") + record1Again, ok := cache.Get(originID1) + require.True(t, ok, "expected record to still exist") + require.NotNil(t, record1Again, "expected non-nil record") + require.Equal(t, originID1, record1Again.OriginId, "expected record to have correct origin ID") + require.False(t, record1Again.DisallowListed, "expected record not to be disallow listed") + require.Equal(t, cache.Size(), uint(1), "expected cache to still have one record") + + // adjusting a spam record for a different origin ID should initialize the record. + // this is to ensure that the record factory is called only once. + initializedPenalty, err = cache.Adjust(originID2, adjustFuncIncrement) + require.NoError(t, err, "expected no error") + require.Equal(t, float64(1), initializedPenalty, "expected initialized penalty to be 1") + record2, ok := cache.Get(originID2) + require.True(t, ok, "expected record to exist") + require.NotNil(t, record2, "expected non-nil record") + require.Equal(t, originID2, record2.OriginId, "expected record to have correct origin ID") + require.False(t, record2.DisallowListed, "expected record not to be disallow listed") + require.Equal(t, cache.Size(), uint(2), "expected cache to have two records") +} + +// TestSpamRecordCache_Adjust tests the Adjust method of the SpamRecordCache. +// The test covers the following scenarios: +// 1. Adjusting a spam record for an existing origin ID. +// 2. Attempting to adjust a spam record with an adjustFunc that returns an error. +func TestSpamRecordCache_Adjust_Error(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originID1 := unittest.IdentifierFixture() + originID2 := unittest.IdentifierFixture() + + // initialize spam records for originID1 and originID2 + penalty, err := cache.Adjust(originID1, adjustFnNoOp) + require.NoError(t, err, "expected no error") + require.Equal(t, 0.0, penalty, "expected penalty to be 0") + penalty, err = cache.Adjust(originID2, adjustFnNoOp) + require.NoError(t, err, "expected no error") + require.Equal(t, 0.0, penalty, "expected penalty to be 0") + + // test adjusting the spam record for an existing origin ID + adjustFunc := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + record.Penalty -= 10 + return record, nil + } + penalty, err = cache.Adjust(originID1, adjustFunc) + require.NoError(t, err) + require.Equal(t, -10.0, penalty) + + record1, ok := cache.Get(originID1) + require.True(t, ok) + require.NotNil(t, record1) + require.Equal(t, -10.0, record1.Penalty) + + // test adjusting the spam record with an adjustFunc that returns an error + adjustFuncError := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, errors.New("adjustment error") + } + _, err = cache.Adjust(originID1, adjustFuncError) + require.Error(t, err) + + // even though the adjustFunc returned an error, the record should be intact. + record1, ok = cache.Get(originID1) + require.True(t, ok) + require.NotNil(t, record1) + require.Equal(t, -10.0, record1.Penalty) +} + +// TestSpamRecordCache_Identities tests the Identities method of the SpamRecordCache. +// The test covers the following scenarios: +// 1. Initializing the cache with multiple spam records. +// 2. Checking if the Identities method returns the correct set of origin IDs. +func TestSpamRecordCache_Identities(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originID1 := unittest.IdentifierFixture() + originID2 := unittest.IdentifierFixture() + originID3 := unittest.IdentifierFixture() + + // initialize spam records for a few origin IDs + _, err := cache.Adjust(originID1, adjustFnNoOp) + require.NoError(t, err) + _, err = cache.Adjust(originID2, adjustFnNoOp) + require.NoError(t, err) + _, err = cache.Adjust(originID3, adjustFnNoOp) + require.NoError(t, err) + + // check if the Identities method returns the correct set of origin IDs + identities := cache.Identities() + require.Equal(t, 3, len(identities)) + + identityMap := make(map[flow.Identifier]struct{}) + for _, id := range identities { + identityMap[id] = struct{}{} + } + + require.Contains(t, identityMap, originID1) + require.Contains(t, identityMap, originID2) + require.Contains(t, identityMap, originID3) +} + +// TestSpamRecordCache_Remove tests the Remove method of the SpamRecordCache. +// The test covers the following scenarios: +// 1. Initializing the cache with multiple spam records. +// 2. Removing a spam record and checking if it is removed correctly. +// 3. Ensuring the other spam records are still in the cache after removal. +// 4. Attempting to remove a non-existent origin ID. +func TestSpamRecordCache_Remove(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originID1 := unittest.IdentifierFixture() + originID2 := unittest.IdentifierFixture() + originID3 := unittest.IdentifierFixture() + + // initialize spam records for a few origin IDs + _, err := cache.Adjust(originID1, adjustFnNoOp) + require.NoError(t, err) + _, err = cache.Adjust(originID2, adjustFnNoOp) + require.NoError(t, err) + _, err = cache.Adjust(originID3, adjustFnNoOp) + require.NoError(t, err) + + // remove originID1 and check if the record is removed + require.True(t, cache.Remove(originID1)) + _, exists := cache.Get(originID1) + require.False(t, exists) + + // check if the other origin IDs are still in the cache + _, exists = cache.Get(originID2) + require.True(t, exists) + _, exists = cache.Get(originID3) + require.True(t, exists) + + // attempt to remove a non-existent origin ID + originID4 := unittest.IdentifierFixture() + require.False(t, cache.Remove(originID4)) +} + +// TestSpamRecordCache_EdgeCasesAndInvalidInputs tests the edge cases and invalid inputs for SpamRecordCache methods. +// The test covers the following scenarios: +// 1. Initializing a spam record multiple times. +// 2. Adjusting a non-existent spam record. +// 3. Removing a spam record multiple times. +func TestSpamRecordCache_EdgeCasesAndInvalidInputs(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + // 1. initializing a spam record multiple times + originID1 := unittest.IdentifierFixture() + + _, err := cache.Adjust(originID1, adjustFnNoOp) + require.NoError(t, err) + _, err = cache.Adjust(originID1, adjustFnNoOp) + require.NoError(t, err) + + // 2. Test adjusting a non-existent spam record + originID2 := unittest.IdentifierFixture() + initialPenalty, err := cache.Adjust(originID2, func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + record.Penalty -= 10 + return record, nil + }) + require.NoError(t, err) + require.Equal(t, float64(-10), initialPenalty) + + // 3. Test removing a spam record multiple times + originID3 := unittest.IdentifierFixture() + _, err = cache.Adjust(originID3, adjustFnNoOp) + require.NoError(t, err) + require.True(t, cache.Remove(originID3)) + require.False(t, cache.Remove(originID3)) +} + +// TestSpamRecordCache_ConcurrentInitialization tests the concurrent initialization of spam records. +// The test covers the following scenarios: +// 1. Multiple goroutines initializing spam records for different origin IDs. +// 2. Ensuring that all spam records are correctly initialized. +func TestSpamRecordCache_ConcurrentInitialization(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originIDs := unittest.IdentifierListFixture(10) + + var wg sync.WaitGroup + wg.Add(len(originIDs)) + + for _, originID := range originIDs { + go func(id flow.Identifier) { + defer wg.Done() + penalty, err := cache.Adjust(id, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + }(originID) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that all spam records are correctly initialized + for _, originID := range originIDs { + record, found := cache.Get(originID) + require.True(t, found) + require.NotNil(t, record) + require.Equal(t, originID, record.OriginId) + } +} + +// TestSpamRecordCache_ConcurrentSameRecordAdjust tests the concurrent adjust of the same spam record. +// The test covers the following scenarios: +// 1. Multiple goroutines attempting to adjust the same spam record concurrently. +// 2. Only one of the adjust operations succeeds on initializing the record. +// 3. The rest of the adjust operations only update the record (no initialization). +func TestSpamRecordCache_ConcurrentSameRecordAdjust(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFn := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + record.Penalty -= 1.0 + record.DisallowListed = true + record.Decay += 1.0 + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originID := unittest.IdentifierFixture() + const concurrentAttempts = 10 + + var wg sync.WaitGroup + wg.Add(concurrentAttempts) + + for i := 0; i < concurrentAttempts; i++ { + go func() { + defer wg.Done() + penalty, err := cache.Adjust(originID, adjustFn) + require.NoError(t, err) + require.Less(t, penalty, 0.0) // penalty should be negative + }() + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that the record is correctly initialized and adjusted in the cache + initDecay := model.SpamRecordFactory()(originID).Decay + record, found := cache.Get(originID) + require.True(t, found) + require.NotNil(t, record) + require.Equal(t, concurrentAttempts*-1.0, record.Penalty) + require.Equal(t, initDecay+concurrentAttempts*1.0, record.Decay) + require.True(t, record.DisallowListed) + require.Equal(t, originID, record.OriginId) +} + +// TestSpamRecordCache_ConcurrentRemoval tests the concurrent removal of spam records for different origin IDs. +// The test covers the following scenarios: +// 1. Multiple goroutines removing spam records for different origin IDs concurrently. +// 2. The records are correctly removed from the cache. +func TestSpamRecordCache_ConcurrentRemoval(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originIDs := unittest.IdentifierListFixture(10) + for _, originID := range originIDs { + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + } + + var wg sync.WaitGroup + wg.Add(len(originIDs)) + + for _, originID := range originIDs { + go func(id flow.Identifier) { + defer wg.Done() + removed := cache.Remove(id) + require.True(t, removed) + }(originID) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that the records are correctly removed from the cache + for _, originID := range originIDs { + _, found := cache.Get(originID) + require.False(t, found) + } + + // ensure that the cache is empty + require.Equal(t, uint(0), cache.Size()) +} + +// TestSpamRecordCache_ConcurrentUpdatesAndReads tests the concurrent adjustments and reads of spam records for different +// origin IDs. The test covers the following scenarios: +// 1. Multiple goroutines adjusting spam records for different origin IDs concurrently. +// 2. Multiple goroutines getting spam records for different origin IDs concurrently. +// 3. The adjusted records are correctly updated in the cache. +func TestSpamRecordCache_ConcurrentUpdatesAndReads(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originIDs := unittest.IdentifierListFixture(10) + for _, originID := range originIDs { + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + } + + var wg sync.WaitGroup + wg.Add(len(originIDs) * 2) + + adjustFunc := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + record.Penalty -= 1 + return record, nil + } + + for _, originID := range originIDs { + // adjust spam records concurrently + go func(id flow.Identifier) { + defer wg.Done() + _, err := cache.Adjust(id, adjustFunc) + require.NoError(t, err) + }(originID) + + // get spam records concurrently + go func(id flow.Identifier) { + defer wg.Done() + record, found := cache.Get(id) + require.True(t, found) + require.NotNil(t, record) + }(originID) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that the records are correctly updated in the cache + for _, originID := range originIDs { + record, found := cache.Get(originID) + require.True(t, found) + require.Equal(t, -1.0, record.Penalty) + } +} + +// TestSpamRecordCache_ConcurrentInitAndRemove tests the concurrent initialization and removal of spam records for different +// origin IDs. The test covers the following scenarios: +// 1. Multiple goroutines initializing spam records for different origin IDs concurrently. +// 2. Multiple goroutines removing spam records for different origin IDs concurrently. +// 3. The initialized records are correctly added to the cache. +// 4. The removed records are correctly removed from the cache. +func TestSpamRecordCache_ConcurrentInitAndRemove(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originIDs := unittest.IdentifierListFixture(20) + originIDsToAdd := originIDs[:10] + originIDsToRemove := originIDs[10:] + + for _, originID := range originIDsToRemove { + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + } + + var wg sync.WaitGroup + wg.Add(len(originIDs)) + + // initialize spam records concurrently + for _, originID := range originIDsToAdd { + originID := originID // capture range variable + go func() { + defer wg.Done() + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + }() + } + + // remove spam records concurrently + for _, originID := range originIDsToRemove { + go func(id flow.Identifier) { + defer wg.Done() + cache.Remove(id) + }(originID) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that the initialized records are correctly added to the cache + for _, originID := range originIDsToAdd { + record, found := cache.Get(originID) + require.True(t, found) + require.NotNil(t, record) + } + + // ensure that the removed records are correctly removed from the cache + for _, originID := range originIDsToRemove { + _, found := cache.Get(originID) + require.False(t, found) + } +} + +// TestSpamRecordCache_ConcurrentInitRemoveAdjust tests the concurrent initialization, removal, and adjustment of spam +// records for different origin IDs. The test covers the following scenarios: +// 1. Multiple goroutines initializing spam records for different origin IDs concurrently. +// 2. Multiple goroutines removing spam records for different origin IDs concurrently. +// 3. Multiple goroutines adjusting spam records for different origin IDs concurrently. +func TestSpamRecordCache_ConcurrentInitRemoveAdjust(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originIDs := unittest.IdentifierListFixture(30) + originIDsToAdd := originIDs[:10] + originIDsToRemove := originIDs[10:20] + originIDsToAdjust := originIDs[20:] + + for _, originID := range originIDsToRemove { + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + } + + adjustFunc := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + record.Penalty -= 1 + return record, nil + } + + var wg sync.WaitGroup + wg.Add(len(originIDs)) + + // Initialize spam records concurrently + for _, originID := range originIDsToAdd { + originID := originID // capture range variable + go func() { + defer wg.Done() + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + }() + } + + // Remove spam records concurrently + for _, originID := range originIDsToRemove { + go func(id flow.Identifier) { + defer wg.Done() + cache.Remove(id) + }(originID) + } + + // Adjust spam records concurrently + for _, originID := range originIDsToAdjust { + go func(id flow.Identifier) { + defer wg.Done() + _, _ = cache.Adjust(id, adjustFunc) + }(originID) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") +} + +// TestSpamRecordCache_ConcurrentInitRemoveAndAdjust tests the concurrent initialization, removal, and adjustment of spam +// records for different origin IDs. The test covers the following scenarios: +// 1. Multiple goroutines initializing spam records for different origin IDs concurrently. +// 2. Multiple goroutines removing spam records for different origin IDs concurrently. +// 3. Multiple goroutines adjusting spam records for different origin IDs concurrently. +// 4. The initialized records are correctly added to the cache. +// 5. The removed records are correctly removed from the cache. +// 6. The adjusted records are correctly updated in the cache. +func TestSpamRecordCache_ConcurrentInitRemoveAndAdjust(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originIDs := unittest.IdentifierListFixture(30) + originIDsToAdd := originIDs[:10] + originIDsToRemove := originIDs[10:20] + originIDsToAdjust := originIDs[20:] + + for _, originID := range originIDsToRemove { + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + } + + for _, originID := range originIDsToAdjust { + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + } + + var wg sync.WaitGroup + wg.Add(len(originIDs)) + + // initialize spam records concurrently + for _, originID := range originIDsToAdd { + originID := originID + go func() { + defer wg.Done() + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + }() + } + + // remove spam records concurrently + for _, originID := range originIDsToRemove { + originID := originID + go func() { + defer wg.Done() + cache.Remove(originID) + }() + } + + // adjust spam records concurrently + for _, originID := range originIDsToAdjust { + originID := originID + go func() { + defer wg.Done() + _, err := cache.Adjust(originID, func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + record.Penalty -= 1 + return record, nil + }) + require.NoError(t, err) + }() + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that the initialized records are correctly added to the cache + for _, originID := range originIDsToAdd { + record, found := cache.Get(originID) + require.True(t, found) + require.NotNil(t, record) + } + + // ensure that the removed records are correctly removed from the cache + for _, originID := range originIDsToRemove { + _, found := cache.Get(originID) + require.False(t, found) + } + + // ensure that the adjusted records are correctly updated in the cache + for _, originID := range originIDsToAdjust { + record, found := cache.Get(originID) + require.True(t, found) + require.NotNil(t, record) + require.Equal(t, -1.0, record.Penalty) + } +} + +// TestSpamRecordCache_ConcurrentIdentitiesAndOperations tests the concurrent calls to Identities method while +// other goroutines are initializing or removing spam records. The test covers the following scenarios: +// 1. Multiple goroutines initializing spam records for different origin IDs concurrently. +// 2. Multiple goroutines removing spam records for different origin IDs concurrently. +// 3. Multiple goroutines calling Identities method concurrently. +func TestSpamRecordCache_ConcurrentIdentitiesAndOperations(t *testing.T) { + sizeLimit := uint32(100) + logger := zerolog.Nop() + collector := metrics.NewNoopCollector() + recordFactory := func(id flow.Identifier) model.ProtocolSpamRecord { + return protocolSpamRecordFixture(id) + } + adjustFnNoOp := func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + return record, nil // no-op + } + + cache := internal.NewSpamRecordCache(sizeLimit, logger, collector, recordFactory) + require.NotNil(t, cache) + + originIDs := unittest.IdentifierListFixture(20) + originIDsToAdd := originIDs[:10] + originIDsToRemove := originIDs[10:20] + + for _, originID := range originIDsToRemove { + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + } + + var wg sync.WaitGroup + wg.Add(len(originIDs) + 10) + + // initialize spam records concurrently + for _, originID := range originIDsToAdd { + originID := originID + go func() { + defer wg.Done() + penalty, err := cache.Adjust(originID, adjustFnNoOp) + require.NoError(t, err) + require.Equal(t, float64(0), penalty) + retrieved, ok := cache.Get(originID) + require.True(t, ok) + require.NotNil(t, retrieved) + }() + } + + // remove spam records concurrently + for _, originID := range originIDsToRemove { + originID := originID + go func() { + defer wg.Done() + require.True(t, cache.Remove(originID)) + retrieved, ok := cache.Get(originID) + require.False(t, ok) + require.Nil(t, retrieved) + }() + } + + // call Identities method concurrently + for i := 0; i < 10; i++ { + go func() { + defer wg.Done() + ids := cache.Identities() + // the number of returned IDs should be less than or equal to the number of origin IDs + require.True(t, len(ids) <= len(originIDs)) + // the returned IDs should be a subset of the origin IDs + for _, id := range ids { + require.Contains(t, originIDs, id) + } + }() + } + + unittest.RequireReturnsBefore(t, wg.Wait, 1*time.Second, "timed out waiting for goroutines to finish") +} diff --git a/network/alsp/internal/reported_misbehavior_work.go b/network/alsp/internal/reported_misbehavior_work.go new file mode 100644 index 00000000000..c27c52b2225 --- /dev/null +++ b/network/alsp/internal/reported_misbehavior_work.go @@ -0,0 +1,36 @@ +package internal + +import ( + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" +) + +const NonceSize = 8 + +// ReportedMisbehaviorWork is an internal data structure for "temporarily" storing misbehavior reports in the queue +// till they are processed by the worker. +type ReportedMisbehaviorWork struct { + // Channel is the channel that the misbehavior report is about. + Channel channels.Channel + + // OriginId is the ID of the peer that the misbehavior report is about. + OriginId flow.Identifier + + // Reason is the reason of the misbehavior. + Reason network.Misbehavior + + // Nonce is a random nonce value that is used to make the key of the struct unique in the queue even when + // the same misbehavior report is reported multiple times. This is needed as we expect the same misbehavior report + // to be reported multiple times when an attack persists for a while. We don't want to deduplicate the misbehavior + // reports in the queue as we want to penalize the misbehaving node for each report. + Nonce [NonceSize]byte + + // Penalty is the penalty value of the misbehavior. + // We use `rlp:"-"` to ignore this field when serializing the struct to RLP to determine the key of this struct + // when storing in the queue. Hence, the penalty value does "not" contribute to the key for storing in the queue. + // As RLP encoding does not support float64, we cannot use this field as the key of the + // struct. As we use a random nonce value for the key of the struct, we can be sure that we will not have a collision + // in the queue, and duplicate reports will be accepted with unique keys. + Penalty float64 `rlp:"-"` +} diff --git a/network/alsp/manager/README.md b/network/alsp/manager/README.md new file mode 100644 index 00000000000..36bf3deda9d --- /dev/null +++ b/network/alsp/manager/README.md @@ -0,0 +1,84 @@ +# Application Layer Spam Prevention (ASLP) Manager +Implementation of ALSP manager is available here: [manager.go](manager.go) +Note that this readme is primarily focusing on the ALSP manager. For more details regarding the ALSP system please refer to [readme.md](..%2Freadme.md). +--- +## Architectural Overview +### Reporting Misbehavior and Managing Node Penalties +Figure below illustrates the ALSP manager’s role in the reporting of misbehavior and the management of node penalties as +well as the interactions between the ALSP manager and the `LibP2PNode`, `ConnectionGater`, and `PeerManager` components for +the disallow listing and allow listing processes. + +#### Reporting Misbehavior +In the event that an engine detects misbehavior within a channel, +it is imperative to report this finding to the ALSP manager. +This is achieved by invoking the `ReportMisbehavior` method on the conduit corresponding to the engine. + +#### Managing Penalties +The ALSP manager is responsible for maintaining records of misbehavior reports associated with +remote nodes and for calculating their accumulated misbehavior penalties. +Should a node’s misbehavior penalty surpass a certain threshold +(referred to as `DisallowListingThreshold`), the ALSP manager initiates the disallow listing process. When a remote node is disallow-listed, +it is effectively isolated from the network by the `ConnectionGater` and `PeerManager` components, i.e., the existing +connections to that remote node are closed and new connections attempts are rejected. + +##### Disallow Listing Process +1. The ALSP manager communicates with the `LibP2PNode` by calling its `OnDisallowListNotification` method to indicate that a particular remote node has been disallow-listed. +2. In response, the `LibP2PNode` takes two important actions: + + a. It alerts the `PeerManager`, instructing it to sever the connection with the disallow-listed node. + b. It notifies the `ConnectionGater` to block any incoming or outgoing connections to and from the disallow-listed node. +This ensures that the disallow-listed node is effectively isolated from the local node's network. + +##### Penalty Decay and Allow Listing Process +The ALSP manager also includes a penalty decay mechanism, which gradually reduces the penalties of nodes over time upon regular heartbeat intervals (default is every one second). +Once a disallow-listed node's penalty decays back to zero, the node can be reintegrated into the network through the allow listing process. The allow-listing process involves allowing +the `ConnectionGater` to lift the block on the disallow-listed node and instructing the `PeerManager` to initiate an outbound connection with the allow-listed node. + +1. The ALSP manager calls the `OnAllowListNotification` method on the `LibP2PNode` to signify that a previously disallow-listed node is now allow-listed. +2. The `LibP2PNode` responds by: + + a. Instructing the `ConnectionGater` to lift the block, thereby permitting connections with the now allow-listed node. + b. Requesting the `PeerManager` to initiate an outbound connection with the allow-listed node. + +This series of actions allows the rehabilitated node to be reintegrated and actively participate in the network once again. +![alsp-manager.png](alsp-manager.png) +--- + + + +## Developer Guidelines +The ALSP (Application Layer Spam Prevention) Manager handles application layer spamming misbehavior reports and penalizes misbehaving nodes. It also disallow-lists nodes whose penalties drop below a threshold. + + +- **Misbehavior Reports**: When a local engine detects a spamming misbehavior of a remote node, it sends a report to the ALSP manager, by invoking the `HandleMisbehaviorReport` method of the corresponding +conduit on which the misbehavior was detected. The manager handles the report in a thread-safe and non-blocking manner, using worker pools. + +```go +func (m *MisbehaviorReportManager) HandleMisbehaviorReport(channel channels.Channel, report network.MisbehaviorReport) { + // Handle the report +} +``` + +- **Penalties**: Misbehaving nodes are penalized by the manager. +The manager keeps a cache of records with penalties for each node. +The penalties are decayed over time through periodic heartbeats. + +- **Disallow-listing**: Nodes whose penalties drop below a threshold are disallow-listed. + +- **Heartbeats**: Periodic heartbeats allow the manager to perform recurring tasks, such as decaying the penalties of misbehaving nodes. +```go +func (m *MisbehaviorReportManager) heartbeatLoop(ctx irrecoverable.SignalerContext, interval time.Duration) { + // Handle heartbeats +} +``` + +- **Disallow-list Notification Consumer**: is the interface of the consumer of disallow-list notifications, which is +responsible for taking actions when a node is disallow-listed, i.e., closing exisitng connections with the remote disallow-listed +node and blocking any incoming or outgoing connections to that node. The consumer is passed to the manager when it is created. +In the current implementation the consumer is the instance of the `LibP2PNode` component of the node. +```go +disallowListingConsumer network.DisallowListNotificationConsumer +``` + +### Configuration +The configuration includes settings like cache size, heartbeat intervals, and network type. \ No newline at end of file diff --git a/network/alsp/manager/alsp-manager.png b/network/alsp/manager/alsp-manager.png new file mode 100644 index 00000000000..97e111e532b Binary files /dev/null and b/network/alsp/manager/alsp-manager.png differ diff --git a/network/alsp/manager/manager.go b/network/alsp/manager/manager.go new file mode 100644 index 00000000000..a1c3e25bf03 --- /dev/null +++ b/network/alsp/manager/manager.go @@ -0,0 +1,430 @@ +package alspmgr + +import ( + crand "crypto/rand" + "errors" + "fmt" + "math" + "time" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine/common/worker" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mempool/queue" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/alsp" + "github.com/onflow/flow-go/network/alsp/internal" + "github.com/onflow/flow-go/network/alsp/model" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/utils/logging" +) + +const ( + // defaultMisbehaviorReportManagerWorkers is the default number of workers in the worker pool. + defaultMisbehaviorReportManagerWorkers = 2 +) + +var ( + // ErrSpamRecordCacheSizeNotSet is returned when the spam record cache size is not set, it is a fatal irrecoverable error, + // and the ALSP module cannot be initialized. + ErrSpamRecordCacheSizeNotSet = errors.New("spam record cache size is not set") + // ErrSpamReportQueueSizeNotSet is returned when the spam report queue size is not set, it is a fatal irrecoverable error, + // and the ALSP module cannot be initialized. + ErrSpamReportQueueSizeNotSet = errors.New("spam report queue size is not set") + // ErrHeartBeatIntervalNotSet is returned when the heartbeat interval is not set, it is a fatal irrecoverable error, + // and the ALSP module cannot be initialized. + ErrHeartBeatIntervalNotSet = errors.New("heartbeat interval is not set") +) + +type SpamRecordCacheFactory func(zerolog.Logger, uint32, module.HeroCacheMetrics) alsp.SpamRecordCache + +// defaultSpamRecordCacheFactory is the default spam record cache factory. It creates a new spam record cache with the given parameter. +func defaultSpamRecordCacheFactory() SpamRecordCacheFactory { + return func(logger zerolog.Logger, size uint32, cacheMetrics module.HeroCacheMetrics) alsp.SpamRecordCache { + return internal.NewSpamRecordCache( + size, + logger.With().Str("component", "spam_record_cache").Logger(), + cacheMetrics, + model.SpamRecordFactory()) + } +} + +// MisbehaviorReportManager is responsible for handling misbehavior reports, i.e., penalizing the misbehaving node +// and report the node to be disallow-listed if the overall penalty of the misbehaving node drops below the disallow-listing threshold. +type MisbehaviorReportManager struct { + component.Component + logger zerolog.Logger + metrics module.AlspMetrics + // cacheFactory is the factory for creating the spam record cache. MisbehaviorReportManager is coming with a + // default factory that creates a new spam record cache with the given parameter. However, this factory can be + // overridden with a custom factory. + cacheFactory SpamRecordCacheFactory + // cache is the spam record cache that stores the spam records for the authorized nodes. It is initialized by + // invoking the cacheFactory. + cache alsp.SpamRecordCache + // disablePenalty indicates whether applying the penalty to the misbehaving node is disabled. + // When disabled, the ALSP module logs the misbehavior reports and updates the metrics, but does not apply the penalty. + // This is useful for managing production incidents. + // Note: under normal circumstances, the ALSP module should not be disabled. + disablePenalty bool + + // disallowListingConsumer is the consumer for the disallow-listing notifications. + // It is notified when a node is disallow-listed by this manager. + disallowListingConsumer network.DisallowListNotificationConsumer + + // workerPool is the worker pool for handling the misbehavior reports in a thread-safe and non-blocking manner. + workerPool *worker.Pool[internal.ReportedMisbehaviorWork] +} + +var _ network.MisbehaviorReportManager = (*MisbehaviorReportManager)(nil) + +type MisbehaviorReportManagerConfig struct { + Logger zerolog.Logger + // SpamRecordCacheSize is the size of the spam record cache that stores the spam records for the authorized nodes. + // It should be as big as the number of authorized nodes in Flow network. + // Recommendation: for small network sizes 10 * number of authorized nodes to ensure that the cache can hold all the spam records of the authorized nodes. + SpamRecordCacheSize uint32 + // SpamReportQueueSize is the size of the queue that stores the spam records to be processed by the worker pool. + SpamReportQueueSize uint32 + // AlspMetrics is the metrics instance for the alsp module (collecting spam reports). + AlspMetrics module.AlspMetrics + // HeroCacheMetricsFactory is the metrics factory for the HeroCache-related metrics. + // Having factory as part of the config allows to create the metrics locally in the module. + HeroCacheMetricsFactory metrics.HeroCacheMetricsFactory + // DisablePenalty indicates whether applying the penalty to the misbehaving node is disabled. + // When disabled, the ALSP module logs the misbehavior reports and updates the metrics, but does not apply the penalty. + // This is useful for managing production incidents. + // Note: under normal circumstances, the ALSP module should not be disabled. + DisablePenalty bool + // NetworkType is the type of the network it is used to determine whether the ALSP module is utilized in the + // public (unstaked) or private (staked) network. + NetworkType network.NetworkingType + // HeartBeatInterval is the interval between the heartbeats. Heartbeat is a recurring event that is used to + // apply recurring actions, e.g., decay the penalty of the misbehaving nodes. + HeartBeatInterval time.Duration + Opts []MisbehaviorReportManagerOption +} + +// validate validates the MisbehaviorReportManagerConfig instance. It returns an error if the config is invalid. +// It only validates the numeric fields of the config that may yield a stealth error in the production. +// It does not validate the struct fields of the config against a nil value. +// Args: +// +// None. +// +// Returns: +// +// An error if the config is invalid. +func (c MisbehaviorReportManagerConfig) validate() error { + if c.SpamRecordCacheSize == 0 { + return ErrSpamRecordCacheSizeNotSet + } + if c.SpamReportQueueSize == 0 { + return ErrSpamReportQueueSizeNotSet + } + if c.HeartBeatInterval == 0 { + return ErrHeartBeatIntervalNotSet + } + return nil +} + +type MisbehaviorReportManagerOption func(*MisbehaviorReportManager) + +// WithSpamRecordsCacheFactory sets the spam record cache factory for the MisbehaviorReportManager. +// Args: +// +// f: the spam record cache factory. +// +// Returns: +// +// a MisbehaviorReportManagerOption that sets the spam record cache for the MisbehaviorReportManager. +// +// Note: this option is useful primarily for testing purposes. The default factory should be sufficient for the production, and +// do not change it unless you are confident that you know what you are doing. +func WithSpamRecordsCacheFactory(f SpamRecordCacheFactory) MisbehaviorReportManagerOption { + return func(m *MisbehaviorReportManager) { + m.cacheFactory = f + } +} + +// NewMisbehaviorReportManager creates a new instance of the MisbehaviorReportManager. +// Args: +// cfg: the configuration for the MisbehaviorReportManager. +// consumer: the consumer for the disallow-listing notifications. When the manager decides to disallow-list a node, it notifies the consumer to +// perform the lower-level disallow-listing action at the networking layer. +// All connections to the disallow-listed node are closed and the node is removed from the overlay, and +// no further connections are established to the disallow-listed node, either inbound or outbound. +// +// Returns: +// +// A new instance of the MisbehaviorReportManager. +// An error if the config is invalid. The error is considered irrecoverable. +func NewMisbehaviorReportManager(cfg *MisbehaviorReportManagerConfig, consumer network.DisallowListNotificationConsumer) (*MisbehaviorReportManager, error) { + if err := cfg.validate(); err != nil { + return nil, fmt.Errorf("invalid configuration for MisbehaviorReportManager: %w", err) + } + + lg := cfg.Logger.With().Str("module", "misbehavior_report_manager").Logger() + m := &MisbehaviorReportManager{ + logger: lg, + metrics: cfg.AlspMetrics, + disablePenalty: cfg.DisablePenalty, + disallowListingConsumer: consumer, + cacheFactory: defaultSpamRecordCacheFactory(), + } + + store := queue.NewHeroStore( + cfg.SpamReportQueueSize, + lg.With().Str("component", "spam_record_queue").Logger(), + metrics.ApplicationLayerSpamRecordQueueMetricsFactory(cfg.HeroCacheMetricsFactory, cfg.NetworkType)) + + m.workerPool = worker.NewWorkerPoolBuilder[internal.ReportedMisbehaviorWork]( + cfg.Logger, + store, + m.processMisbehaviorReport).Build() + + for _, opt := range cfg.Opts { + opt(m) + } + + m.cache = m.cacheFactory( + lg, + cfg.SpamRecordCacheSize, + metrics.ApplicationLayerSpamRecordCacheMetricFactory(cfg.HeroCacheMetricsFactory, cfg.NetworkType)) + + builder := component.NewComponentManagerBuilder() + builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + m.heartbeatLoop(ctx, cfg.HeartBeatInterval) // blocking call + }) + for i := 0; i < defaultMisbehaviorReportManagerWorkers; i++ { + builder.AddWorker(m.workerPool.WorkerLogic()) + } + + m.Component = builder.Build() + + if m.disablePenalty { + m.logger.Warn().Msg("penalty mechanism of alsp is disabled") + } + return m, nil +} + +// HandleMisbehaviorReport is called upon a new misbehavior is reported. +// The implementation of this function should be thread-safe and non-blocking. +// Args: +// +// channel: the channel on which the misbehavior is reported. +// report: the misbehavior report. +// +// Returns: +// +// none. +func (m *MisbehaviorReportManager) HandleMisbehaviorReport(channel channels.Channel, report network.MisbehaviorReport) { + lg := m.logger.With(). + Str("channel", channel.String()). + Hex("misbehaving_id", logging.ID(report.OriginId())). + Str("reason", report.Reason().String()). + Float64("penalty", report.Penalty()).Logger() + lg.Trace().Msg("received misbehavior report") + m.metrics.OnMisbehaviorReported(channel.String(), report.Reason().String()) + + nonce := [internal.NonceSize]byte{} + nonceSize, err := crand.Read(nonce[:]) + if err != nil { + // this should never happen, but if it does, we should not continue + lg.Fatal().Err(err).Msg("failed to generate nonce") + return + } + if nonceSize != internal.NonceSize { + // this should never happen, but if it does, we should not continue + lg.Fatal().Msgf("nonce size mismatch: expected %d, got %d", internal.NonceSize, nonceSize) + return + } + + if ok := m.workerPool.Submit(internal.ReportedMisbehaviorWork{ + Channel: channel, + OriginId: report.OriginId(), + Reason: report.Reason(), + Penalty: report.Penalty(), + Nonce: nonce, + }); !ok { + lg.Warn().Msg("discarding misbehavior report because either the queue is full or the misbehavior report is duplicate") + } + + lg.Debug().Msg("misbehavior report submitted") +} + +// heartbeatLoop starts the heartbeat ticks ticker to tick at the given intervals. It is a blocking function, and +// should be called in a separate goroutine. It returns when the context is canceled. Hearbeats are recurring events that +// are used to perform periodic tasks. +// Args: +// +// ctx: the context. +// interval: the interval between two ticks. +// +// Returns: +// +// none. +func (m *MisbehaviorReportManager) heartbeatLoop(ctx irrecoverable.SignalerContext, interval time.Duration) { + ticker := time.NewTicker(interval) + m.logger.Info().Dur("interval", interval).Msg("starting heartbeat ticks") + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + m.logger.Debug().Msg("heartbeat ticks stopped") + return + case <-ticker.C: + m.logger.Trace().Msg("new heartbeat ticked") + if err := m.onHeartbeat(); err != nil { + // any error returned from onHeartbeat is considered irrecoverable. + ctx.Throw(fmt.Errorf("failed to perform heartbeat: %w", err)) + } + } + } +} + +// onHeartbeat is called upon a heartbeatLoop. It encapsulates the recurring tasks that should be performed +// during a heartbeat, which currently includes decay of the spam records. +// Args: +// +// none. +// +// Returns: +// +// error: if an error occurs, it is returned. No error is expected during normal operation. Any returned error must +// be considered as irrecoverable. +func (m *MisbehaviorReportManager) onHeartbeat() error { + allIds := m.cache.Identities() + + for _, id := range allIds { + penalty, err := m.cache.Adjust(id, func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + if record.Penalty > 0 { + // sanity check; this should never happen. + return record, fmt.Errorf("illegal state: spam record %x has positive penalty %f", id, record.Penalty) + } + if record.Decay <= 0 { + // sanity check; this should never happen. + return record, fmt.Errorf("illegal state: spam record %x has non-positive decay %f", id, record.Decay) + } + + // TODO: this can be done in batch but at this stage let's send individual notifications. + // (it requires enabling the batch mode end-to-end including the cache in middleware). + // as long as record.Penalty is NOT below model.DisallowListingThreshold, + // the node is considered allow-listed and can conduct inbound and outbound connections. + // Once it falls below model.DisallowListingThreshold, it needs to be disallow listed. + if record.Penalty < model.DisallowListingThreshold && !record.DisallowListed { + // cutoff counter keeps track of how many times the penalty has been below the threshold. + record.CutoffCounter++ + record.DisallowListed = true + m.logger.Warn(). + Str("key", logging.KeySuspicious). + Hex("identifier", logging.ID(id)). + Uint64("cutoff_counter", record.CutoffCounter). + Float64("decay_speed", record.Decay). + Bool("disallow_listed", record.DisallowListed). + Msg("node penalty is below threshold, disallow listing") + m.disallowListingConsumer.OnDisallowListNotification(&network.DisallowListingUpdate{ + FlowIds: flow.IdentifierList{id}, + Cause: network.DisallowListedCauseAlsp, // sets the ALSP disallow listing cause on node + }) + } + // each time we decay the penalty by the decay speed, the penalty is a negative number, and the decay speed + // is a positive number. So the penalty is getting closer to zero. + // We use math.Min() to make sure the penalty is never positive. + record.Penalty = math.Min(record.Penalty+record.Decay, 0) + + // TODO: this can be done in batch but at this stage let's send individual notifications. + // (it requires enabling the batch mode end-to-end including the cache in middleware). + if record.Penalty == float64(0) && record.DisallowListed { + record.DisallowListed = false + m.logger.Info(). + Hex("identifier", logging.ID(id)). + Uint64("cutoff_counter", record.CutoffCounter). + Float64("decay_speed", record.Decay). + Bool("disallow_listed", record.DisallowListed). + Msg("allow-listing a node that was disallow listed") + // Penalty has fully decayed to zero and the node can be back in the allow list. + m.disallowListingConsumer.OnAllowListNotification(&network.AllowListingUpdate{ + FlowIds: flow.IdentifierList{id}, + Cause: network.DisallowListedCauseAlsp, // clears the ALSP disallow listing cause from node + }) + } + + m.logger.Trace(). + Hex("identifier", logging.ID(id)). + Uint64("cutoff_counter", record.CutoffCounter). + Float64("decay_speed", record.Decay). + Bool("disallow_listed", record.DisallowListed). + Msg("spam record decayed successfully") + return record, nil + }) + + // any error here is fatal because it indicates a bug in the cache. All ids being iterated over are in the cache, + // and adjust function above should not return an error unless there is a bug. + if err != nil { + return fmt.Errorf("failed to decay spam record %x: %w", id, err) + } + + m.logger.Trace(). + Hex("identifier", logging.ID(id)). + Float64("updated_penalty", penalty). + Msg("spam record decayed") + } + + return nil +} + +// processMisbehaviorReport is the worker function that processes the misbehavior reports. +// It is called by the worker pool. +// It applies the penalty to the misbehaving node and updates the spam record cache. +// Implementation must be thread-safe so that it can be called concurrently. +// Args: +// +// report: the misbehavior report to be processed. +// +// Returns: +// +// error: the error that occurred during the processing of the misbehavior report. The returned error is +// irrecoverable and the node should crash if it occurs (indicating a bug in the ALSP module). +func (m *MisbehaviorReportManager) processMisbehaviorReport(report internal.ReportedMisbehaviorWork) error { + lg := m.logger.With(). + Str("channel", report.Channel.String()). + Hex("misbehaving_id", logging.ID(report.OriginId)). + Str("reason", report.Reason.String()). + Float64("penalty", report.Penalty).Logger() + + if m.disablePenalty { + // when penalty mechanism disabled, the misbehavior is logged and metrics are updated, + // but no further actions are taken. + lg.Trace().Msg("discarding misbehavior report because alsp penalty is disabled") + return nil + } + + // Adjust will first try to apply the penalty to the spam record, if it does not exist, the Adjust method will initialize + // a spam record for the peer first and then applies the penalty. In other words, Adjust uses an optimistic update by + // first assuming that the spam record exists and then initializing it if it does not exist. In this way, we avoid + // acquiring the lock twice per misbehavior report, reducing the contention on the lock and improving the performance. + updatedPenalty, err := m.cache.Adjust(report.OriginId, func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + if report.Penalty > 0 { + // this should never happen, unless there is a bug in the misbehavior report handling logic. + // we should crash the node in this case to prevent further misbehavior reports from being lost and fix the bug. + // we return the error as it is considered as a fatal error. + return record, fmt.Errorf("penalty value is positive, expected negative %f", report.Penalty) + } + record.Penalty += report.Penalty // penalty value is negative. We add it to the current penalty. + return record, nil + }) + if err != nil { + // this should never happen, unless there is a bug in the spam record cache implementation. + // we should crash the node in this case to prevent further misbehavior reports from being lost and fix the bug. + return fmt.Errorf("failed to apply penalty to the spam record: %w", err) + } + lg.Debug().Float64("updated_penalty", updatedPenalty).Msg("misbehavior report handled") + return nil +} diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go new file mode 100644 index 00000000000..b013688bf8b --- /dev/null +++ b/network/alsp/manager/manager_test.go @@ -0,0 +1,1620 @@ +package alspmgr_test + +import ( + "context" + "math" + "math/rand" + "sync" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/config" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + mockmodule "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/alsp" + "github.com/onflow/flow-go/network/alsp/internal" + alspmgr "github.com/onflow/flow-go/network/alsp/manager" + mockalsp "github.com/onflow/flow-go/network/alsp/mock" + "github.com/onflow/flow-go/network/alsp/model" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/internal/testutils" + "github.com/onflow/flow-go/network/mocknetwork" + "github.com/onflow/flow-go/network/p2p" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/network/slashing" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestHandleReportedMisbehavior tests the handling of reported misbehavior by the network. +// +// The test sets up a mock MisbehaviorReportManager and a conduitFactory with this manager. +// It generates a single node network with the conduitFactory and starts it. +// It then uses a mock engine to register a channel with the network. +// It prepares a set of misbehavior reports and reports them to the conduit on the test channel. +// The test ensures that the MisbehaviorReportManager receives and handles all reported misbehavior +// without any duplicate reports and within a specified time. +func TestNetworkPassesReportedMisbehavior(t *testing.T) { + misbehaviorReportManger := mocknetwork.NewMisbehaviorReportManager(t) + misbehaviorReportManger.On("Start", mock.Anything).Return().Once() + + readyDoneChan := func() <-chan struct{} { + ch := make(chan struct{}) + close(ch) + return ch + }() + + misbehaviorReportManger.On("Ready").Return(readyDoneChan).Once() + misbehaviorReportManger.On("Done").Return(readyDoneChan).Once() + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 1) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0]) + net, err := p2p.NewNetwork(networkCfg, p2p.WithAlspManager(misbehaviorReportManger)) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.Network{net}, 100*time.Millisecond) + defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) + defer cancel() + + e := mocknetwork.NewEngine(t) + con, err := net.Register(channels.TestNetworkChannel, e) + require.NoError(t, err) + + reports := testutils.MisbehaviorReportsFixture(t, 10) + allReportsManaged := sync.WaitGroup{} + allReportsManaged.Add(len(reports)) + var seenReports []network.MisbehaviorReport + misbehaviorReportManger.On("HandleMisbehaviorReport", channels.TestNetworkChannel, mock.Anything).Run(func(args mock.Arguments) { + report := args.Get(1).(network.MisbehaviorReport) + require.Contains(t, reports, report) // ensures that the report is one of the reports we expect. + require.NotContainsf(t, seenReports, report, "duplicate report: %v", report) // ensures that we have not seen this report before. + seenReports = append(seenReports, report) // adds the report to the list of seen reports. + allReportsManaged.Done() + }).Return(nil) + + for _, report := range reports { + con.ReportMisbehavior(report) // reports the misbehavior + } + + unittest.RequireReturnsBefore(t, allReportsManaged.Wait, 100*time.Millisecond, "did not receive all reports") +} + +// TestHandleReportedMisbehavior tests the handling of reported misbehavior by the network. +// +// The test sets up a mock MisbehaviorReportManager and a conduitFactory with this manager. +// It generates a single node network with the conduitFactory and starts it. +// It then uses a mock engine to register a channel with the network. +// It prepares a set of misbehavior reports and reports them to the conduit on the test channel. +// The test ensures that the MisbehaviorReportManager receives and handles all reported misbehavior +// without any duplicate reports and within a specified time. +func TestHandleReportedMisbehavior_Cache_Integration(t *testing.T) { + cfg := managerCfgFixture(t) + + // this test is assessing the integration of the ALSP manager with the network. As the ALSP manager is an attribute + // of the network, we need to configure the ALSP manager via the network configuration, and let the network create + // the ALSP manager. + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 1) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) + net, err := p2p.NewNetwork(networkCfg) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.Network{net}, 100*time.Millisecond) + defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) + defer cancel() + + e := mocknetwork.NewEngine(t) + con, err := net.Register(channels.TestNetworkChannel, e) + require.NoError(t, err) + + // create a map of origin IDs to their respective misbehavior reports (10 peers, 5 reports each) + numPeers := 10 + numReportsPerPeer := 5 + peersReports := make(map[flow.Identifier][]network.MisbehaviorReport) + + for i := 0; i < numPeers; i++ { + originID := unittest.IdentifierFixture() + reports := createRandomMisbehaviorReportsForOriginId(t, originID, numReportsPerPeer) + peersReports[originID] = reports + } + + wg := sync.WaitGroup{} + for _, reports := range peersReports { + wg.Add(len(reports)) + // reports the misbehavior + for _, report := range reports { + r := report // capture range variable + go func() { + defer wg.Done() + + con.ReportMisbehavior(r) + }() + } + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") + + // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache + require.Eventually(t, func() bool { + for originID, reports := range peersReports { + totalPenalty := float64(0) + for _, report := range reports { + totalPenalty += report.Penalty() + } + + record, ok := cache.Get(originID) + if !ok { + return false + } + require.NotNil(t, record) + + require.Equal(t, totalPenalty, record.Penalty) + // with just reporting a single misbehavior report, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // with just reporting a single misbehavior report, the node should not be disallowed. + require.False(t, record.DisallowListed) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + } + + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") +} + +// TestHandleReportedMisbehavior_And_DisallowListing_Integration implements an end-to-end integration test for the +// handling of reported misbehavior and disallow listing. +// +// The test sets up 3 nodes, one victim, one honest, and one (alleged) spammer. +// Initially, the test ensures that all nodes are connected to each other. +// Then, test imitates that victim node reports the spammer node for spamming. +// The test generates enough spam reports to trigger the disallow-listing of the victim node. +// The test ensures that the victim node is disconnected from the spammer node. +// The test ensures that despite attempting on connections, no inbound or outbound connections between the victim and +// the disallow-listed spammer node are established. +func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) { + cfg := managerCfgFixture(t) + + // this test is assessing the integration of the ALSP manager with the network. As the ALSP manager is an attribute + // of the network, we need to configure the ALSP manager via the network configuration, and let the network create + // the ALSP manager. + var victimSpamRecordCache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + victimSpamRecordCache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return victimSpamRecordCache + }), + } + + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 3, + p2ptest.WithPeerManagerEnabled(p2ptest.PeerManagerConfigFixture(), nil)) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) + victimNetwork, err := p2p.NewNetwork(networkCfg) + require.NoError(t, err) + + // index of the victim node in the nodes slice. + victimIndex := 0 + // index of the spammer node in the nodes slice (the node that will be reported for misbehavior and disallow-listed by victim). + spammerIndex := 1 + // other node (not victim and not spammer) that we have to ensure is not affected by the disallow-listing of the spammer. + honestIndex := 2 + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.Network{victimNetwork}, 100*time.Millisecond) + defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) + defer cancel() + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + // initially victim and spammer should be able to connect to each other. + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + e := mocknetwork.NewEngine(t) + con, err := victimNetwork.Register(channels.TestNetworkChannel, e) + require.NoError(t, err) + + // creates a misbehavior report for the spammer + report := misbehaviorReportFixtureWithPenalty(t, ids[spammerIndex].NodeID, model.DefaultPenaltyValue) + + // simulates the victim node reporting the spammer node misbehavior 120 times + // to the network. As each report has the default penalty, ideally the spammer should be disallow-listed after + // 100 reports (each having 0.01 * disallow-listing penalty). But we take 120 as a safe number to ensure that + // the spammer is definitely disallow-listed. + reportCount := 120 + wg := sync.WaitGroup{} + for i := 0; i < reportCount; i++ { + wg.Add(1) + // reports the misbehavior + r := report // capture range variable + go func() { + defer wg.Done() + + con.ReportMisbehavior(r) + }() + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") + + // ensures that the spammer is disallow-listed by the victim + p2ptest.RequireEventuallyNotConnected(t, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[spammerIndex]}, 100*time.Millisecond, 2*time.Second) + + // despite disallow-listing spammer, ensure that (victim and honest) and (honest and spammer) are still connected. + p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[spammerIndex], nodes[honestIndex]}, 1*time.Millisecond, 100*time.Millisecond) + p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestIndex], nodes[victimIndex]}, 1*time.Millisecond, 100*time.Millisecond) + + // while the spammer node is disallow-listed, it cannot connect to the victim node. Also, the victim node cannot directly dial and connect to the spammer node, unless + // it is allow-listed again. + p2ptest.EnsureNotConnectedBetweenGroups(t, ctx, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[spammerIndex]}) +} + +// TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration implements an end-to-end integration test for the +// handling of reported misbehavior from the slashing.ViolationsConsumer. +// +// The test sets up one victim, one honest, and one (alleged) spammer for each of the current slashing violations. +// Initially, the test ensures that all nodes are connected to each other. +// Then, test imitates the slashing violations consumer on the victim node reporting misbehavior's for each slashing violation. +// The test generates enough slashing violations to trigger the connection to each of the spamming nodes to be eventually pruned. +// The test ensures that the victim node is disconnected from all spammer nodes. +// The test ensures that despite attempting on connections, no inbound or outbound connections between the victim and +// the pruned spammer nodes are established. +func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t *testing.T) { + + // create 1 victim node, 1 honest node and a node for each slashing violation + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 7) // creates 7 nodes (1 victim, 1 honest, 5 spammer nodes one for each slashing violation). + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(managerCfgFixture(t))) + victimNetwork, err := p2p.NewNetwork(networkCfg) + require.NoError(t, err) + + // create slashing violations consumer with victim node network providing the network.MisbehaviorReportConsumer interface + violationsConsumer := slashing.NewSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), victimNetwork) + mws[0].SetSlashingViolationsConsumer(violationsConsumer) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.Network{victimNetwork}, 100*time.Millisecond) + defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) + defer cancel() + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + // initially victim and misbehaving nodes should be able to connect to each other. + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // each slashing violation func is mapped to a violation with the identity of one of the misbehaving nodes + // index of the victim node in the nodes slice. + victimIndex := 0 + honestNodeIndex := 1 + invalidMessageIndex := 2 + senderEjectedIndex := 3 + unauthorizedUnicastOnChannelIndex := 4 + unauthorizedPublishOnChannelIndex := 5 + unknownMsgTypeIndex := 6 + slashingViolationTestCases := []struct { + violationsConsumerFunc func(violation *network.Violation) + violation *network.Violation + }{ + {violationsConsumer.OnUnAuthorizedSenderError, &network.Violation{Identity: ids[invalidMessageIndex]}}, + {violationsConsumer.OnSenderEjectedError, &network.Violation{Identity: ids[senderEjectedIndex]}}, + {violationsConsumer.OnUnauthorizedUnicastOnChannel, &network.Violation{Identity: ids[unauthorizedUnicastOnChannelIndex]}}, + {violationsConsumer.OnUnauthorizedPublishOnChannel, &network.Violation{Identity: ids[unauthorizedPublishOnChannelIndex]}}, + {violationsConsumer.OnUnknownMsgTypeError, &network.Violation{Identity: ids[unknownMsgTypeIndex]}}, + } + + violationsWg := sync.WaitGroup{} + violationCount := 120 + for _, testCase := range slashingViolationTestCases { + for i := 0; i < violationCount; i++ { + testCase := testCase + violationsWg.Add(1) + go func() { + defer violationsWg.Done() + testCase.violationsConsumerFunc(testCase.violation) + }() + } + } + unittest.RequireReturnsBefore(t, violationsWg.Wait, 100*time.Millisecond, "slashing violations not reported in time") + + forEachMisbehavingNode := func(f func(i int)) { + for misbehavingNodeIndex := 2; misbehavingNodeIndex <= len(nodes)-1; misbehavingNodeIndex++ { + f(misbehavingNodeIndex) + } + } + + // ensures all misbehaving nodes are disconnected from the victim node + forEachMisbehavingNode(func(misbehavingNodeIndex int) { + p2ptest.RequireEventuallyNotConnected(t, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[misbehavingNodeIndex]}, 100*time.Millisecond, 2*time.Second) + }) + + // despite being disconnected from the victim node, misbehaving nodes and the honest node are still connected. + forEachMisbehavingNode(func(misbehavingNodeIndex int) { + p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestNodeIndex], nodes[misbehavingNodeIndex]}, 1*time.Millisecond, 100*time.Millisecond) + }) + + // despite disconnecting misbehaving nodes, ensure that (victim and honest) are still connected. + p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestNodeIndex], nodes[victimIndex]}, 1*time.Millisecond, 100*time.Millisecond) + + // while misbehaving nodes are disconnected, they cannot connect to the victim node. Also, the victim node cannot directly dial and connect to the misbehaving nodes until each node's peer score decays. + forEachMisbehavingNode(func(misbehavingNodeIndex int) { + p2ptest.EnsureNotConnectedBetweenGroups(t, ctx, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[misbehavingNodeIndex]}) + }) +} + +// TestMisbehaviorReportMetrics tests the recording of misbehavior report metrics. +// It checks that when a misbehavior report is received by the ALSP manager, the metrics are recorded. +// It fails the test if the metrics are not recorded or if they are recorded incorrectly. +func TestMisbehaviorReportMetrics(t *testing.T) { + cfg := managerCfgFixture(t) + + // this test is assessing the integration of the ALSP manager with the network. As the ALSP manager is an attribute + // of the network, we need to configure the ALSP manager via the network configuration, and let the network create + // the ALSP manager. + alspMetrics := mockmodule.NewAlspMetrics(t) + cfg.AlspMetrics = alspMetrics + + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 1) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) + net, err := p2p.NewNetwork(networkCfg) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.Network{net}, 100*time.Millisecond) + defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) + defer cancel() + + e := mocknetwork.NewEngine(t) + con, err := net.Register(channels.TestNetworkChannel, e) + require.NoError(t, err) + + report := testutils.MisbehaviorReportFixture(t) + + // this channel is used to signal that the metrics have been recorded by the ALSP manager correctly. + reported := make(chan struct{}) + + // ensures that the metrics are recorded when a misbehavior report is received. + alspMetrics.On("OnMisbehaviorReported", channels.TestNetworkChannel.String(), report.Reason().String()).Run(func(args mock.Arguments) { + close(reported) + }).Once() + + con.ReportMisbehavior(report) // reports the misbehavior + + unittest.RequireCloseBefore(t, reported, 100*time.Millisecond, "metrics for the misbehavior report were not recorded") +} + +// The TestReportCreation tests the creation of misbehavior reports using the alsp.NewMisbehaviorReport function. +// The function tests the creation of both valid and invalid misbehavior reports by setting different penalty amplification values. +func TestReportCreation(t *testing.T) { + + // creates a valid misbehavior report (i.e., amplification between 1 and 100) + report, err := alsp.NewMisbehaviorReport( + unittest.IdentifierFixture(), + testutils.MisbehaviorTypeFixture(t), + alsp.WithPenaltyAmplification(10)) + require.NoError(t, err) + require.NotNil(t, report) + + // creates a valid misbehavior report with default amplification. + report, err = alsp.NewMisbehaviorReport( + unittest.IdentifierFixture(), + testutils.MisbehaviorTypeFixture(t)) + require.NoError(t, err) + require.NotNil(t, report) + + // creates an in valid misbehavior report (i.e., amplification greater than 100 and less than 1) + report, err = alsp.NewMisbehaviorReport( + unittest.IdentifierFixture(), + testutils.MisbehaviorTypeFixture(t), + alsp.WithPenaltyAmplification(100*rand.Float64()-101)) + require.Error(t, err) + require.Nil(t, report) + + report, err = alsp.NewMisbehaviorReport( + unittest.IdentifierFixture(), + testutils.MisbehaviorTypeFixture(t), + alsp.WithPenaltyAmplification(100*rand.Float64()+101)) + require.Error(t, err) + require.Nil(t, report) + + // 0 is not a valid amplification + report, err = alsp.NewMisbehaviorReport( + unittest.IdentifierFixture(), + testutils.MisbehaviorTypeFixture(t), + alsp.WithPenaltyAmplification(0)) + require.Error(t, err) + require.Nil(t, report) +} + +// TestNewMisbehaviorReportManager tests the creation of a new ALSP manager. +// It is a minimum viable test that ensures that a non-nil ALSP manager is created with expected set of inputs. +// In other words, variation of input values do not cause a nil ALSP manager to be created or a panic. +func TestNewMisbehaviorReportManager(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + + t.Run("with default values", func(t *testing.T) { + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + assert.NotNil(t, m) + }) + + t.Run("with a custom spam record cache", func(t *testing.T) { + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + assert.NotNil(t, m) + }) + + t.Run("with ALSP module enabled", func(t *testing.T) { + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + assert.NotNil(t, m) + }) + + t.Run("with ALSP module disabled", func(t *testing.T) { + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + assert.NotNil(t, m) + }) +} + +// TestMisbehaviorReportManager_InitializationError tests the creation of a new ALSP manager with invalid inputs. +// It is a minimum viable test that ensures that a nil ALSP manager is created with invalid set of inputs. +func TestMisbehaviorReportManager_InitializationError(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + t.Run("missing spam report queue size", func(t *testing.T) { + cfg.SpamReportQueueSize = 0 + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.Error(t, err) + require.ErrorIs(t, err, alspmgr.ErrSpamReportQueueSizeNotSet) + assert.Nil(t, m) + }) + + t.Run("missing spam record cache size", func(t *testing.T) { + cfg.SpamRecordCacheSize = 0 + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.Error(t, err) + require.ErrorIs(t, err, alspmgr.ErrSpamRecordCacheSizeNotSet) + assert.Nil(t, m) + }) + + t.Run("missing heartbeat intervals", func(t *testing.T) { + cfg.HeartBeatInterval = 0 + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.Error(t, err) + require.ErrorIs(t, err, alspmgr.ErrSpamRecordCacheSizeNotSet) + assert.Nil(t, m) + }) +} + +// TestHandleMisbehaviorReport_SinglePenaltyReport tests the handling of a single misbehavior report. +// The test ensures that the misbehavior report is handled correctly and the penalty is applied to the peer in the cache. +func TestHandleMisbehaviorReport_SinglePenaltyReport(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + // create a new MisbehaviorReportManager + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // create a mock misbehavior report with a negative penalty value + penalty := float64(-5) + report := mocknetwork.NewMisbehaviorReport(t) + report.On("OriginId").Return(unittest.IdentifierFixture()) + report.On("Reason").Return(alsp.InvalidMessage) + report.On("Penalty").Return(penalty) + + channel := channels.Channel("test-channel") + + // handle the misbehavior report + m.HandleMisbehaviorReport(channel, report) + + require.Eventually(t, func() bool { + // check if the misbehavior report has been processed by verifying that the Adjust method was called on the cache + record, ok := cache.Get(report.OriginId()) + if !ok { + return false + } + require.NotNil(t, record) + require.Equal(t, penalty, record.Penalty) + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet + require.Equal(t, uint64(0), record.CutoffCounter) // with just reporting a misbehavior, the cutoff counter should not be incremented. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) // the decay should be the default decay value. + + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") +} + +// TestHandleMisbehaviorReport_SinglePenaltyReport_PenaltyDisable tests the handling of a single misbehavior report when the penalty is disabled. +// The test ensures that the misbehavior is reported on metrics but the penalty is not applied to the peer in the cache. +func TestHandleMisbehaviorReport_SinglePenaltyReport_PenaltyDisable(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + cfg.DisablePenalty = true // disable penalty for misbehavior reports + alspMetrics := mockmodule.NewAlspMetrics(t) + cfg.AlspMetrics = alspMetrics + + // we use a mock cache but we do not expect any calls to the cache, since the penalty is disabled. + var cache *mockalsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = mockalsp.NewSpamRecordCache(t) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // create a mock misbehavior report with a negative penalty value + penalty := float64(-5) + report := mocknetwork.NewMisbehaviorReport(t) + report.On("OriginId").Return(unittest.IdentifierFixture()) + report.On("Reason").Return(alsp.InvalidMessage) + report.On("Penalty").Return(penalty) + + channel := channels.Channel("test-channel") + + // this channel is used to signal that the metrics have been recorded by the ALSP manager correctly. + // even in case of a disabled penalty, the metrics should be recorded. + reported := make(chan struct{}) + + // ensures that the metrics are recorded when a misbehavior report is received. + alspMetrics.On("OnMisbehaviorReported", channel.String(), report.Reason().String()).Run(func(args mock.Arguments) { + close(reported) + }).Once() + + // handle the misbehavior report + m.HandleMisbehaviorReport(channel, report) + + unittest.RequireCloseBefore(t, reported, 100*time.Millisecond, "metrics for the misbehavior report were not recorded") + + // since the penalty is disabled, we do not expect any calls to the cache. + cache.AssertNotCalled(t, "Adjust", mock.Anything, mock.Anything) +} + +// TestHandleMisbehaviorReport_MultiplePenaltyReportsForSinglePeer_Sequentially tests the handling of multiple misbehavior reports for a single peer. +// Reports are coming in sequentially. +// The test ensures that each misbehavior report is handled correctly and the penalties are cumulatively applied to the peer in the cache. +func TestHandleMisbehaviorReport_MultiplePenaltyReportsForSinglePeer_Sequentially(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + // create a new MisbehaviorReportManager + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // creates a list of mock misbehavior reports with negative penalty values for a single peer + originId := unittest.IdentifierFixture() + reports := createRandomMisbehaviorReportsForOriginId(t, originId, 5) + + channel := channels.Channel("test-channel") + + // handle the misbehavior reports + totalPenalty := float64(0) + for _, report := range reports { + totalPenalty += report.Penalty() + m.HandleMisbehaviorReport(channel, report) + } + + require.Eventually(t, func() bool { + // check if the misbehavior report has been processed by verifying that the Adjust method was called on the cache + record, ok := cache.Get(originId) + if !ok { + return false + } + require.NotNil(t, record) + + if totalPenalty != record.Penalty { + // all the misbehavior reports should be processed by now, so the penalty should be equal to the total penalty + return false + } + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. + // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") +} + +// TestHandleMisbehaviorReport_MultiplePenaltyReportsForSinglePeer_Sequential tests the handling of multiple misbehavior reports for a single peer. +// Reports are coming in concurrently. +// The test ensures that each misbehavior report is handled correctly and the penalties are cumulatively applied to the peer in the cache. +func TestHandleMisbehaviorReport_MultiplePenaltyReportsForSinglePeer_Concurrently(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // creates a list of mock misbehavior reports with negative penalty values for a single peer + originId := unittest.IdentifierFixture() + reports := createRandomMisbehaviorReportsForOriginId(t, originId, 5) + + channel := channels.Channel("test-channel") + + wg := sync.WaitGroup{} + wg.Add(len(reports)) + // handle the misbehavior reports + totalPenalty := float64(0) + for _, report := range reports { + r := report // capture range variable + totalPenalty += report.Penalty() + go func() { + defer wg.Done() + + m.HandleMisbehaviorReport(channel, r) + }() + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") + + require.Eventually(t, func() bool { + // check if the misbehavior report has been processed by verifying that the Adjust method was called on the cache + record, ok := cache.Get(originId) + if !ok { + return false + } + require.NotNil(t, record) + + if totalPenalty != record.Penalty { + // all the misbehavior reports should be processed by now, so the penalty should be equal to the total penalty + return false + } + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. + // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") +} + +// TestHandleMisbehaviorReport_SinglePenaltyReportsForMultiplePeers_Sequentially tests the handling of single misbehavior reports for multiple peers. +// Reports are coming in sequentially. +// The test ensures that each misbehavior report is handled correctly and the penalties are applied to the corresponding peers in the cache. +func TestHandleMisbehaviorReport_SinglePenaltyReportsForMultiplePeers_Sequentially(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // creates a list of single misbehavior reports for multiple peers (10 peers) + numPeers := 10 + reports := createRandomMisbehaviorReports(t, numPeers) + + channel := channels.Channel("test-channel") + + // handle the misbehavior reports + for _, report := range reports { + m.HandleMisbehaviorReport(channel, report) + } + + // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache + require.Eventually(t, func() bool { + for _, report := range reports { + originID := report.OriginId() + record, ok := cache.Get(originID) + if !ok { + return false + } + require.NotNil(t, record) + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. + require.Equal(t, report.Penalty(), record.Penalty) + // with just reporting a single misbehavior report, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + } + + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") + +} + +// TestHandleMisbehaviorReport_SinglePenaltyReportsForMultiplePeers_Concurrently tests the handling of single misbehavior reports for multiple peers. +// Reports are coming in concurrently. +// The test ensures that each misbehavior report is handled correctly and the penalties are applied to the corresponding peers in the cache. +func TestHandleMisbehaviorReport_SinglePenaltyReportsForMultiplePeers_Concurrently(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // creates a list of single misbehavior reports for multiple peers (10 peers) + numPeers := 10 + reports := createRandomMisbehaviorReports(t, numPeers) + + channel := channels.Channel("test-channel") + + wg := sync.WaitGroup{} + wg.Add(len(reports)) + // handle the misbehavior reports + totalPenalty := float64(0) + for _, report := range reports { + r := report // capture range variable + totalPenalty += report.Penalty() + go func() { + defer wg.Done() + + m.HandleMisbehaviorReport(channel, r) + }() + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") + + // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache + require.Eventually(t, func() bool { + for _, report := range reports { + originID := report.OriginId() + record, ok := cache.Get(originID) + if !ok { + return false + } + require.NotNil(t, record) + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. + require.Equal(t, report.Penalty(), record.Penalty) + // with just reporting a single misbehavior report, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + } + + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") +} + +// TestHandleMisbehaviorReport_MultiplePenaltyReportsForMultiplePeers_Sequentially tests the handling of multiple misbehavior reports for multiple peers. +// Reports are coming in sequentially. +// The test ensures that each misbehavior report is handled correctly and the penalties are cumulatively applied to the corresponding peers in the cache. +func TestHandleMisbehaviorReport_MultiplePenaltyReportsForMultiplePeers_Sequentially(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // create a map of origin IDs to their respective misbehavior reports (10 peers, 5 reports each) + numPeers := 10 + numReportsPerPeer := 5 + peersReports := make(map[flow.Identifier][]network.MisbehaviorReport) + + for i := 0; i < numPeers; i++ { + originID := unittest.IdentifierFixture() + reports := createRandomMisbehaviorReportsForOriginId(t, originID, numReportsPerPeer) + peersReports[originID] = reports + } + + channel := channels.Channel("test-channel") + + wg := sync.WaitGroup{} + // handle the misbehavior reports + for _, reports := range peersReports { + wg.Add(len(reports)) + for _, report := range reports { + r := report // capture range variable + go func() { + defer wg.Done() + + m.HandleMisbehaviorReport(channel, r) + }() + } + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") + + // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache + require.Eventually(t, func() bool { + for originID, reports := range peersReports { + totalPenalty := float64(0) + for _, report := range reports { + totalPenalty += report.Penalty() + } + + record, ok := cache.Get(originID) + if !ok { + return false + } + require.NotNil(t, record) + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. + require.Equal(t, totalPenalty, record.Penalty) + // with just reporting a single misbehavior report, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + } + + return true + }, 2*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") +} + +// TestHandleMisbehaviorReport_MultiplePenaltyReportsForMultiplePeers_Sequentially tests the handling of multiple misbehavior reports for multiple peers. +// Reports are coming in concurrently. +// The test ensures that each misbehavior report is handled correctly and the penalties are cumulatively applied to the corresponding peers in the cache. +func TestHandleMisbehaviorReport_MultiplePenaltyReportsForMultiplePeers_Concurrently(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // create a map of origin IDs to their respective misbehavior reports (10 peers, 5 reports each) + numPeers := 10 + numReportsPerPeer := 5 + peersReports := make(map[flow.Identifier][]network.MisbehaviorReport) + + for i := 0; i < numPeers; i++ { + originID := unittest.IdentifierFixture() + reports := createRandomMisbehaviorReportsForOriginId(t, originID, numReportsPerPeer) + peersReports[originID] = reports + } + + channel := channels.Channel("test-channel") + + // handle the misbehavior reports + for _, reports := range peersReports { + for _, report := range reports { + m.HandleMisbehaviorReport(channel, report) + } + } + + // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache + require.Eventually(t, func() bool { + for originID, reports := range peersReports { + totalPenalty := float64(0) + for _, report := range reports { + totalPenalty += report.Penalty() + } + + record, ok := cache.Get(originID) + if !ok { + return false + } + require.NotNil(t, record) + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. + require.Equal(t, totalPenalty, record.Penalty) + // with just reporting a single misbehavior report, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + } + + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") +} + +// TestHandleMisbehaviorReport_DuplicateReportsForSinglePeer_Concurrently tests the handling of duplicate misbehavior reports for a single peer. +// Reports are coming in concurrently. +// The test ensures that each misbehavior report is handled correctly and the penalties are cumulatively applied to the peer in the cache, in +// other words, the duplicate reports are not ignored. This is important because the misbehavior reports are assumed each uniquely reporting +// a different misbehavior even though they are coming with the same description. This is similar to the traffic tickets, where each ticket +// is uniquely identifying a traffic violation, even though the description of the violation is the same. +func TestHandleMisbehaviorReport_DuplicateReportsForSinglePeer_Concurrently(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // creates a single misbehavior report + originId := unittest.IdentifierFixture() + report := misbehaviorReportFixture(t, originId) + + channel := channels.Channel("test-channel") + + times := 100 // number of times the duplicate misbehavior report is reported concurrently + wg := sync.WaitGroup{} + wg.Add(times) + + // concurrently reports the same misbehavior report twice + for i := 0; i < times; i++ { + go func() { + defer wg.Done() + + m.HandleMisbehaviorReport(channel, report) + }() + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") + + require.Eventually(t, func() bool { + // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache + record, ok := cache.Get(originId) + if !ok { + return false + } + require.NotNil(t, record) + + // eventually, the penalty should be the accumulated penalty of all the duplicate misbehavior reports. + if record.Penalty != report.Penalty()*float64(times) { + return false + } + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. + // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") +} + +// TestDecayMisbehaviorPenalty_SingleHeartbeat tests the decay of the misbehavior penalty. The test ensures that the misbehavior penalty +// is decayed after a single heartbeat. The test guarantees waiting for at least one heartbeat by waiting for the first decay to happen. +func TestDecayMisbehaviorPenalty_SingleHeartbeat(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // creates a single misbehavior report + originId := unittest.IdentifierFixture() + report := misbehaviorReportFixtureWithDefaultPenalty(t, originId) + require.Less(t, report.Penalty(), float64(0)) // ensure the penalty is negative + + channel := channels.Channel("test-channel") + + // number of times the duplicate misbehavior report is reported concurrently + times := 10 + wg := sync.WaitGroup{} + wg.Add(times) + + // concurrently reports the same misbehavior report twice + for i := 0; i < times; i++ { + go func() { + defer wg.Done() + + m.HandleMisbehaviorReport(channel, report) + }() + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") + + // phase-1: eventually all the misbehavior reports should be processed. + penaltyBeforeDecay := float64(0) + require.Eventually(t, func() bool { + // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache + record, ok := cache.Get(originId) + if !ok { + return false + } + require.NotNil(t, record) + + // eventually, the penalty should be the accumulated penalty of all the duplicate misbehavior reports. + if record.Penalty != report.Penalty()*float64(times) { + return false + } + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. + // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + + penaltyBeforeDecay = record.Penalty + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") + + // phase-2: wait enough for at least one heartbeat to be processed. + time.Sleep(1 * time.Second) + + // phase-3: check if the penalty was decayed for at least one heartbeat. + record, ok := cache.Get(originId) + require.True(t, ok) // the record should be in the cache + require.NotNil(t, record) + + // with at least a single heartbeat, the penalty should be greater than the penalty before the decay. + require.Greater(t, record.Penalty, penaltyBeforeDecay) + // we waited for at most one heartbeat, so the decayed penalty should be still less than the value after 2 heartbeats. + require.Less(t, record.Penalty, penaltyBeforeDecay+2*record.Decay) + // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) +} + +// TestDecayMisbehaviorPenalty_MultipleHeartbeat tests the decay of the misbehavior penalty under multiple heartbeats. +// The test ensures that the misbehavior penalty is decayed with a linear progression within multiple heartbeats. +func TestDecayMisbehaviorPenalty_MultipleHeartbeats(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // creates a single misbehavior report + originId := unittest.IdentifierFixture() + report := misbehaviorReportFixtureWithDefaultPenalty(t, originId) + require.Less(t, report.Penalty(), float64(0)) // ensure the penalty is negative + + channel := channels.Channel("test-channel") + + // number of times the duplicate misbehavior report is reported concurrently + times := 10 + wg := sync.WaitGroup{} + wg.Add(times) + + // concurrently reports the same misbehavior report twice + for i := 0; i < times; i++ { + go func() { + defer wg.Done() + + m.HandleMisbehaviorReport(channel, report) + }() + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") + + // phase-1: eventually all the misbehavior reports should be processed. + penaltyBeforeDecay := float64(0) + require.Eventually(t, func() bool { + // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache + record, ok := cache.Get(originId) + if !ok { + return false + } + require.NotNil(t, record) + + // eventually, the penalty should be the accumulated penalty of all the duplicate misbehavior reports. + if record.Penalty != report.Penalty()*float64(times) { + return false + } + // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + + penaltyBeforeDecay = record.Penalty + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") + + // phase-2: wait for 3 heartbeats to be processed. + time.Sleep(3 * time.Second) + + // phase-3: check if the penalty was decayed in a linear progression. + record, ok := cache.Get(originId) + require.True(t, ok) // the record should be in the cache + require.NotNil(t, record) + + // with 3 heartbeats processed, the penalty should be greater than the penalty before the decay. + require.Greater(t, record.Penalty, penaltyBeforeDecay) + // with 3 heartbeats processed, the decayed penalty should be less than the value after 4 heartbeats. + require.Less(t, record.Penalty, penaltyBeforeDecay+4*record.Decay) + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. + // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) +} + +// TestDecayMisbehaviorPenalty_MultipleHeartbeat tests the decay of the misbehavior penalty under multiple heartbeats. +// The test ensures that the misbehavior penalty is decayed with a linear progression within multiple heartbeats. +func TestDecayMisbehaviorPenalty_DecayToZero(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // creates a single misbehavior report + originId := unittest.IdentifierFixture() + report := misbehaviorReportFixture(t, originId) // penalties are between -1 and -10 + require.Less(t, report.Penalty(), float64(0)) // ensure the penalty is negative + + channel := channels.Channel("test-channel") + + // number of times the duplicate misbehavior report is reported concurrently + times := 10 + wg := sync.WaitGroup{} + wg.Add(times) + + // concurrently reports the same misbehavior report twice + for i := 0; i < times; i++ { + go func() { + defer wg.Done() + + m.HandleMisbehaviorReport(channel, report) + }() + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") + + // phase-1: eventually all the misbehavior reports should be processed. + require.Eventually(t, func() bool { + // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache + record, ok := cache.Get(originId) + if !ok { + return false + } + require.NotNil(t, record) + + // eventually, the penalty should be the accumulated penalty of all the duplicate misbehavior reports. + if record.Penalty != report.Penalty()*float64(times) { + return false + } + // with just reporting a few misbehavior reports, the cutoff counter should not be incremented. + require.Equal(t, uint64(0), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") + + // phase-2: default decay speed is 1000 and with 10 penalties in range of [-1, -10], the penalty should be decayed to zero in + // a single heartbeat. + time.Sleep(1 * time.Second) + + // phase-3: check if the penalty was decayed to zero. + record, ok := cache.Get(originId) + require.True(t, ok) // the record should be in the cache + require.NotNil(t, record) + + require.False(t, record.DisallowListed) // the peer should not be disallow listed yet. + // with a single heartbeat and decay speed of 1000, the penalty should be decayed to zero. + require.Equal(t, float64(0), record.Penalty) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) +} + +// TestDecayMisbehaviorPenalty_DecayToZero_AllowListing tests that when the misbehavior penalty of an already disallow-listed +// peer is decayed to zero, the peer is allow-listed back in the network, and its spam record cache is updated accordingly. +func TestDecayMisbehaviorPenalty_DecayToZero_AllowListing(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // simulates a disallow-listed peer in cache. + originId := unittest.IdentifierFixture() + penalty, err := cache.Adjust(originId, func(record model.ProtocolSpamRecord) (model.ProtocolSpamRecord, error) { + record.Penalty = -10 // set the penalty to -10 to simulate that the penalty has already been decayed for a while. + record.CutoffCounter = 1 + record.DisallowListed = true + record.OriginId = originId + record.Decay = model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay + return record, nil + }) + require.NoError(t, err) + require.Equal(t, float64(-10), penalty) + + // sanity check + record, ok := cache.Get(originId) + require.True(t, ok) // the record should be in the cache + require.NotNil(t, record) + require.Equal(t, float64(-10), record.Penalty) + require.True(t, record.DisallowListed) + require.Equal(t, uint64(1), record.CutoffCounter) + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + + // eventually, we expect the ALSP manager to emit an allow list notification to the network layer when the penalty is decayed to zero. + consumer.On("OnAllowListNotification", &network.AllowListingUpdate{ + FlowIds: flow.IdentifierList{originId}, + Cause: network.DisallowListedCauseAlsp, + }).Return(nil).Once() + + // wait for at most two heartbeats; default decay speed is 1000 and with a penalty of -10, the penalty should be decayed to zero in a single heartbeat. + require.Eventually(t, func() bool { + record, ok = cache.Get(originId) + if !ok { + return false + } + if record.DisallowListed { + return false // the peer should not be allow-listed yet. + } + if record.Penalty != float64(0) { + return false // the penalty should be decayed to zero. + } + if record.CutoffCounter != 1 { + return false // the cutoff counter should be incremented. + } + if record.Decay != model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay { + return false // the decay should be the default decay value. + } + + return true + + }, 2*time.Second, 10*time.Millisecond, "penalty was not decayed to zero") + +} + +// TestDisallowListNotification tests the emission of the allow list notification to the network layer when the misbehavior +// penalty of a node is dropped below the disallow-listing threshold. The test ensures that the disallow list notification is +// emitted to the network layer when the misbehavior penalty is dropped below the disallow-listing threshold and that the +// cutoff counter of the spam record for the misbehaving node is incremented indicating that the node is disallow-listed once. +func TestDisallowListNotification(t *testing.T) { + cfg := managerCfgFixture(t) + consumer := mocknetwork.NewDisallowListNotificationConsumer(t) + + var cache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + cache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return cache + }), + } + m, err := alspmgr.NewMisbehaviorReportManager(cfg, consumer) + require.NoError(t, err) + + // start the ALSP manager + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + unittest.RequireCloseBefore(t, m.Done(), 100*time.Millisecond, "ALSP manager did not stop") + }() + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + m.Start(signalerCtx) + unittest.RequireCloseBefore(t, m.Ready(), 100*time.Millisecond, "ALSP manager did not start") + + // creates a single misbehavior report + originId := unittest.IdentifierFixture() + report := misbehaviorReportFixtureWithDefaultPenalty(t, originId) + require.Less(t, report.Penalty(), float64(0)) // ensure the penalty is negative + + channel := channels.Channel("test-channel") + + // reporting the same misbehavior 120 times, should result in a single disallow list notification, since each + // misbehavior report is reported with the same penalty 0.01 * diallowlisting-threshold. We go over the threshold + // to ensure that the disallow list notification is emitted only once. + times := 120 + wg := sync.WaitGroup{} + wg.Add(times) + + // concurrently reports the same misbehavior report twice + for i := 0; i < times; i++ { + go func() { + defer wg.Done() + + m.HandleMisbehaviorReport(channel, report) + }() + } + + // at this point, we expect a single disallow list notification to be emitted to the network layer when all the misbehavior + // reports are processed by the ALSP manager (the notification is emitted when at the next heartbeat). + consumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: flow.IdentifierList{report.OriginId()}, + Cause: network.DisallowListedCauseAlsp, + }).Return().Once() + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "not all misbehavior reports have been processed") + + require.Eventually(t, func() bool { + // check if the misbehavior reports have been processed by verifying that the Adjust method was called on the cache + record, ok := cache.Get(originId) + if !ok { + return false + } + require.NotNil(t, record) + + // eventually, the penalty should be the accumulated penalty of all the duplicate misbehavior reports (with the default decay). + // the decay is added to the penalty as we allow for a single heartbeat before the disallow list notification is emitted. + if record.Penalty != report.Penalty()*float64(times)+record.Decay { + return false + } + require.True(t, record.DisallowListed) // the peer should be disallow-listed. + // cuttoff counter should be incremented since the penalty is above the disallowlisting threshold. + require.Equal(t, uint64(1), record.CutoffCounter) + // the decay should be the default decay value. + require.Equal(t, model.SpamRecordFactory()(unittest.IdentifierFixture()).Decay, record.Decay) + + return true + }, 1*time.Second, 10*time.Millisecond, "ALSP manager did not handle the misbehavior report") +} + +// misbehaviorReportFixture creates a mock misbehavior report for a single origin id. +// Args: +// - t: the testing.T instance +// - originID: the origin id of the misbehavior report +// Returns: +// - network.MisbehaviorReport: the misbehavior report +// Note: the penalty of the misbehavior report is randomly chosen between -1 and -10. +func misbehaviorReportFixture(t *testing.T, originID flow.Identifier) network.MisbehaviorReport { + return misbehaviorReportFixtureWithPenalty(t, originID, math.Min(-1, float64(-1-rand.Intn(10)))) +} + +func misbehaviorReportFixtureWithDefaultPenalty(t *testing.T, originID flow.Identifier) network.MisbehaviorReport { + return misbehaviorReportFixtureWithPenalty(t, originID, model.DefaultPenaltyValue) +} + +func misbehaviorReportFixtureWithPenalty(t *testing.T, originID flow.Identifier, penalty float64) network.MisbehaviorReport { + report := mocknetwork.NewMisbehaviorReport(t) + report.On("OriginId").Return(originID) + report.On("Reason").Return(alsp.AllMisbehaviorTypes()[rand.Intn(len(alsp.AllMisbehaviorTypes()))]) + report.On("Penalty").Return(penalty) + + return report +} + +// createRandomMisbehaviorReportsForOriginId creates a slice of random misbehavior reports for a single origin id. +// Args: +// - t: the testing.T instance +// - originID: the origin id of the misbehavior reports +// - numReports: the number of misbehavior reports to create +// Returns: +// - []network.MisbehaviorReport: the slice of misbehavior reports +// Note: the penalty of the misbehavior reports is randomly chosen between -1 and -10. +func createRandomMisbehaviorReportsForOriginId(t *testing.T, originID flow.Identifier, numReports int) []network.MisbehaviorReport { + reports := make([]network.MisbehaviorReport, numReports) + + for i := 0; i < numReports; i++ { + reports[i] = misbehaviorReportFixture(t, originID) + } + + return reports +} + +// createRandomMisbehaviorReports creates a slice of random misbehavior reports. +// Args: +// - t: the testing.T instance +// - numReports: the number of misbehavior reports to create +// Returns: +// - []network.MisbehaviorReport: the slice of misbehavior reports +// Note: the penalty of the misbehavior reports is randomly chosen between -1 and -10. +func createRandomMisbehaviorReports(t *testing.T, numReports int) []network.MisbehaviorReport { + reports := make([]network.MisbehaviorReport, numReports) + + for i := 0; i < numReports; i++ { + reports[i] = misbehaviorReportFixture(t, unittest.IdentifierFixture()) + } + + return reports +} + +// managerCfgFixture creates a new MisbehaviorReportManagerConfig with default values for testing. +func managerCfgFixture(t *testing.T) *alspmgr.MisbehaviorReportManagerConfig { + c, err := config.DefaultConfig() + require.NoError(t, err) + return &alspmgr.MisbehaviorReportManagerConfig{ + Logger: unittest.Logger(), + SpamRecordCacheSize: c.NetworkConfig.AlspConfig.SpamRecordCacheSize, + SpamReportQueueSize: c.NetworkConfig.AlspConfig.SpamReportQueueSize, + HeartBeatInterval: c.NetworkConfig.AlspConfig.HearBeatInterval, + AlspMetrics: metrics.NewNoopCollector(), + HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), + } +} diff --git a/network/alsp/misbehavior.go b/network/alsp/misbehavior.go new file mode 100644 index 00000000000..af4921cd06a --- /dev/null +++ b/network/alsp/misbehavior.go @@ -0,0 +1,62 @@ +package alsp + +import "github.com/onflow/flow-go/network" + +const ( + // StaleMessage is a misbehavior that is reported when an engine receives a message that is deemed stale based on the + // local view of the engine. The decision to consider a message stale is up to the engine. + StaleMessage network.Misbehavior = "misbehavior-stale-message" + + // ResourceIntensiveRequest is a misbehavior that is reported when an engine receives a request that takes an unreasonable amount + // of resources by the engine to process, e.g., a request for a large number of blocks. The decision to consider a + // request heavy is up to the engine. + ResourceIntensiveRequest network.Misbehavior = "misbehavior-resource-intensive-request" + + // RedundantMessage is a misbehavior that is reported when an engine receives a message that is redundant, i.e., the + // message is already known to the engine. The decision to consider a message redundant is up to the engine. + RedundantMessage network.Misbehavior = "misbehavior-redundant-message" + + // UnsolicitedMessage is a misbehavior that is reported when an engine receives a message that is not solicited by the + // engine. The decision to consider a message unsolicited is up to the engine. + UnsolicitedMessage network.Misbehavior = "misbehavior-unsolicited-message" + + // InvalidMessage is a misbehavior that is reported when an engine receives a message that is invalid, i.e., + // the message is not valid according to the engine's validation logic. The decision to consider a message invalid + // is up to the engine. + InvalidMessage network.Misbehavior = "misbehavior-invalid-message" + + // UnExpectedValidationError is a misbehavior that is reported when a validation error is encountered during message validation before the message + // is processed by an engine. + UnExpectedValidationError network.Misbehavior = "unexpected-validation-error" + + // UnknownMsgType is a misbehavior that is reported when a message of unknown type is received from a peer. + UnknownMsgType network.Misbehavior = "unknown-message-type" + + // SenderEjected is a misbehavior that is reported when a message is received from an ejected peer. + SenderEjected network.Misbehavior = "sender-ejected" + + // UnauthorizedUnicastOnChannel is a misbehavior that is reported when a message not authorized to be sent via unicast is received via unicast. + UnauthorizedUnicastOnChannel network.Misbehavior = "unauthorized-unicast-on-channel" + + // UnAuthorizedSender is a misbehavior that is reported when a message is sent by an unauthorized role. + UnAuthorizedSender network.Misbehavior = "unauthorized-sender" + + // UnauthorizedPublishOnChannel is a misbehavior that is reported when a message not authorized to be sent via pubsub is received via pubsub. + UnauthorizedPublishOnChannel network.Misbehavior = "unauthorized-pubsub-on-channel" +) + +func AllMisbehaviorTypes() []network.Misbehavior { + return []network.Misbehavior{ + StaleMessage, + ResourceIntensiveRequest, + RedundantMessage, + UnsolicitedMessage, + InvalidMessage, + UnExpectedValidationError, + UnknownMsgType, + SenderEjected, + UnauthorizedUnicastOnChannel, + UnauthorizedPublishOnChannel, + UnAuthorizedSender, + } +} diff --git a/network/alsp/mock/misbehavior_report_opt.go b/network/alsp/mock/misbehavior_report_opt.go new file mode 100644 index 00000000000..e3fe6b57941 --- /dev/null +++ b/network/alsp/mock/misbehavior_report_opt.go @@ -0,0 +1,42 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockalsp + +import ( + alsp "github.com/onflow/flow-go/network/alsp" + mock "github.com/stretchr/testify/mock" +) + +// MisbehaviorReportOpt is an autogenerated mock type for the MisbehaviorReportOpt type +type MisbehaviorReportOpt struct { + mock.Mock +} + +// Execute provides a mock function with given fields: r +func (_m *MisbehaviorReportOpt) Execute(r *alsp.MisbehaviorReport) error { + ret := _m.Called(r) + + var r0 error + if rf, ok := ret.Get(0).(func(*alsp.MisbehaviorReport) error); ok { + r0 = rf(r) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewMisbehaviorReportOpt interface { + mock.TestingT + Cleanup(func()) +} + +// NewMisbehaviorReportOpt creates a new instance of MisbehaviorReportOpt. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMisbehaviorReportOpt(t mockConstructorTestingTNewMisbehaviorReportOpt) *MisbehaviorReportOpt { + mock := &MisbehaviorReportOpt{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/alsp/mock/spam_record_cache.go b/network/alsp/mock/spam_record_cache.go new file mode 100644 index 00000000000..ecc9f4ae1a5 --- /dev/null +++ b/network/alsp/mock/spam_record_cache.go @@ -0,0 +1,124 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockalsp + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/network/alsp/model" +) + +// SpamRecordCache is an autogenerated mock type for the SpamRecordCache type +type SpamRecordCache struct { + mock.Mock +} + +// Adjust provides a mock function with given fields: originId, adjustFunc +func (_m *SpamRecordCache) Adjust(originId flow.Identifier, adjustFunc model.RecordAdjustFunc) (float64, error) { + ret := _m.Called(originId, adjustFunc) + + var r0 float64 + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier, model.RecordAdjustFunc) (float64, error)); ok { + return rf(originId, adjustFunc) + } + if rf, ok := ret.Get(0).(func(flow.Identifier, model.RecordAdjustFunc) float64); ok { + r0 = rf(originId, adjustFunc) + } else { + r0 = ret.Get(0).(float64) + } + + if rf, ok := ret.Get(1).(func(flow.Identifier, model.RecordAdjustFunc) error); ok { + r1 = rf(originId, adjustFunc) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Get provides a mock function with given fields: originId +func (_m *SpamRecordCache) Get(originId flow.Identifier) (*model.ProtocolSpamRecord, bool) { + ret := _m.Called(originId) + + var r0 *model.ProtocolSpamRecord + var r1 bool + if rf, ok := ret.Get(0).(func(flow.Identifier) (*model.ProtocolSpamRecord, bool)); ok { + return rf(originId) + } + if rf, ok := ret.Get(0).(func(flow.Identifier) *model.ProtocolSpamRecord); ok { + r0 = rf(originId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.ProtocolSpamRecord) + } + } + + if rf, ok := ret.Get(1).(func(flow.Identifier) bool); ok { + r1 = rf(originId) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// Identities provides a mock function with given fields: +func (_m *SpamRecordCache) Identities() []flow.Identifier { + ret := _m.Called() + + var r0 []flow.Identifier + if rf, ok := ret.Get(0).(func() []flow.Identifier); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]flow.Identifier) + } + } + + return r0 +} + +// Remove provides a mock function with given fields: originId +func (_m *SpamRecordCache) Remove(originId flow.Identifier) bool { + ret := _m.Called(originId) + + var r0 bool + if rf, ok := ret.Get(0).(func(flow.Identifier) bool); ok { + r0 = rf(originId) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// Size provides a mock function with given fields: +func (_m *SpamRecordCache) Size() uint { + ret := _m.Called() + + var r0 uint + if rf, ok := ret.Get(0).(func() uint); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint) + } + + return r0 +} + +type mockConstructorTestingTNewSpamRecordCache interface { + mock.TestingT + Cleanup(func()) +} + +// NewSpamRecordCache creates a new instance of SpamRecordCache. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewSpamRecordCache(t mockConstructorTestingTNewSpamRecordCache) *SpamRecordCache { + mock := &SpamRecordCache{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/alsp/model/params.go b/network/alsp/model/params.go new file mode 100644 index 00000000000..04a53a8f0c8 --- /dev/null +++ b/network/alsp/model/params.go @@ -0,0 +1,47 @@ +package model + +// To give a summary with the default value: +// 1. The penalty of each misbehavior is 0.01 * DisallowListingThreshold = -864 +// 2. The penalty of each misbehavior is decayed by a decay value at each decay interval. The default decay value is 1000. +// This means that by default if a node misbehaves 100 times in a second, it gets disallow-listed, and takes 86.4 seconds to recover. +// We emphasize on the default penalty value can be amplified by the engine that reports the misbehavior. +// 3. Each time a node is disallow-listed, its decay speed is decreased by 90%. This means that if a node is disallow-listed +// for the first time, it takes 86.4 seconds to recover. If the node is disallow-listed for the second time, its decay +// speed is decreased by 90% from 1000 to 100, and it takes around 15 minutes to recover. If the node is disallow-listed +// for the third time, its decay speed is decreased by 90% from 100 to 10, and it takes around 2.5 hours to recover. +// If the node is disallow-listed for the fourth time, its decay speed is decreased by 90% from 10 to 1, and it takes +// around a day to recover. From this point on, the decay speed is 1, and it takes around a day to recover from each +// disallow-listing. +const ( + // DisallowListingThreshold is the threshold for concluding a node behavior is malicious and disallow-listing the node. + // If the overall penalty of this node drops below this threshold, the node is reported to be disallow-listed by + // the networking layer, i.e., existing connections to the node are closed and the node is no longer allowed to connect till + // its penalty is decayed back to zero. + // maximum block-list period is 1 day + DisallowListingThreshold = -24 * 60 * 60 // (Don't change this value) + + // DefaultPenaltyValue is the default penalty value for misbehaving nodes. + // By default, each reported infringement will be penalized by this value. However, the penalty can be amplified + // by the engine that reports the misbehavior. The penalty system is designed in a way that more than 100 misbehavior/sec + // at the default penalty value will result in disallow-listing the node. By amplifying the penalty, the engine can + // decrease the number of misbehavior/sec that will result in disallow-listing the node. For example, if the engine + // amplifies the penalty by 10, the number of misbehavior/sec that will result in disallow-listing the node will be + // 10 times less than the default penalty value and the node will be disallow-listed after 10 times more misbehavior/sec. + DefaultPenaltyValue = 0.01 * DisallowListingThreshold // (Don't change this value) + + // InitialDecaySpeed is the initial decay speed of the penalty of a misbehaving node. + // The decay speed is applied on an arithmetic progression. The penalty value of the node is the first term of the + // progression and the decay speed is the common difference of the progression, i.e., p(n) = p(0) + n * d, where + // p(n) is the penalty value of the node after n decay intervals, p(0) is the initial penalty value of the node, and + // d is the decay speed. Decay intervals are set to 1 second (protocol invariant). Hence, with the initial decay speed + // of 1000, the penalty value of the node will be decreased by 1000 every second. This means that if a node misbehaves + // 100 times in a second, it gets disallow-listed, and takes 86.4 seconds to recover. + // In mature implementation of the protocol, the decay speed of a node is decreased by 90% each time the node is + // disallow-listed. This means that if a node is disallow-listed for the first time, it takes 86.4 seconds to recover. + // If the node is disallow-listed for the second time, its decay speed is decreased by 90% from 1000 to 100, and it + // takes around 15 minutes to recover. If the node is disallow-listed for the third time, its decay speed is decreased + // by 90% from 100 to 10, and it takes around 2.5 hours to recover. If the node is disallow-listed for the fourth time, + // its decay speed is decreased by 90% from 10 to 1, and it takes around a day to recover. From this point on, the decay + // speed is 1, and it takes around a day to recover from each disallow-listing. + InitialDecaySpeed = 1000 // (Don't change this value) +) diff --git a/network/alsp/model/record.go b/network/alsp/model/record.go new file mode 100644 index 00000000000..088e7d81916 --- /dev/null +++ b/network/alsp/model/record.go @@ -0,0 +1,59 @@ +package model + +import ( + "github.com/onflow/flow-go/model/flow" +) + +// ProtocolSpamRecord is a record of a misbehaving node. It is used to keep track of the Penalty value of the node +// and the number of times it has been slashed due to its Penalty value dropping below the disallow-listing threshold. +type ProtocolSpamRecord struct { + // OriginId is the node id of the misbehaving node. It is assumed an authorized (i.e., staked) node at the + // time of the misbehavior report creation (otherwise, the networking layer should not have dispatched the + // message to the Flow protocol layer in the first place). + OriginId flow.Identifier + + // Decay speed of Penalty for this misbehaving node. Each node may have a different Decay speed based on its behavior. + Decay float64 + + // CutoffCounter is a counter that is used to determine how many times the connections to the node has been cut due to + // its Penalty value dropping below the disallow-listing threshold. + // Note that the cutoff connections are recovered after a certain amount of time. + CutoffCounter uint64 + + // DisallowListed indicates whether the node is currently disallow-listed or not. When a node is in the disallow-list, + // the existing connections to the node are cut and no new connections are allowed to be established, neither incoming + // nor outgoing. + DisallowListed bool + + // total Penalty value of the misbehaving node. Should be a negative value. + Penalty float64 +} + +// RecordAdjustFunc is a function that is used to adjust the fields of a ProtocolSpamRecord. +// The function is called with the current record and should return the adjusted record. +// Returned error indicates that the adjustment is not applied, and the record should not be updated. +// In BFT setup, the returned error should be treated as a fatal error. +type RecordAdjustFunc func(ProtocolSpamRecord) (ProtocolSpamRecord, error) + +// SpamRecordFactoryFunc is a function that creates a new protocol spam record with the given origin id and initial values. +// Args: +// - originId: the origin id of the spam record. +// Returns: +// - ProtocolSpamRecord, the created record. +type SpamRecordFactoryFunc func(flow.Identifier) ProtocolSpamRecord + +// SpamRecordFactory returns the default factory function for creating a new protocol spam record. +// Returns: +// - SpamRecordFactoryFunc, the default factory function. +// Note that the default factory function creates a new record with the initial values. +func SpamRecordFactory() SpamRecordFactoryFunc { + return func(originId flow.Identifier) ProtocolSpamRecord { + return ProtocolSpamRecord{ + OriginId: originId, + Decay: InitialDecaySpeed, + DisallowListed: false, + CutoffCounter: uint64(0), + Penalty: float64(0), + } + } +} diff --git a/network/alsp/readme.md b/network/alsp/readme.md new file mode 100644 index 00000000000..0267f58c91f --- /dev/null +++ b/network/alsp/readme.md @@ -0,0 +1,74 @@ +# Application Layer Spam Prevention (ALSP) +## Overview +The Application Layer Spam Prevention (ALSP) is a module that provides a mechanism to prevent the malicious nodes from +spamming the Flow nodes at the application layer (i.e., the engines). ALSP is not a multi-party protocol, i.e., +it does not require the nodes to exchange any messages with each other for the purpose of spam prevention. Rather, it is +a local mechanism that is implemented by each node to protect itself from malicious nodes. ALSP is not meant to replace +the existing spam prevention mechanisms at the network layer (e.g., the Libp2p and GossipSub). +Rather, it is meant to complement the existing mechanisms by providing an additional layer of protection. +ALSP is concerned with the spamming of the application layer through messages that appear valid to the networking layer and hence +are not filtered out by the existing mechanisms. + +ALSP relies on the application layer to detect and report the misbehaviors that +lead to spamming. It enforces a penalty system to penalize the misbehaving nodes that are reported by the application layer. ALSP also takes +extra measures to protect the network from malicious nodes that attempt an active spamming attack. Once the penalty of a remote node +reaches a certain threshold, the local node will disconnect from the remote node and no-longer accept any incoming connections from the remote node +until the penalty is reduced to zero again through a decaying interval. + +## Features +- Spam prevention at the application layer. +- Penalizes misbehaving nodes based on their behavior. +- Configurable penalty values and decay intervals. +- Misbehavior reports with customizable penalty amplification. +- Thread-safe and non-blocking implementation. +- Maintains the safety and liveness of the Flow blockchain system by disallow-listing malicious nodes (i.e., application layer spammers). + +## Architectural Principles +- **Non-intrusive**: ALSP is a local mechanism that is implemented by each node to protect itself from malicious nodes. It is not a multi-party protocol, i.e., it does not require the nodes to exchange any messages with each other for the purpose of spam prevention. +- **Non-blocking**: ALSP is non-blocking and does not affect the performance of the networking layer. It is implemented in a way that does not require the networking layer to wait for the ALSP to complete its operations. Non-blocking behavior is mandatory for the networking layer to maintain its performance. +- **Thread-safe**: ALSP is thread-safe and can be used concurrently by multiple threads, e.g., concurrent engine calls on reporting misbehaviors. + +## Usage +ALSP is enabled by default through the networking layer. It is not necessary to explicitly enable it. One can disable it by setting the `alsp-enable` flag to `false`. +The network.Conduit interface provides the following method to report misbehaviors: +- `ReportMisbehavior(*MisbehaviorReport)`: Reports a misbehavior to the ALSP. The misbehavior report contains the misbehavior type and the penalty value. The penalty value is used to increase the penalty of the remote node. The penalty value is amplified by the penalty amplification factor before being applied to the remote node. + +By default, the penalty amplification factor is set to 0.01 * disallow-listing threshold. The disallow-listing threshold is the penalty threshold at which the local node will disconnect from the remote node and no-longer accept any incoming connections from the remote node until the penalty is reduced to zero again through a decaying interval. +Hence, by default, every time a misbehavior is reported, the penalty of the remote node is increased by 0.01 * disallow-listing threshold. This penalty value is configurable through an option function on the `MisbehaviorReport` struct. +The example below shows how to create a misbehavior report with a penalty amplification factor of 10, i.e., the penalty value of the misbehavior report is amplified by 10 before being applied to the remote node. This is equal to +increasing the penalty of the remote node by 10 * 0.01 * disallow-listing threshold. The `misbehavingId` is the Flow identifier of the remote node that is misbehaving. The `misbehaviorType` is the reason for reporting the misbehavior. +```go +report, err := NewMisbehaviorReport(misbehavingId, misbehaviorType, WithPenaltyAmplification(10)) +if err != nil { + // handle the error +} +``` + +Once a misbehavior report is created, it can be reported to the ALSP by calling the `ReportMisbehavior` method on the network conduit. The example below shows how to report a misbehavior to the ALSP. +```go +// let con be network.Conduit +err := con.ReportMisbehavior(report) +if err != nil { + // handle the error +} +``` + +## Misbehavior Types (`MisbehaviorType`) +ALSP package defines several constants that represent various types of misbehaviors that can be reported by engines. These misbehavior types help categorize node behavior and improve the accuracy of the penalty system. + +### Constants +The following constants represent misbehavior types that can be reported: + +- `StaleMessage`: This misbehavior is reported when an engine receives a message that is deemed stale based on the local view of the engine. A stale message is one that is outdated, irrelevant, or already processed by the engine. +- `ResourceIntensiveRequest`: This misbehavior is reported when an engine receives a request that takes an unreasonable amount of resources for the engine to process, e.g., a request for a large number of blocks. The decision to consider a request heavy is up to the engine. Heavy requests can potentially slow down the engine, causing performance issues. +- `RedundantMessage`: This misbehavior is reported when an engine receives a message that is redundant, i.e., the message is already known to the engine. The decision to consider a message redundant is up to the engine. Redundant messages can increase network traffic and waste processing resources. +- `UnsolicitedMessage`: This misbehavior is reported when an engine receives a message that is not solicited by the engine. The decision to consider a message unsolicited is up to the engine. Unsolicited messages can be a sign of spamming or malicious behavior. +- `InvalidMessage`: This misbehavior is reported when an engine receives a message that is invalid and fails the validation logic as specified by the engine, i.e., the message is malformed or does not follow the protocol specification. The decision to consider a message invalid is up to the engine. Invalid messages can be a sign of spamming or malicious behavior. +## Thresholds and Parameters +The ALSP provides various constants and options to customize the penalty system: +- `misbehaviorDisallowListingThreshold`: The threshold for concluding a node behavior is malicious and disallow-listing the node. Once the penalty of a remote node reaches this threshold, the local node will disconnect from the remote node and no-longer accept any incoming connections from the remote node until the penalty is reduced to zero again through a decaying interval. +- `defaultPenaltyValue`: The default penalty value for misbehaving nodes. This value is used when the penalty value is not specified in the misbehavior report. By default, the penalty value is set to `0.01 * misbehaviorDisallowListingThreshold`. However, this value can be amplified by a positive integer in [1-100] using the `WithPenaltyAmplification` option function on the `MisbehaviorReport` struct. Note that amplifying at 100 means that a single misbehavior report will disallow-list the remote node. +- `misbehaviorDecayHeartbeatInterval`: The interval at which the penalty of the misbehaving nodes is decayed. Decaying is used to reduce the penalty of the misbehaving nodes over time. So that the penalty of the misbehaving nodes is reduced to zero after a certain period of time and the node is no-longer considered misbehaving. This is to avoid persisting the penalty of a node forever. +- `defaultDecayValue`: The default value that is deducted from the penalty of the misbehaving nodes at each decay interval. +- `decayValueSpeedPenalty`: The penalty for the decay speed. This is a multiplier that is applied to the `defaultDecayValue` at each decay interval. The purpose of this penalty is to slow down the decay process of the penalty of the nodes that make a habit of misbehaving. +- `minimumDecayValue`: The minimum decay value that is used to decay the penalty of the misbehaving nodes. The decay value is capped at this value. diff --git a/network/alsp/report.go b/network/alsp/report.go new file mode 100644 index 00000000000..8653b6c34f4 --- /dev/null +++ b/network/alsp/report.go @@ -0,0 +1,80 @@ +package alsp + +import ( + "fmt" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/alsp/model" +) + +// MisbehaviorReport is a report that is sent to the networking layer to penalize the misbehaving node. +// A MisbehaviorReport reports the misbehavior of a node on sending a message to the current node that appears valid +// based on the networking layer but is considered invalid by the current node based on the Flow protocol. +// +// A MisbehaviorReport consists of a reason and a penalty. The reason is a string that describes the misbehavior. +// The penalty is a value that is deducted from the overall score of the misbehaving node. The score is +// decayed at each decay interval. If the overall penalty of the misbehaving node drops below the disallow-listing +// threshold, the node is reported to be disallow-listed by the networking layer, i.e., existing connections to the +// node are closed and the node is no longer allowed to connect till its penalty is decayed back to zero. +type MisbehaviorReport struct { + id flow.Identifier // the ID of the misbehaving node + reason network.Misbehavior // the reason of the misbehavior + penalty float64 // the penalty value of the misbehavior +} + +var _ network.MisbehaviorReport = (*MisbehaviorReport)(nil) + +// MisbehaviorReportOpt is an option that can be used to configure a misbehavior report. +type MisbehaviorReportOpt func(r *MisbehaviorReport) error + +// WithPenaltyAmplification returns an option that can be used to amplify the penalty value. +// The penalty value is multiplied by the given value. The value should be between 1-100. +// If the value is not in the range, an error is returned. +// The returned error by this option indicates that the option is not applied. In BFT setup, the returned error +// should be treated as a fatal error. +func WithPenaltyAmplification(v float64) MisbehaviorReportOpt { + return func(r *MisbehaviorReport) error { + if v <= 0 || v > 100 { + return fmt.Errorf("penalty value should be between 1-100: %v", v) + } + r.penalty *= v + return nil + } +} + +// OriginId returns the ID of the misbehaving node. +func (r MisbehaviorReport) OriginId() flow.Identifier { + return r.id +} + +// Reason returns the reason of the misbehavior. +func (r MisbehaviorReport) Reason() network.Misbehavior { + return r.reason +} + +// Penalty returns the penalty value of the misbehavior. +func (r MisbehaviorReport) Penalty() float64 { + return r.penalty +} + +// NewMisbehaviorReport creates a new misbehavior report with the given reason and options. +// If no options are provided, the default penalty value is used. +// The returned error by this function indicates that the report is not created. In BFT setup, the returned error +// should be treated as a fatal error. +// The default penalty value is 0.01 * misbehaviorDisallowListingThreshold = -86.4 +func NewMisbehaviorReport(misbehavingId flow.Identifier, reason network.Misbehavior, opts ...MisbehaviorReportOpt) (*MisbehaviorReport, error) { + m := &MisbehaviorReport{ + id: misbehavingId, + reason: reason, + penalty: model.DefaultPenaltyValue, + } + + for _, opt := range opts { + if err := opt(m); err != nil { + return nil, fmt.Errorf("failed to apply misbehavior report option: %w", err) + } + } + + return m, nil +} diff --git a/network/cache/rcvcache.go b/network/cache/rcvcache.go index be685ae670d..bdab2ad894a 100644 --- a/network/cache/rcvcache.go +++ b/network/cache/rcvcache.go @@ -29,7 +29,8 @@ func (r receiveCacheEntry) Checksum() flow.Identifier { } // NewHeroReceiveCache returns a new HeroCache-based receive cache. -func NewHeroReceiveCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *ReceiveCache { +func NewHeroReceiveCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, +) *ReceiveCache { backData := herocache.NewCache(sizeLimit, herocache.DefaultOversizeFactor, heropool.LRUEjection, // receive cache must be LRU. diff --git a/network/channels/channels.go b/network/channels/channels.go index b9394b12c64..817e11c54db 100644 --- a/network/channels/channels.go +++ b/network/channels/channels.go @@ -277,13 +277,52 @@ func ChannelFromTopic(topic Topic) (Channel, bool) { return "", false } -// SporkIDFromTopic returns the spork ID from a topic. -// All errors returned from this function can be considered benign. -func SporkIDFromTopic(topic Topic) (flow.Identifier, error) { +// sporkIdFromTopic returns the pre-pended spork ID flow identifier for the topic. +// A valid channel has a spork ID suffix: +// +// channel/spork_id +// +// A generic error is returned if an error is encountered while converting the spork ID to flow Identifier or +// the spork ID is missing. +func sporkIdFromTopic(topic Topic) (flow.Identifier, error) { if index := strings.LastIndex(topic.String(), "/"); index != -1 { - return flow.HexStringToIdentifier(string(topic)[index+1:]) + id, err := flow.HexStringToIdentifier(string(topic)[index+1:]) + if err != nil { + return flow.Identifier{}, fmt.Errorf("failed to get spork ID from topic %s: %w", topic, err) + } + + return id, nil + } + return flow.Identifier{}, fmt.Errorf("spork id missing from topic") +} + +// sporkIdStrFromTopic returns the pre-pended spork ID string for the topic. +// A valid channel has a spork ID suffix: +// +// channel/spork_id +// +// A generic error is returned if an error is encountered while deriving the spork ID from the topic +func sporkIdStrFromTopic(topic Topic) (string, error) { + sporkId, err := sporkIdFromTopic(topic) + if err != nil { + return "", err } - return flow.Identifier{}, fmt.Errorf("spork ID is missing") + return sporkId.String(), nil +} + +// clusterIDStrFromTopic returns the appended cluster ID in flow.ChainID format for the cluster prefixed topic. +// A valid cluster-prefixed channel includes the cluster prefix and cluster ID suffix: +// +// sync-cluster/some_cluster_id +// +// A generic error is returned if the topic is malformed. +func clusterIDStrFromTopic(topic Topic) (flow.ChainID, error) { + for prefix := range clusterChannelPrefixRoleMap { + if strings.HasPrefix(topic.String(), prefix) { + return flow.ChainID(strings.TrimPrefix(topic.String(), fmt.Sprintf("%s-", prefix))), nil + } + } + return "", fmt.Errorf("failed to get cluster ID from topic %s", topic) } // ConsensusCluster returns a dynamic cluster consensus channel based on @@ -298,33 +337,63 @@ func SyncCluster(clusterID flow.ChainID) Channel { return Channel(fmt.Sprintf("%s-%s", SyncClusterPrefix, clusterID)) } -// IsValidFlowTopic ensures the topic is a valid Flow network topic. -// A valid Topic has the following properties: -// - A Channel can be derived from the Topic and that channel exists. -// - The sporkID part of the Topic is equal to the current network sporkID. -// All errors returned from this function can be considered benign. -func IsValidFlowTopic(topic Topic, expectedSporkID flow.Identifier) error { - channel, ok := ChannelFromTopic(topic) - if !ok { - return fmt.Errorf("invalid topic: failed to get channel from topic") - } - err := IsValidFlowChannel(channel) +// IsValidNonClusterFlowTopic ensures the topic is a valid Flow network topic and +// ensures the sporkID part of the Topic is equal to the current network sporkID. +// Expected errors: +// - InvalidTopicErr if the topic is not a if the topic is not a valid topic for the given spork. +func IsValidNonClusterFlowTopic(topic Topic, expectedSporkID flow.Identifier) error { + sporkID, err := sporkIdStrFromTopic(topic) if err != nil { - return fmt.Errorf("invalid topic: %w", err) + return NewInvalidTopicErr(topic, fmt.Errorf("failed to get spork ID from topic: %w", err)) } - if IsClusterChannel(channel) { - return nil + if sporkID != expectedSporkID.String() { + return NewInvalidTopicErr(topic, fmt.Errorf("invalid flow topic mismatch spork ID expected spork ID %s actual spork ID %s", expectedSporkID, sporkID)) } - sporkID, err := SporkIDFromTopic(topic) + return isValidFlowTopic(topic) +} + +// IsValidFlowClusterTopic ensures the topic is a valid Flow network topic and +// ensures the cluster ID part of the Topic is equal to one of the provided active cluster IDs. +// All errors returned from this function can be considered benign. +// Expected errors: +// - InvalidTopicErr if the topic is not a valid Flow topic or the cluster ID cannot be derived from the topic. +// - UnknownClusterIDErr if the cluster ID from the topic is not in the activeClusterIDS list. +func IsValidFlowClusterTopic(topic Topic, activeClusterIDS flow.ChainIDList) error { + err := isValidFlowTopic(topic) if err != nil { return err } - if sporkID != expectedSporkID { - return fmt.Errorf("invalid topic: wrong spork ID %s the current spork ID is %s", sporkID, expectedSporkID) + + clusterID, err := clusterIDStrFromTopic(topic) + if err != nil { + return NewInvalidTopicErr(topic, fmt.Errorf("failed to get cluster ID from topic: %w", err)) + } + + for _, activeClusterID := range activeClusterIDS { + if clusterID == activeClusterID { + return nil + } } + return NewUnknownClusterIdErr(clusterID, activeClusterIDS) +} + +// isValidFlowTopic ensures the topic is a valid Flow network topic. +// A valid Topic has the following properties: +// - A Channel can be derived from the Topic and that channel exists. +// Expected errors: +// - InvalidTopicErr if the topic is not a valid Flow topic. +func isValidFlowTopic(topic Topic) error { + channel, ok := ChannelFromTopic(topic) + if !ok { + return NewInvalidTopicErr(topic, fmt.Errorf("invalid topic: failed to get channel from topic")) + } + err := IsValidFlowChannel(channel) + if err != nil { + return NewInvalidTopicErr(topic, fmt.Errorf("invalid topic: %w", err)) + } return nil } diff --git a/network/channels/errors.go b/network/channels/errors.go new file mode 100644 index 00000000000..3be8c826417 --- /dev/null +++ b/network/channels/errors.go @@ -0,0 +1,50 @@ +package channels + +import ( + "errors" + "fmt" + + "github.com/onflow/flow-go/model/flow" +) + +// InvalidTopicErr error wrapper that indicates an error when checking if a Topic is a valid Flow Topic. +type InvalidTopicErr struct { + topic Topic + err error +} + +func (e InvalidTopicErr) Error() string { + return fmt.Errorf("invalid topic %s: %w", e.topic, e.err).Error() +} + +// NewInvalidTopicErr returns a new ErrMalformedTopic +func NewInvalidTopicErr(topic Topic, err error) InvalidTopicErr { + return InvalidTopicErr{topic: topic, err: err} +} + +// IsInvalidTopicErr returns true if an error is InvalidTopicErr +func IsInvalidTopicErr(err error) bool { + var e InvalidTopicErr + return errors.As(err, &e) +} + +// UnknownClusterIDErr error wrapper that indicates an invalid topic with an unknown cluster ID prefix. +type UnknownClusterIDErr struct { + clusterId flow.ChainID + activeClusterIds flow.ChainIDList +} + +func (e UnknownClusterIDErr) Error() string { + return fmt.Errorf("cluster ID %s not found in active cluster IDs list %s", e.clusterId, e.activeClusterIds).Error() +} + +// NewUnknownClusterIdErr returns a new UnknownClusterIDErr +func NewUnknownClusterIdErr(clusterId flow.ChainID, activeClusterIds flow.ChainIDList) UnknownClusterIDErr { + return UnknownClusterIDErr{clusterId: clusterId, activeClusterIds: activeClusterIds} +} + +// IsUnknownClusterIDErr returns true if an error is UnknownClusterIDErr +func IsUnknownClusterIDErr(err error) bool { + var e UnknownClusterIDErr + return errors.As(err, &e) +} diff --git a/network/channels/errors_test.go b/network/channels/errors_test.go new file mode 100644 index 00000000000..56dd23cd09b --- /dev/null +++ b/network/channels/errors_test.go @@ -0,0 +1,46 @@ +package channels + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/onflow/flow-go/model/flow" +) + +// TestInvalidTopicErrRoundTrip ensures correct error formatting for InvalidTopicErr. +func TestInvalidTopicErrRoundTrip(t *testing.T) { + topic := Topic("invalid-topic") + wrapErr := fmt.Errorf("this err should be wrapped with topic to add context") + err := NewInvalidTopicErr(topic, wrapErr) + + // tests the error message formatting. + expectedErrMsg := fmt.Errorf("invalid topic %s: %w", topic, wrapErr).Error() + assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") + + // tests the IsErrActiveClusterIDsNotSet function. + assert.True(t, IsInvalidTopicErr(err), "IsInvalidTopicErr should return true for InvalidTopicErr error") + + // test IsErrActiveClusterIDsNotSet with a different error type. + dummyErr := fmt.Errorf("dummy error") + assert.False(t, IsInvalidTopicErr(dummyErr), "IsInvalidTopicErr should return false for non-IsInvalidTopicErr error") +} + +// TestUnknownClusterIDErrRoundTrip ensures correct error formatting for UnknownClusterIDErr. +func TestUnknownClusterIDErrRoundTrip(t *testing.T) { + clusterId := flow.ChainID("cluster-id") + activeClusterIds := flow.ChainIDList{"active", "cluster", "ids"} + err := NewUnknownClusterIdErr(clusterId, activeClusterIds) + + // tests the error message formatting. + expectedErrMsg := fmt.Errorf("cluster ID %s not found in active cluster IDs list %s", clusterId, activeClusterIds).Error() + assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") + + // tests the IsErrActiveClusterIDsNotSet function. + assert.True(t, IsUnknownClusterIDErr(err), "IsUnknownClusterIDErr should return true for UnknownClusterIDErr error") + + // test IsErrActiveClusterIDsNotSet with a different error type. + dummyErr := fmt.Errorf("dummy error") + assert.False(t, IsUnknownClusterIDErr(dummyErr), "IsUnknownClusterIDErr should return false for non-UnknownClusterIDErr error") +} diff --git a/network/conduit.go b/network/conduit.go index f650c88fcb9..fa6e891e09a 100644 --- a/network/conduit.go +++ b/network/conduit.go @@ -29,7 +29,7 @@ type ConduitFactory interface { // a network-agnostic way. In the background, the network layer connects all // engines with the same ID over a shared bus, accessible through the conduit. type Conduit interface { - + MisbehaviorReporter // Publish submits an event to the network layer for unreliable delivery // to subscribers of the given event on the network layer. It uses a // publish-subscribe layer and can thus not guarantee that the specified diff --git a/network/converter/network.go b/network/converter/network.go index f5faf792db8..a30bb683d61 100644 --- a/network/converter/network.go +++ b/network/converter/network.go @@ -11,6 +11,8 @@ type Network struct { to channels.Channel } +var _ network.Network = (*Network)(nil) + func NewNetwork(net network.Network, from channels.Channel, to channels.Channel) *Network { return &Network{net, from, to} } diff --git a/network/disallow.go b/network/disallow.go new file mode 100644 index 00000000000..feaa6d2b27b --- /dev/null +++ b/network/disallow.go @@ -0,0 +1,49 @@ +package network + +import ( + "github.com/onflow/flow-go/model/flow" +) + +// DisallowListedCause is a type representing the cause of disallow listing. A remote node may be disallow-listed by the +// current node for a variety of reasons. This type is used to represent the reason for disallow-listing, so that if +// a node is disallow-listed for reasons X and Y, allow-listing it back for reason X does not automatically allow-list +// it for reason Y. +type DisallowListedCause string + +func (c DisallowListedCause) String() string { + return string(c) +} + +const ( + // DisallowListedCauseAdmin is the cause of disallow-listing a node by an admin command. + DisallowListedCauseAdmin DisallowListedCause = "disallow-listed-admin" + // DisallowListedCauseAlsp is the cause of disallow-listing a node by the ALSP (Application Layer Spam Prevention). + DisallowListedCauseAlsp DisallowListedCause = "disallow-listed-alsp" +) + +// DisallowListingUpdate is a notification of a new disallow list update, it contains a list of Flow identities that +// are now disallow listed for a specific reason. +type DisallowListingUpdate struct { + FlowIds flow.IdentifierList + Cause DisallowListedCause +} + +// AllowListingUpdate is a notification of a new allow list update, it contains a list of Flow identities that +// are now allow listed for a specific reason, i.e., their disallow list entry for that reason is removed. +type AllowListingUpdate struct { + FlowIds flow.IdentifierList + Cause DisallowListedCause +} + +// DisallowListNotificationConsumer is an interface for consuming disallow/allow list update notifications. +type DisallowListNotificationConsumer interface { + // OnDisallowListNotification is called when a new disallow list update notification is distributed. + // Any error on consuming an event must be handled internally. + // The implementation must be concurrency safe. + OnDisallowListNotification(*DisallowListingUpdate) + + // OnAllowListNotification is called when a new allow list update notification is distributed. + // Any error on consuming an event must be handled internally. + // The implementation must be concurrency safe. + OnAllowListNotification(*AllowListingUpdate) +} diff --git a/network/engine.go b/network/engine.go index fc33c6f9563..a6c2fd6707a 100644 --- a/network/engine.go +++ b/network/engine.go @@ -46,5 +46,15 @@ type Engine interface { // (including invalid message types, malformed messages, etc.). Because of this, // node-internal messages should NEVER be submitted to a component using Process. type MessageProcessor interface { + // Process is exposed by engines to accept messages from the networking layer. + // Implementations of Process should be non-blocking. In general, Process should + // only queue the message internally by the engine for later async processing. + // + // TODO: This function should not return an error. + // The networking layer's responsibility is fulfilled once it delivers a message to an engine. + // It does not possess the context required to handle errors that may arise during an engine's processing + // of the message, as error handling for message processing falls outside the domain of the networking layer. + // Consequently, it is reasonable to remove the error from the Process function's signature, + // since returning an error to the networking layer would not be useful in this context. Process(channel channels.Channel, originID flow.Identifier, message interface{}) error } diff --git a/network/internal/p2pfixtures/fixtures.go b/network/internal/p2pfixtures/fixtures.go index 0ede4e64cf6..7b7365d57ab 100644 --- a/network/internal/p2pfixtures/fixtures.go +++ b/network/internal/p2pfixtures/fixtures.go @@ -21,25 +21,24 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/metrics" + flownet "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2putils" - "github.com/onflow/flow-go/network/internal/testutils" "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/p2p" p2pdht "github.com/onflow/flow-go/network/p2p/dht" - "github.com/onflow/flow-go/network/p2p/distributor" "github.com/onflow/flow-go/network/p2p/keyutils" "github.com/onflow/flow-go/network/p2p/p2pbuilder" - inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/unicast" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" - validator "github.com/onflow/flow-go/network/validator/pubsub" "github.com/onflow/flow-go/utils/unittest" ) @@ -98,33 +97,50 @@ func WithSubscriptionFilter(filter pubsub.SubscriptionFilter) nodeOpt { } } +// TODO: this should be replaced by node fixture: https://github.com/onflow/flow-go/blob/master/network/p2p/test/fixtures.go func CreateNode(t *testing.T, networkKey crypto.PrivateKey, sporkID flow.Identifier, logger zerolog.Logger, nodeIds flow.IdentityList, opts ...nodeOpt) p2p.LibP2PNode { idProvider := id.NewFixedIdentityProvider(nodeIds) - - meshTracer := tracer.NewGossipSubMeshTracer( - logger, - metrics.NewNoopCollector(), - idProvider, - p2pbuilder.DefaultGossipSubConfig().LocalMeshLogInterval) - - rpcInspectors, err := inspectorbuilder.NewGossipSubInspectorBuilder(logger, sporkID, inspectorbuilder.DefaultGossipSubRPCInspectorsConfig(), distributor.DefaultGossipSubInspectorNotificationDistributor(logger)).Build() + defaultFlowConfig, err := config.DefaultConfig() require.NoError(t, err) + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: logger, + Metrics: metrics.NewNoopCollector(), + IDProvider: idProvider, + LoggerInterval: defaultFlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, + RpcSentTrackerCacheSize: defaultFlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + RpcSentTrackerWorkerQueueCacheSize: defaultFlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, + RpcSentTrackerNumOfWorkers: defaultFlowConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, + HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), + NetworkingType: flownet.PublicNetwork, + } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) + builder := p2pbuilder.NewNodeBuilder( logger, - metrics.NewNoopCollector(), + &p2pconfig.MetricsConfig{ + HeroCacheFactory: metrics.NewNoopHeroCacheMetricsFactory(), + Metrics: metrics.NewNoopCollector(), + }, + flownet.PrivateNetwork, unittest.DefaultAddress, networkKey, sporkID, - p2pbuilder.DefaultResourceManagerConfig()). + idProvider, + &defaultFlowConfig.NetworkConfig.ResourceManagerConfig, + &defaultFlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, + p2pconfig.PeerManagerDisableConfig(), + &p2p.DisallowListCacheConfig{ + MaxSize: uint32(1000), + Metrics: metrics.NewNoopCollector(), + }). SetRoutingSystem(func(c context.Context, h host.Host) (routing.Routing, error) { return p2pdht.NewDHT(c, h, protocols.FlowDHTProtocolID(sporkID), zerolog.Nop(), metrics.NewNoopCollector()) }). - SetResourceManager(testutils.NewResourceManager(t)). + SetResourceManager(&network.NullResourceManager{}). SetStreamCreationRetryInterval(unicast.DefaultRetryDelay). SetGossipSubTracer(meshTracer). - SetGossipSubScoreTracerInterval(p2pbuilder.DefaultGossipSubConfig().ScoreTracerInterval). - SetGossipSubRPCInspectors(rpcInspectors...) + SetGossipSubScoreTracerInterval(defaultFlowConfig.NetworkConfig.GossipSubConfig.ScoreTracerInterval) for _, opt := range opts { opt(builder) @@ -241,66 +257,12 @@ func EnsureNotConnected(t *testing.T, ctx context.Context, from []p2p.LibP2PNode // Hence, we instead check for any trace of the connection being established in the receiver side. _ = this.Host().Connect(ctx, other.Host().Peerstore().PeerInfo(other.Host().ID())) // ensures that other node has never received a connection from this node. - require.Equal(t, other.Host().Network().Connectedness(thisId), network.NotConnected) + require.Equal(t, network.NotConnected, other.Host().Network().Connectedness(thisId)) require.Empty(t, other.Host().Network().ConnsToPeer(thisId)) } } } -// EnsureNotConnectedBetweenGroups ensures no connection exists between the given groups of nodes. -func EnsureNotConnectedBetweenGroups(t *testing.T, ctx context.Context, groupA []p2p.LibP2PNode, groupB []p2p.LibP2PNode) { - // ensure no connection from group A to group B - EnsureNotConnected(t, ctx, groupA, groupB) - // ensure no connection from group B to group A - EnsureNotConnected(t, ctx, groupB, groupA) -} - -// EnsureNoPubsubMessageExchange ensures that the no pubsub message is exchanged "from" the given nodes "to" the given nodes. -func EnsureNoPubsubMessageExchange(t *testing.T, ctx context.Context, from []p2p.LibP2PNode, to []p2p.LibP2PNode, messageFactory func() (interface{}, channels.Topic)) { - _, topic := messageFactory() - - subs := make([]p2p.Subscription, len(to)) - tv := validator.TopicValidator( - unittest.Logger(), - unittest.AllowAllPeerFilter()) - var err error - for _, node := range from { - _, err = node.Subscribe(topic, tv) - require.NoError(t, err) - } - - for i, node := range to { - s, err := node.Subscribe(topic, tv) - require.NoError(t, err) - subs[i] = s - } - - // let subscriptions propagate - time.Sleep(1 * time.Second) - - for _, node := range from { - // creates a unique message to be published by the node. - msg, _ := messageFactory() - channel, ok := channels.ChannelFromTopic(topic) - require.True(t, ok) - data := MustEncodeEvent(t, msg, channel) - - // ensure the message is NOT received by any of the nodes. - require.NoError(t, node.Publish(ctx, topic, data)) - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - SubsMustNeverReceiveAnyMessage(t, ctx, subs) - cancel() - } -} - -// EnsureNoPubsubExchangeBetweenGroups ensures that no pubsub message is exchanged between the given groups of nodes. -func EnsureNoPubsubExchangeBetweenGroups(t *testing.T, ctx context.Context, groupA []p2p.LibP2PNode, groupB []p2p.LibP2PNode, messageFactory func() (interface{}, channels.Topic)) { - // ensure no message exchange from group A to group B - EnsureNoPubsubMessageExchange(t, ctx, groupA, groupB, messageFactory) - // ensure no message exchange from group B to group A - EnsureNoPubsubMessageExchange(t, ctx, groupB, groupA, messageFactory) -} - // EnsureMessageExchangeOverUnicast ensures that the given nodes exchange arbitrary messages on through unicasting (i.e., stream creation). // It fails the test if any of the nodes does not receive the message from the other nodes. // The "inbounds" parameter specifies the inbound channel of the nodes on which the messages are received. diff --git a/network/internal/testutils/fixtures.go b/network/internal/testutils/fixtures.go new file mode 100644 index 00000000000..b2fff20abbb --- /dev/null +++ b/network/internal/testutils/fixtures.go @@ -0,0 +1,54 @@ +package testutils + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/alsp" + "github.com/onflow/flow-go/utils/unittest" +) + +// MisbehaviorReportFixture generates a random misbehavior report. +// Args: +// - t: the test object. +// +// This is used in tests to generate random misbehavior reports. +// It fails the test if it cannot generate a valid report. +func MisbehaviorReportFixture(t *testing.T) network.MisbehaviorReport { + + // pick a random misbehavior type + misbehaviorType := alsp.AllMisbehaviorTypes()[rand.Intn(len(alsp.AllMisbehaviorTypes()))] + + amplification := 100 * rand.Float64() + report, err := alsp.NewMisbehaviorReport( + unittest.IdentifierFixture(), + misbehaviorType, + alsp.WithPenaltyAmplification(amplification)) + require.NoError(t, err) + return report +} + +// MisbehaviorReportsFixture generates a slice of random misbehavior reports. +// Args: +// - t: the test object. +// +// It fails the test if it cannot generate a valid report. +// This is used in tests to generate random misbehavior reports. +func MisbehaviorReportsFixture(t *testing.T, count int) []network.MisbehaviorReport { + reports := make([]network.MisbehaviorReport, 0, count) + for i := 0; i < count; i++ { + reports = append(reports, MisbehaviorReportFixture(t)) + } + + return reports +} + +// MisbehaviorTypeFixture generates a random misbehavior type. +// Args: +// - t: the test object (used to emphasize that this is a test helper). +func MisbehaviorTypeFixture(_ *testing.T) network.Misbehavior { + return alsp.AllMisbehaviorTypes()[rand.Intn(len(alsp.AllMisbehaviorTypes()))] +} diff --git a/network/internal/testutils/testUtil.go b/network/internal/testutils/testUtil.go index fd8803c7499..95707ee9e3c 100644 --- a/network/internal/testutils/testUtil.go +++ b/network/internal/testutils/testUtil.go @@ -1,7 +1,6 @@ package testutils import ( - "context" "fmt" "reflect" "runtime" @@ -10,17 +9,11 @@ import ( "testing" "time" - dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p/core/connmgr" - "github.com/libp2p/go-libp2p/core/host" - p2pNetwork "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" - pc "github.com/libp2p/go-libp2p/core/protocol" - "github.com/libp2p/go-libp2p/core/routing" "github.com/rs/zerolog" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" @@ -31,22 +24,18 @@ import ( "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/module/observable" "github.com/onflow/flow-go/network" + alspmgr "github.com/onflow/flow-go/network/alsp/manager" netcache "github.com/onflow/flow-go/network/cache" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/codec/cbor" + "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/conduit" "github.com/onflow/flow-go/network/p2p/connection" - p2pdht "github.com/onflow/flow-go/network/p2p/dht" - "github.com/onflow/flow-go/network/p2p/distributor" "github.com/onflow/flow-go/network/p2p/middleware" - "github.com/onflow/flow-go/network/p2p/p2pbuilder" - inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" "github.com/onflow/flow-go/network/p2p/subscription" + p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/translator" - "github.com/onflow/flow-go/network/p2p/unicast" - "github.com/onflow/flow-go/network/p2p/unicast/protocols" - "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/utils/unittest" ) @@ -111,7 +100,7 @@ func (tw *TagWatchingConnManager) Unprotect(id peer.ID, tag string) bool { } // NewTagWatchingConnManager creates a new TagWatchingConnManager with the given config. It returns an error if the config is invalid. -func NewTagWatchingConnManager(log zerolog.Logger, metrics module.LibP2PConnectionMetrics, config *connection.ManagerConfig) (*TagWatchingConnManager, error) { +func NewTagWatchingConnManager(log zerolog.Logger, metrics module.LibP2PConnectionMetrics, config *netconf.ConnectionManagerConfig) (*TagWatchingConnManager, error) { cm, err := connection.NewConnManager(log, metrics, config) if err != nil { return nil, fmt.Errorf("could not create connection manager: %w", err) @@ -124,137 +113,108 @@ func NewTagWatchingConnManager(log zerolog.Logger, metrics module.LibP2PConnecti }, nil } -// GenerateIDs is a test helper that generate flow identities with a valid port and libp2p nodes. -func GenerateIDs(t *testing.T, logger zerolog.Logger, n int, opts ...func(*optsConfig)) (flow.IdentityList, - []p2p.LibP2PNode, - []observable.Observable) { - libP2PNodes := make([]p2p.LibP2PNode, n) - tagObservables := make([]observable.Observable, n) - - identities := unittest.IdentityListFixture(n, unittest.WithAllRoles()) - idProvider := NewUpdatableIDProvider(identities) - o := &optsConfig{ - peerUpdateInterval: connection.DefaultPeerUpdateInterval, - unicastRateLimiterDistributor: ratelimit.NewUnicastRateLimiterDistributor(), - connectionGater: NewConnectionGater(idProvider, func(p peer.ID) error { - return nil - }), - createStreamRetryInterval: unicast.DefaultRetryDelay, - } - for _, opt := range opts { - opt(o) - } - - for _, identity := range identities { - for _, idOpt := range o.idOpts { - idOpt(identity) - } - } - - // generates keys and address for the node - for i, identity := range identities { - // generate key - key, err := generateNetworkingKey(identity.NodeID) - require.NoError(t, err) - - var opts []nodeBuilderOption - - opts = append(opts, withDHT(o.dhtPrefix, o.dhtOpts...)) - opts = append(opts, withPeerManagerOptions(connection.ConnectionPruningEnabled, o.peerUpdateInterval)) - opts = append(opts, withRateLimiterDistributor(o.unicastRateLimiterDistributor)) - opts = append(opts, withConnectionGater(o.connectionGater)) - opts = append(opts, withUnicastManagerOpts(o.createStreamRetryInterval)) +// LibP2PNodeForMiddlewareFixture is a test helper that generate flow identities with a valid port and libp2p nodes. +// Note that the LibP2PNode created by this fixture is meant to used with a middleware component. +// If you want to create a standalone LibP2PNode without network and middleware components, please use p2ptest.NodeFixture. +// Args: +// +// t: testing.T- the test object +// +// n: int - number of nodes to create +// opts: []p2ptest.NodeFixtureParameterOption - options to configure the nodes +// Returns: +// +// flow.IdentityList - list of identities created for the nodes, one for each node. +// +// []p2p.LibP2PNode - list of libp2p nodes created. +// []observable.Observable - list of observables created for each node. +func LibP2PNodeForMiddlewareFixture(t *testing.T, n int, opts ...p2ptest.NodeFixtureParameterOption) (flow.IdentityList, []p2p.LibP2PNode, []observable.Observable) { + + libP2PNodes := make([]p2p.LibP2PNode, 0) + identities := make(flow.IdentityList, 0) + tagObservables := make([]observable.Observable, 0) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) + defaultFlowConfig, err := config.DefaultConfig() + require.NoError(t, err) - libP2PNodes[i], tagObservables[i] = generateLibP2PNode(t, logger, key, opts...) + opts = append(opts, p2ptest.WithUnicastHandlerFunc(nil)) - _, port, err := libP2PNodes[i].GetIPPort() + for i := 0; i < n; i++ { + // TODO: generating a tag watching connection manager can be moved to a separate function, as only a few tests need this. + // For the rest of tests, the node can run on the default connection manager without setting and option. + connManager, err := NewTagWatchingConnManager(unittest.Logger(), metrics.NewNoopCollector(), &defaultFlowConfig.NetworkConfig.ConnectionManagerConfig) require.NoError(t, err) - identities[i].Address = unittest.IPPort(port) - identities[i].NetworkPubKey = key.PublicKey() - } - + opts = append(opts, p2ptest.WithConnectionManager(connManager)) + node, nodeId := p2ptest.NodeFixture(t, + sporkID, + t.Name(), + idProvider, + opts...) + libP2PNodes = append(libP2PNodes, node) + identities = append(identities, &nodeId) + tagObservables = append(tagObservables, connManager) + } + idProvider.SetIdentities(identities) return identities, libP2PNodes, tagObservables } -// GenerateMiddlewares creates and initializes middleware instances for all the identities -func GenerateMiddlewares(t *testing.T, - logger zerolog.Logger, - identities flow.IdentityList, - libP2PNodes []p2p.LibP2PNode, - codec network.Codec, - consumer slashing.ViolationsConsumer, - opts ...func(*optsConfig)) ([]network.Middleware, []*UpdatableIDProvider) { - mws := make([]network.Middleware, len(identities)) - idProviders := make([]*UpdatableIDProvider, len(identities)) - bitswapmet := metrics.NewNoopCollector() - o := &optsConfig{ - peerUpdateInterval: connection.DefaultPeerUpdateInterval, - unicastRateLimiters: ratelimit.NoopRateLimiters(), - networkMetrics: metrics.NewNoopCollector(), - peerManagerFilters: []p2p.PeerFilter{}, - } - - for _, opt := range opts { - opt(o) - } - - total := len(identities) - for i := 0; i < total; i++ { - // casts libP2PNode instance to a local variable to avoid closure - node := libP2PNodes[i] - nodeId := identities[i].NodeID +// MiddlewareConfigFixture is a test helper that generates a middleware config for testing. +// Args: +// - t: the test instance. +// Returns: +// - a middleware config. +func MiddlewareConfigFixture(t *testing.T) *middleware.Config { + return &middleware.Config{ + Logger: unittest.Logger(), + BitSwapMetrics: metrics.NewNoopCollector(), + RootBlockID: sporkID, + UnicastMessageTimeout: middleware.DefaultUnicastTimeout, + Codec: unittest.NetworkCodec(), + } +} + +// MiddlewareFixtures is a test helper that generates middlewares with the given identities and libp2p nodes. +// It also generates a list of UpdatableIDProvider that can be used to update the identities of the middlewares. +// The number of identities and libp2p nodes must be the same. +// Args: +// - identities: a list of flow identities that correspond to the libp2p nodes. +// - libP2PNodes: a list of libp2p nodes that correspond to the identities. +// - cfg: the middleware config. +// - opts: a list of middleware option functions. +// Returns: +// - a list of middlewares - one for each identity. +// - a list of UpdatableIDProvider - one for each identity. +func MiddlewareFixtures(t *testing.T, identities flow.IdentityList, libP2PNodes []p2p.LibP2PNode, cfg *middleware.Config, consumer network.ViolationsConsumer, opts ...middleware.OptionFn) ([]network.Middleware, []*unittest.UpdatableIDProvider) { + require.Equal(t, len(identities), len(libP2PNodes)) - idProviders[i] = NewUpdatableIDProvider(identities) + mws := make([]network.Middleware, len(identities)) + idProviders := make([]*unittest.UpdatableIDProvider, len(identities)) - // creating middleware of nodes - mws[i] = middleware.NewMiddleware( - logger, - node, - nodeId, - bitswapmet, - sporkID, - middleware.DefaultUnicastTimeout, - translator.NewIdentityProviderIDTranslator(idProviders[i]), - codec, - consumer, - middleware.WithUnicastRateLimiters(o.unicastRateLimiters), - middleware.WithPeerManagerFilters(o.peerManagerFilters)) + for i := 0; i < len(identities); i++ { + i := i + cfg.Libp2pNode = libP2PNodes[i] + cfg.FlowId = identities[i].NodeID + idProviders[i] = unittest.NewUpdatableIDProvider(identities) + cfg.IdTranslator = translator.NewIdentityProviderIDTranslator(idProviders[i]) + mws[i] = middleware.NewMiddleware(cfg, opts...) + mws[i].SetSlashingViolationsConsumer(consumer) } return mws, idProviders } -// GenerateNetworks generates the network for the given middlewares -func GenerateNetworks(t *testing.T, - log zerolog.Logger, +// NetworksFixture generates the network for the given middlewares +func NetworksFixture(t *testing.T, ids flow.IdentityList, - mws []network.Middleware, - sms []network.SubscriptionManager) []network.Network { + mws []network.Middleware) []network.Network { + count := len(ids) nets := make([]network.Network, 0) for i := 0; i < count; i++ { - // creates and mocks me - me := &mock.Local{} - me.On("NodeID").Return(ids[i].NodeID) - me.On("NotMeFilter").Return(filter.Not(filter.HasNodeID(me.NodeID()))) - me.On("Address").Return(ids[i].Address) - - receiveCache := netcache.NewHeroReceiveCache(p2p.DefaultReceiveCacheSize, log, metrics.NewNoopCollector()) - - // create the network - net, err := p2p.NewNetwork(&p2p.NetworkParameters{ - Logger: log, - Codec: cbor.NewCodec(), - Me: me, - MiddlewareFactory: func() (network.Middleware, error) { return mws[i], nil }, - Topology: unittest.NetworkTopology(), - SubscriptionManager: sms[i], - Metrics: metrics.NewNoopCollector(), - IdentityProvider: id.NewFixedIdentityProvider(ids), - ReceiveCache: receiveCache, - }) + params := NetworkConfigFixture(t, *ids[i], ids, mws[i]) + net, err := p2p.NewNetwork(params) require.NoError(t, err) nets = append(nets, net) @@ -263,98 +223,52 @@ func GenerateNetworks(t *testing.T, return nets } -// GenerateIDsAndMiddlewares returns nodeIDs, libp2pNodes, middlewares, and observables which can be subscirbed to in order to witness protect events from pubsub -func GenerateIDsAndMiddlewares(t *testing.T, - n int, - logger zerolog.Logger, - codec network.Codec, - consumer slashing.ViolationsConsumer, - opts ...func(*optsConfig)) (flow.IdentityList, []p2p.LibP2PNode, []network.Middleware, []observable.Observable, []*UpdatableIDProvider) { - - ids, libP2PNodes, protectObservables := GenerateIDs(t, logger, n, opts...) - mws, providers := GenerateMiddlewares(t, logger, ids, libP2PNodes, codec, consumer, opts...) - return ids, libP2PNodes, mws, protectObservables, providers -} +func NetworkConfigFixture( + t *testing.T, + myId flow.Identity, + allIds flow.IdentityList, + mw network.Middleware, + opts ...p2p.NetworkConfigOption) *p2p.NetworkConfig { -type optsConfig struct { - idOpts []func(*flow.Identity) - dhtPrefix string - dhtOpts []dht.Option - unicastRateLimiters *ratelimit.RateLimiters - peerUpdateInterval time.Duration - networkMetrics module.NetworkMetrics - peerManagerFilters []p2p.PeerFilter - unicastRateLimiterDistributor p2p.UnicastRateLimiterDistributor - connectionGater connmgr.ConnectionGater - createStreamRetryInterval time.Duration -} - -func WithCreateStreamRetryInterval(delay time.Duration) func(*optsConfig) { - return func(o *optsConfig) { - o.createStreamRetryInterval = delay - } -} - -func WithUnicastRateLimiterDistributor(distributor p2p.UnicastRateLimiterDistributor) func(*optsConfig) { - return func(o *optsConfig) { - o.unicastRateLimiterDistributor = distributor - } -} - -func WithIdentityOpts(idOpts ...func(*flow.Identity)) func(*optsConfig) { - return func(o *optsConfig) { - o.idOpts = idOpts - } -} + me := mock.NewLocal(t) + me.On("NodeID").Return(myId.NodeID).Maybe() + me.On("NotMeFilter").Return(filter.Not(filter.HasNodeID(me.NodeID()))).Maybe() + me.On("Address").Return(myId.Address).Maybe() -func WithDHT(prefix string, dhtOpts ...dht.Option) func(*optsConfig) { - return func(o *optsConfig) { - o.dhtPrefix = prefix - o.dhtOpts = dhtOpts - } -} - -func WithPeerUpdateInterval(interval time.Duration) func(*optsConfig) { - return func(o *optsConfig) { - o.peerUpdateInterval = interval - } -} - -func WithPeerManagerFilters(filters ...p2p.PeerFilter) func(*optsConfig) { - return func(o *optsConfig) { - o.peerManagerFilters = filters - } -} - -func WithUnicastRateLimiters(limiters *ratelimit.RateLimiters) func(*optsConfig) { - return func(o *optsConfig) { - o.unicastRateLimiters = limiters - } -} + defaultFlowConfig, err := config.DefaultConfig() + require.NoError(t, err) -func WithConnectionGater(connectionGater connmgr.ConnectionGater) func(*optsConfig) { - return func(o *optsConfig) { - o.connectionGater = connectionGater + receiveCache := netcache.NewHeroReceiveCache( + defaultFlowConfig.NetworkConfig.NetworkReceivedMessageCacheSize, + unittest.Logger(), + metrics.NewNoopCollector()) + subMgr := subscription.NewChannelSubscriptionManager(mw) + params := &p2p.NetworkConfig{ + Logger: unittest.Logger(), + Codec: unittest.NetworkCodec(), + Me: me, + MiddlewareFactory: func() (network.Middleware, error) { return mw, nil }, + Topology: unittest.NetworkTopology(), + SubscriptionManager: subMgr, + Metrics: metrics.NewNoopCollector(), + IdentityProvider: id.NewFixedIdentityProvider(allIds), + ReceiveCache: receiveCache, + ConduitFactory: conduit.NewDefaultConduitFactory(), + AlspCfg: &alspmgr.MisbehaviorReportManagerConfig{ + Logger: unittest.Logger(), + SpamRecordCacheSize: defaultFlowConfig.NetworkConfig.AlspConfig.SpamRecordCacheSize, + SpamReportQueueSize: defaultFlowConfig.NetworkConfig.AlspConfig.SpamReportQueueSize, + HeartBeatInterval: defaultFlowConfig.NetworkConfig.AlspConfig.HearBeatInterval, + AlspMetrics: metrics.NewNoopCollector(), + HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), + }, } -} -func WithNetworkMetrics(m module.NetworkMetrics) func(*optsConfig) { - return func(o *optsConfig) { - o.networkMetrics = m + for _, opt := range opts { + opt(params) } -} -func GenerateIDsMiddlewaresNetworks(t *testing.T, - n int, - log zerolog.Logger, - codec network.Codec, - consumer slashing.ViolationsConsumer, - opts ...func(*optsConfig)) (flow.IdentityList, []p2p.LibP2PNode, []network.Middleware, []network.Network, []observable.Observable) { - ids, libp2pNodes, mws, observables, _ := GenerateIDsAndMiddlewares(t, n, log, codec, consumer, opts...) - sms := GenerateSubscriptionManagers(t, mws) - networks := GenerateNetworks(t, log, ids, mws, sms) - - return ids, libp2pNodes, mws, networks, observables + return params } // GenerateEngines generates MeshEngines for the given networks @@ -368,16 +282,36 @@ func GenerateEngines(t *testing.T, nets []network.Network) []*MeshEngine { return engs } -// StartNodesAndNetworks starts the provided networks and libp2p nodes, returning the irrecoverable error channel -func StartNodesAndNetworks(ctx irrecoverable.SignalerContext, t *testing.T, nodes []p2p.LibP2PNode, nets []network.Network, duration time.Duration) { +// StartNodesAndNetworks starts the provided networks and libp2p nodes, returning the irrecoverable error channel. +// Arguments: +// - ctx: the irrecoverable context to use for starting the nodes and networks. +// - t: the test object. +// - nodes: the libp2p nodes to start. +// - nets: the networks to start. +// - timeout: the timeout to use for waiting for the nodes and networks to start. +// +// This function fails the test if the nodes or networks do not start within the given timeout. +func StartNodesAndNetworks(ctx irrecoverable.SignalerContext, t *testing.T, nodes []p2p.LibP2PNode, nets []network.Network, timeout time.Duration) { + StartNetworks(ctx, t, nets, timeout) + + // start up nodes and Peer managers + StartNodes(ctx, t, nodes, timeout) +} + +// StartNetworks starts the provided networks using the provided irrecoverable context +// Arguments: +// - ctx: the irrecoverable context to use for starting the networks. +// - t: the test object. +// - nets: the networks to start. +// - duration: the timeout to use for waiting for the networks to start. +// +// This function fails the test if the networks do not start within the given timeout. +func StartNetworks(ctx irrecoverable.SignalerContext, t *testing.T, nets []network.Network, duration time.Duration) { // start up networks (this will implicitly start middlewares) for _, net := range nets { net.Start(ctx) unittest.RequireComponentsReadyBefore(t, duration, net) } - - // start up nodes and Peer managers - StartNodes(ctx, t, nodes, duration) } // StartNodes starts the provided nodes and their peer managers using the provided irrecoverable context @@ -403,77 +337,6 @@ func StopComponents[R module.ReadyDoneAware](t *testing.T, rda []R, duration tim unittest.RequireComponentsDoneBefore(t, duration, comps...) } -type nodeBuilderOption func(p2p.NodeBuilder) - -func withDHT(prefix string, dhtOpts ...dht.Option) nodeBuilderOption { - return func(nb p2p.NodeBuilder) { - nb.SetRoutingSystem(func(c context.Context, h host.Host) (routing.Routing, error) { - return p2pdht.NewDHT(c, h, pc.ID(protocols.FlowDHTProtocolIDPrefix+prefix), zerolog.Nop(), metrics.NewNoopCollector(), dhtOpts...) - }) - } -} - -func withPeerManagerOptions(connectionPruning bool, updateInterval time.Duration) nodeBuilderOption { - return func(nb p2p.NodeBuilder) { - nb.SetPeerManagerOptions(connectionPruning, updateInterval) - } -} - -func withRateLimiterDistributor(distributor p2p.UnicastRateLimiterDistributor) nodeBuilderOption { - return func(nb p2p.NodeBuilder) { - nb.SetRateLimiterDistributor(distributor) - } -} - -func withConnectionGater(connectionGater connmgr.ConnectionGater) nodeBuilderOption { - return func(nb p2p.NodeBuilder) { - nb.SetConnectionGater(connectionGater) - } -} - -func withUnicastManagerOpts(delay time.Duration) nodeBuilderOption { - return func(nb p2p.NodeBuilder) { - nb.SetStreamCreationRetryInterval(delay) - } -} - -// generateLibP2PNode generates a `LibP2PNode` on localhost using a port assigned by the OS -func generateLibP2PNode(t *testing.T, - logger zerolog.Logger, - key crypto.PrivateKey, - opts ...nodeBuilderOption) (p2p.LibP2PNode, observable.Observable) { - - noopMetrics := metrics.NewNoopCollector() - - // Inject some logic to be able to observe connections of this node - connManager, err := NewTagWatchingConnManager(logger, noopMetrics, connection.DefaultConnManagerConfig()) - require.NoError(t, err) - - rpcInspectors, err := inspectorbuilder.NewGossipSubInspectorBuilder(logger, sporkID, inspectorbuilder.DefaultGossipSubRPCInspectorsConfig(), distributor.DefaultGossipSubInspectorNotificationDistributor(logger)).Build() - require.NoError(t, err) - - builder := p2pbuilder.NewNodeBuilder( - logger, - metrics.NewNoopCollector(), - unittest.DefaultAddress, - key, - sporkID, - p2pbuilder.DefaultResourceManagerConfig()). - SetConnectionManager(connManager). - SetResourceManager(NewResourceManager(t)). - SetStreamCreationRetryInterval(unicast.DefaultRetryDelay). - SetGossipSubRPCInspectors(rpcInspectors...) - - for _, opt := range opts { - opt(builder) - } - - libP2PNode, err := builder.Build() - require.NoError(t, err) - - return libP2PNode, connManager -} - // OptionalSleep introduces a sleep to allow nodes to heartbeat and discover each other (only needed when using PubSub) func OptionalSleep(send ConduitSendWrapperFunc) { sendFuncName := runtime.FuncForPC(reflect.ValueOf(send).Pointer()).Name() @@ -482,24 +345,6 @@ func OptionalSleep(send ConduitSendWrapperFunc) { } } -// generateNetworkingKey generates a Flow ECDSA key using the given seed -func generateNetworkingKey(s flow.Identifier) (crypto.PrivateKey, error) { - seed := make([]byte, crypto.KeyGenSeedMinLen) - copy(seed, s[:]) - return crypto.GeneratePrivateKey(crypto.ECDSASecp256k1, seed) -} - -// GenerateSubscriptionManagers creates and returns a ChannelSubscriptionManager for each middleware object. -func GenerateSubscriptionManagers(t *testing.T, mws []network.Middleware) []network.SubscriptionManager { - require.NotEmpty(t, mws) - - sms := make([]network.SubscriptionManager, len(mws)) - for i, mw := range mws { - sms[i] = subscription.NewChannelSubscriptionManager(mw) - } - return sms -} - // NetworkPayloadFixture creates a blob of random bytes with the given size (in bytes) and returns it. // The primary goal of utilizing this helper function is to apply stress tests on the network layer by // sending large messages to transmit. @@ -537,27 +382,12 @@ func NetworkPayloadFixture(t *testing.T, size uint) []byte { return payload } -// NewResourceManager creates a new resource manager for testing with no limits. -func NewResourceManager(t *testing.T) p2pNetwork.ResourceManager { - return &p2pNetwork.NullResourceManager{} -} - -// NewConnectionGater creates a new connection gater for testing with given allow listing filter. -func NewConnectionGater(idProvider module.IdentityProvider, allowListFilter p2p.PeerFilter) connmgr.ConnectionGater { - filters := []p2p.PeerFilter{allowListFilter} - return connection.NewConnGater(unittest.Logger(), - idProvider, - connection.WithOnInterceptPeerDialFilters(filters), - connection.WithOnInterceptSecuredFilters(filters)) -} - // IsRateLimitedPeerFilter returns a p2p.PeerFilter that will return an error if the peer is rate limited. func IsRateLimitedPeerFilter(rateLimiter p2p.RateLimiter) p2p.PeerFilter { return func(p peer.ID) error { if rateLimiter.IsRateLimited(p) { return fmt.Errorf("peer is rate limited") } - return nil } } diff --git a/network/message/gossipsub.go b/network/message/gossipsub.go new file mode 100644 index 00000000000..ede1b09878e --- /dev/null +++ b/network/message/gossipsub.go @@ -0,0 +1 @@ +package message diff --git a/network/middleware.go b/network/middleware.go index 7bc600fbc8f..d8e14ee82c1 100644 --- a/network/middleware.go +++ b/network/middleware.go @@ -17,10 +17,14 @@ import ( // connections, as well as reading & writing to/from the connections. type Middleware interface { component.Component + DisallowListNotificationConsumer // SetOverlay sets the overlay used by the middleware. This must be called before the middleware can be Started. SetOverlay(Overlay) + // SetSlashingViolationsConsumer sets the slashing violations consumer. + SetSlashingViolationsConsumer(ViolationsConsumer) + // SendDirect sends msg on a 1-1 direct connection to the target ID. It models a guaranteed delivery asynchronous // direct one-to-one connection on the underlying network. No intermediate node on the overlay is utilized // as the router. diff --git a/network/mocknetwork/adapter.go b/network/mocknetwork/adapter.go index 6cf0775432d..2700f6eb0cc 100644 --- a/network/mocknetwork/adapter.go +++ b/network/mocknetwork/adapter.go @@ -7,6 +7,8 @@ import ( channels "github.com/onflow/flow-go/network/channels" mock "github.com/stretchr/testify/mock" + + network "github.com/onflow/flow-go/network" ) // Adapter is an autogenerated mock type for the Adapter type @@ -56,6 +58,11 @@ func (_m *Adapter) PublishOnChannel(_a0 channels.Channel, _a1 interface{}, _a2 . return r0 } +// ReportMisbehaviorOnChannel provides a mock function with given fields: channel, report +func (_m *Adapter) ReportMisbehaviorOnChannel(channel channels.Channel, report network.MisbehaviorReport) { + _m.Called(channel, report) +} + // UnRegisterChannel provides a mock function with given fields: channel func (_m *Adapter) UnRegisterChannel(channel channels.Channel) error { ret := _m.Called(channel) diff --git a/network/mocknetwork/conduit.go b/network/mocknetwork/conduit.go index 4d7504c3a6d..06bb0f9f5f2 100644 --- a/network/mocknetwork/conduit.go +++ b/network/mocknetwork/conduit.go @@ -5,6 +5,8 @@ package mocknetwork import ( flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" + + network "github.com/onflow/flow-go/network" ) // Conduit is an autogenerated mock type for the Conduit type @@ -68,6 +70,11 @@ func (_m *Conduit) Publish(event interface{}, targetIDs ...flow.Identifier) erro return r0 } +// ReportMisbehavior provides a mock function with given fields: _a0 +func (_m *Conduit) ReportMisbehavior(_a0 network.MisbehaviorReport) { + _m.Called(_a0) +} + // Unicast provides a mock function with given fields: event, targetID func (_m *Conduit) Unicast(event interface{}, targetID flow.Identifier) error { ret := _m.Called(event, targetID) diff --git a/network/mocknetwork/connector.go b/network/mocknetwork/connector.go index 7f6a50e317c..deedbd4f815 100644 --- a/network/mocknetwork/connector.go +++ b/network/mocknetwork/connector.go @@ -15,9 +15,9 @@ type Connector struct { mock.Mock } -// UpdatePeers provides a mock function with given fields: ctx, peerIDs -func (_m *Connector) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) { - _m.Called(ctx, peerIDs) +// Connect provides a mock function with given fields: ctx, peerChan +func (_m *Connector) Connect(ctx context.Context, peerChan <-chan peer.AddrInfo) { + _m.Called(ctx, peerChan) } type mockConstructorTestingTNewConnector interface { diff --git a/network/mocknetwork/connector_factory.go b/network/mocknetwork/connector_factory.go new file mode 100644 index 00000000000..b1baeb4f749 --- /dev/null +++ b/network/mocknetwork/connector_factory.go @@ -0,0 +1,56 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocknetwork + +import ( + host "github.com/libp2p/go-libp2p/core/host" + mock "github.com/stretchr/testify/mock" + + p2p "github.com/onflow/flow-go/network/p2p" +) + +// ConnectorFactory is an autogenerated mock type for the ConnectorFactory type +type ConnectorFactory struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0 +func (_m *ConnectorFactory) Execute(_a0 host.Host) (p2p.Connector, error) { + ret := _m.Called(_a0) + + var r0 p2p.Connector + var r1 error + if rf, ok := ret.Get(0).(func(host.Host) (p2p.Connector, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(host.Host) p2p.Connector); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(p2p.Connector) + } + } + + if rf, ok := ret.Get(1).(func(host.Host) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewConnectorFactory interface { + mock.TestingT + Cleanup(func()) +} + +// NewConnectorFactory creates a new instance of ConnectorFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnectorFactory(t mockConstructorTestingTNewConnectorFactory) *ConnectorFactory { + mock := &ConnectorFactory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/mocknetwork/connector_host.go b/network/mocknetwork/connector_host.go new file mode 100644 index 00000000000..e656391a11f --- /dev/null +++ b/network/mocknetwork/connector_host.go @@ -0,0 +1,116 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocknetwork + +import ( + network "github.com/libp2p/go-libp2p/core/network" + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// ConnectorHost is an autogenerated mock type for the ConnectorHost type +type ConnectorHost struct { + mock.Mock +} + +// ClosePeer provides a mock function with given fields: peerId +func (_m *ConnectorHost) ClosePeer(peerId peer.ID) error { + ret := _m.Called(peerId) + + var r0 error + if rf, ok := ret.Get(0).(func(peer.ID) error); ok { + r0 = rf(peerId) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Connections provides a mock function with given fields: +func (_m *ConnectorHost) Connections() []network.Conn { + ret := _m.Called() + + var r0 []network.Conn + if rf, ok := ret.Get(0).(func() []network.Conn); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]network.Conn) + } + } + + return r0 +} + +// ID provides a mock function with given fields: +func (_m *ConnectorHost) ID() peer.ID { + ret := _m.Called() + + var r0 peer.ID + if rf, ok := ret.Get(0).(func() peer.ID); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(peer.ID) + } + + return r0 +} + +// IsConnectedTo provides a mock function with given fields: peerId +func (_m *ConnectorHost) IsConnectedTo(peerId peer.ID) bool { + ret := _m.Called(peerId) + + var r0 bool + if rf, ok := ret.Get(0).(func(peer.ID) bool); ok { + r0 = rf(peerId) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// IsProtected provides a mock function with given fields: peerId +func (_m *ConnectorHost) IsProtected(peerId peer.ID) bool { + ret := _m.Called(peerId) + + var r0 bool + if rf, ok := ret.Get(0).(func(peer.ID) bool); ok { + r0 = rf(peerId) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// PeerInfo provides a mock function with given fields: peerId +func (_m *ConnectorHost) PeerInfo(peerId peer.ID) peer.AddrInfo { + ret := _m.Called(peerId) + + var r0 peer.AddrInfo + if rf, ok := ret.Get(0).(func(peer.ID) peer.AddrInfo); ok { + r0 = rf(peerId) + } else { + r0 = ret.Get(0).(peer.AddrInfo) + } + + return r0 +} + +type mockConstructorTestingTNewConnectorHost interface { + mock.TestingT + Cleanup(func()) +} + +// NewConnectorHost creates a new instance of ConnectorHost. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnectorHost(t mockConstructorTestingTNewConnectorHost) *ConnectorHost { + mock := &ConnectorHost{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/mocknetwork/disallow_list_notification_consumer.go b/network/mocknetwork/disallow_list_notification_consumer.go new file mode 100644 index 00000000000..802caddf023 --- /dev/null +++ b/network/mocknetwork/disallow_list_notification_consumer.go @@ -0,0 +1,38 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocknetwork + +import ( + network "github.com/onflow/flow-go/network" + mock "github.com/stretchr/testify/mock" +) + +// DisallowListNotificationConsumer is an autogenerated mock type for the DisallowListNotificationConsumer type +type DisallowListNotificationConsumer struct { + mock.Mock +} + +// OnAllowListNotification provides a mock function with given fields: _a0 +func (_m *DisallowListNotificationConsumer) OnAllowListNotification(_a0 *network.AllowListingUpdate) { + _m.Called(_a0) +} + +// OnDisallowListNotification provides a mock function with given fields: _a0 +func (_m *DisallowListNotificationConsumer) OnDisallowListNotification(_a0 *network.DisallowListingUpdate) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewDisallowListNotificationConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewDisallowListNotificationConsumer creates a new instance of DisallowListNotificationConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDisallowListNotificationConsumer(t mockConstructorTestingTNewDisallowListNotificationConsumer) *DisallowListNotificationConsumer { + mock := &DisallowListNotificationConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/mocknetwork/disallow_list_oracle.go b/network/mocknetwork/disallow_list_oracle.go new file mode 100644 index 00000000000..3bae6e851f3 --- /dev/null +++ b/network/mocknetwork/disallow_list_oracle.go @@ -0,0 +1,46 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocknetwork + +import ( + network "github.com/onflow/flow-go/network" + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// DisallowListOracle is an autogenerated mock type for the DisallowListOracle type +type DisallowListOracle struct { + mock.Mock +} + +// GetAllDisallowedListCausesFor provides a mock function with given fields: _a0 +func (_m *DisallowListOracle) GetAllDisallowedListCausesFor(_a0 peer.ID) []network.DisallowListedCause { + ret := _m.Called(_a0) + + var r0 []network.DisallowListedCause + if rf, ok := ret.Get(0).(func(peer.ID) []network.DisallowListedCause); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]network.DisallowListedCause) + } + } + + return r0 +} + +type mockConstructorTestingTNewDisallowListOracle interface { + mock.TestingT + Cleanup(func()) +} + +// NewDisallowListOracle creates a new instance of DisallowListOracle. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDisallowListOracle(t mockConstructorTestingTNewDisallowListOracle) *DisallowListOracle { + mock := &DisallowListOracle{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/mocknetwork/middleware.go b/network/mocknetwork/middleware.go index 457d8fd7360..18cdaed21b0 100644 --- a/network/mocknetwork/middleware.go +++ b/network/mocknetwork/middleware.go @@ -101,6 +101,16 @@ func (_m *Middleware) NewPingService(pingProtocol protocol.ID, provider network. return r0 } +// OnAllowListNotification provides a mock function with given fields: _a0 +func (_m *Middleware) OnAllowListNotification(_a0 *network.AllowListingUpdate) { + _m.Called(_a0) +} + +// OnDisallowListNotification provides a mock function with given fields: _a0 +func (_m *Middleware) OnDisallowListNotification(_a0 *network.DisallowListingUpdate) { + _m.Called(_a0) +} + // Publish provides a mock function with given fields: msg func (_m *Middleware) Publish(msg *network.OutgoingMessageScope) error { ret := _m.Called(msg) @@ -150,6 +160,11 @@ func (_m *Middleware) SetOverlay(_a0 network.Overlay) { _m.Called(_a0) } +// SetSlashingViolationsConsumer provides a mock function with given fields: _a0 +func (_m *Middleware) SetSlashingViolationsConsumer(_a0 network.ViolationsConsumer) { + _m.Called(_a0) +} + // Start provides a mock function with given fields: _a0 func (_m *Middleware) Start(_a0 irrecoverable.SignalerContext) { _m.Called(_a0) diff --git a/network/mocknetwork/misbehavior_report.go b/network/mocknetwork/misbehavior_report.go new file mode 100644 index 00000000000..150e24eadd7 --- /dev/null +++ b/network/mocknetwork/misbehavior_report.go @@ -0,0 +1,74 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocknetwork + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" + + network "github.com/onflow/flow-go/network" +) + +// MisbehaviorReport is an autogenerated mock type for the MisbehaviorReport type +type MisbehaviorReport struct { + mock.Mock +} + +// OriginId provides a mock function with given fields: +func (_m *MisbehaviorReport) OriginId() flow.Identifier { + ret := _m.Called() + + var r0 flow.Identifier + if rf, ok := ret.Get(0).(func() flow.Identifier); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.Identifier) + } + } + + return r0 +} + +// Penalty provides a mock function with given fields: +func (_m *MisbehaviorReport) Penalty() float64 { + ret := _m.Called() + + var r0 float64 + if rf, ok := ret.Get(0).(func() float64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(float64) + } + + return r0 +} + +// Reason provides a mock function with given fields: +func (_m *MisbehaviorReport) Reason() network.Misbehavior { + ret := _m.Called() + + var r0 network.Misbehavior + if rf, ok := ret.Get(0).(func() network.Misbehavior); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(network.Misbehavior) + } + + return r0 +} + +type mockConstructorTestingTNewMisbehaviorReport interface { + mock.TestingT + Cleanup(func()) +} + +// NewMisbehaviorReport creates a new instance of MisbehaviorReport. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMisbehaviorReport(t mockConstructorTestingTNewMisbehaviorReport) *MisbehaviorReport { + mock := &MisbehaviorReport{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/mocknetwork/misbehavior_report_consumer.go b/network/mocknetwork/misbehavior_report_consumer.go new file mode 100644 index 00000000000..8731a6ae8fe --- /dev/null +++ b/network/mocknetwork/misbehavior_report_consumer.go @@ -0,0 +1,35 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocknetwork + +import ( + channels "github.com/onflow/flow-go/network/channels" + mock "github.com/stretchr/testify/mock" + + network "github.com/onflow/flow-go/network" +) + +// MisbehaviorReportConsumer is an autogenerated mock type for the MisbehaviorReportConsumer type +type MisbehaviorReportConsumer struct { + mock.Mock +} + +// ReportMisbehaviorOnChannel provides a mock function with given fields: channel, report +func (_m *MisbehaviorReportConsumer) ReportMisbehaviorOnChannel(channel channels.Channel, report network.MisbehaviorReport) { + _m.Called(channel, report) +} + +type mockConstructorTestingTNewMisbehaviorReportConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewMisbehaviorReportConsumer creates a new instance of MisbehaviorReportConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMisbehaviorReportConsumer(t mockConstructorTestingTNewMisbehaviorReportConsumer) *MisbehaviorReportConsumer { + mock := &MisbehaviorReportConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/mocknetwork/misbehavior_report_manager.go b/network/mocknetwork/misbehavior_report_manager.go new file mode 100644 index 00000000000..93ee2dfc6de --- /dev/null +++ b/network/mocknetwork/misbehavior_report_manager.go @@ -0,0 +1,74 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocknetwork + +import ( + irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + channels "github.com/onflow/flow-go/network/channels" + + mock "github.com/stretchr/testify/mock" + + network "github.com/onflow/flow-go/network" +) + +// MisbehaviorReportManager is an autogenerated mock type for the MisbehaviorReportManager type +type MisbehaviorReportManager struct { + mock.Mock +} + +// Done provides a mock function with given fields: +func (_m *MisbehaviorReportManager) Done() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// HandleMisbehaviorReport provides a mock function with given fields: _a0, _a1 +func (_m *MisbehaviorReportManager) HandleMisbehaviorReport(_a0 channels.Channel, _a1 network.MisbehaviorReport) { + _m.Called(_a0, _a1) +} + +// Ready provides a mock function with given fields: +func (_m *MisbehaviorReportManager) Ready() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// Start provides a mock function with given fields: _a0 +func (_m *MisbehaviorReportManager) Start(_a0 irrecoverable.SignalerContext) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewMisbehaviorReportManager interface { + mock.TestingT + Cleanup(func()) +} + +// NewMisbehaviorReportManager creates a new instance of MisbehaviorReportManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMisbehaviorReportManager(t mockConstructorTestingTNewMisbehaviorReportManager) *MisbehaviorReportManager { + mock := &MisbehaviorReportManager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/mocknetwork/misbehavior_reporter.go b/network/mocknetwork/misbehavior_reporter.go new file mode 100644 index 00000000000..101d7e32f90 --- /dev/null +++ b/network/mocknetwork/misbehavior_reporter.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocknetwork + +import ( + network "github.com/onflow/flow-go/network" + mock "github.com/stretchr/testify/mock" +) + +// MisbehaviorReporter is an autogenerated mock type for the MisbehaviorReporter type +type MisbehaviorReporter struct { + mock.Mock +} + +// ReportMisbehavior provides a mock function with given fields: _a0 +func (_m *MisbehaviorReporter) ReportMisbehavior(_a0 network.MisbehaviorReport) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewMisbehaviorReporter interface { + mock.TestingT + Cleanup(func()) +} + +// NewMisbehaviorReporter creates a new instance of MisbehaviorReporter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMisbehaviorReporter(t mockConstructorTestingTNewMisbehaviorReporter) *MisbehaviorReporter { + mock := &MisbehaviorReporter{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/mocknetwork/violations_consumer.go b/network/mocknetwork/violations_consumer.go index 9c6f252b095..2af1bf2b80f 100644 --- a/network/mocknetwork/violations_consumer.go +++ b/network/mocknetwork/violations_consumer.go @@ -3,7 +3,7 @@ package mocknetwork import ( - slashing "github.com/onflow/flow-go/network/slashing" + network "github.com/onflow/flow-go/network" mock "github.com/stretchr/testify/mock" ) @@ -13,32 +13,37 @@ type ViolationsConsumer struct { } // OnInvalidMsgError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnInvalidMsgError(violation *slashing.Violation) { +func (_m *ViolationsConsumer) OnInvalidMsgError(violation *network.Violation) { _m.Called(violation) } // OnSenderEjectedError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnSenderEjectedError(violation *slashing.Violation) { +func (_m *ViolationsConsumer) OnSenderEjectedError(violation *network.Violation) { _m.Called(violation) } // OnUnAuthorizedSenderError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnAuthorizedSenderError(violation *slashing.Violation) { +func (_m *ViolationsConsumer) OnUnAuthorizedSenderError(violation *network.Violation) { + _m.Called(violation) +} + +// OnUnauthorizedPublishOnChannel provides a mock function with given fields: violation +func (_m *ViolationsConsumer) OnUnauthorizedPublishOnChannel(violation *network.Violation) { _m.Called(violation) } // OnUnauthorizedUnicastOnChannel provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnauthorizedUnicastOnChannel(violation *slashing.Violation) { +func (_m *ViolationsConsumer) OnUnauthorizedUnicastOnChannel(violation *network.Violation) { _m.Called(violation) } // OnUnexpectedError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnexpectedError(violation *slashing.Violation) { +func (_m *ViolationsConsumer) OnUnexpectedError(violation *network.Violation) { _m.Called(violation) } // OnUnknownMsgTypeError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnknownMsgTypeError(violation *slashing.Violation) { +func (_m *ViolationsConsumer) OnUnknownMsgTypeError(violation *network.Violation) { _m.Called(violation) } diff --git a/network/netconf/config.go b/network/netconf/config.go new file mode 100644 index 00000000000..f3bcfed1f93 --- /dev/null +++ b/network/netconf/config.go @@ -0,0 +1,70 @@ +package netconf + +import ( + "time" + + "github.com/onflow/flow-go/network/p2p/p2pconf" +) + +// Config encapsulation of configuration structs for all components related to the Flow network. +type Config struct { + // UnicastRateLimitersConfig configuration for all unicast rate limiters. + UnicastRateLimitersConfig `mapstructure:",squash"` + p2pconf.ResourceManagerConfig `mapstructure:",squash"` + ConnectionManagerConfig `mapstructure:",squash"` + // GossipSubConfig core gossipsub configuration. + p2pconf.GossipSubConfig `mapstructure:",squash"` + AlspConfig `mapstructure:",squash"` + + // NetworkConnectionPruning determines whether connections to nodes + // that are not part of protocol state should be trimmed + // TODO: solely a fallback mechanism, can be removed upon reliable behavior in production. + NetworkConnectionPruning bool `mapstructure:"networking-connection-pruning"` + // PreferredUnicastProtocols list of unicast protocols in preferred order + PreferredUnicastProtocols []string `mapstructure:"preferred-unicast-protocols"` + NetworkReceivedMessageCacheSize uint32 `validate:"gt=0" mapstructure:"received-message-cache-size"` + PeerUpdateInterval time.Duration `validate:"gt=0s" mapstructure:"peerupdate-interval"` + UnicastMessageTimeout time.Duration `validate:"gt=0s" mapstructure:"unicast-message-timeout"` + // UnicastCreateStreamRetryDelay initial delay used in the exponential backoff for create stream retries + UnicastCreateStreamRetryDelay time.Duration `validate:"gt=0s" mapstructure:"unicast-create-stream-retry-delay"` + DNSCacheTTL time.Duration `validate:"gt=0s" mapstructure:"dns-cache-ttl"` + // DisallowListNotificationCacheSize size of the queue for notifications about new peers in the disallow list. + DisallowListNotificationCacheSize uint32 `validate:"gt=0" mapstructure:"disallow-list-notification-cache-size"` +} + +// UnicastRateLimitersConfig unicast rate limiter configuration for the message and bandwidth rate limiters. +type UnicastRateLimitersConfig struct { + // DryRun setting this to true will disable connection disconnects and gating when unicast rate limiters are configured + DryRun bool `mapstructure:"unicast-dry-run"` + // LockoutDuration the number of seconds a peer will be forced to wait before being allowed to successfully reconnect to the node + // after being rate limited. + LockoutDuration time.Duration `validate:"gte=0" mapstructure:"unicast-lockout-duration"` + // MessageRateLimit amount of unicast messages that can be sent by a peer per second. + MessageRateLimit int `validate:"gte=0" mapstructure:"unicast-message-rate-limit"` + // BandwidthRateLimit bandwidth size in bytes a peer is allowed to send via unicast streams per second. + BandwidthRateLimit int `validate:"gte=0" mapstructure:"unicast-bandwidth-rate-limit"` + // BandwidthBurstLimit bandwidth size in bytes a peer is allowed to send via unicast streams at once. + BandwidthBurstLimit int `validate:"gte=0" mapstructure:"unicast-bandwidth-burst-limit"` +} + +// AlspConfig is the config for the Application Layer Spam Prevention (ALSP) protocol. +type AlspConfig struct { + // Size of the cache for spam records. There is at most one spam record per authorized (i.e., staked) node. + // Recommended size is 10 * number of authorized nodes to allow for churn. + SpamRecordCacheSize uint32 `mapstructure:"alsp-spam-record-cache-size"` + + // SpamReportQueueSize is the size of the queue for spam records. The queue is used to store spam records + // temporarily till they are picked by the workers. When the queue is full, new spam records are dropped. + // Recommended size is 100 * number of authorized nodes to allow for churn. + SpamReportQueueSize uint32 `mapstructure:"alsp-spam-report-queue-size"` + + // DisablePenalty indicates whether applying the penalty to the misbehaving node is disabled. + // When disabled, the ALSP module logs the misbehavior reports and updates the metrics, but does not apply the penalty. + // This is useful for managing production incidents. + // Note: under normal circumstances, the ALSP module should not be disabled. + DisablePenalty bool `mapstructure:"alsp-disable-penalty"` + + // HeartBeatInterval is the interval between heartbeats sent by the ALSP module. The heartbeats are recurring + // events that are used to perform critical ALSP tasks, such as updating the spam records cache. + HearBeatInterval time.Duration `mapstructure:"alsp-heart-beat-interval"` +} diff --git a/network/netconf/config_test.go b/network/netconf/config_test.go new file mode 100644 index 00000000000..39578f68a5c --- /dev/null +++ b/network/netconf/config_test.go @@ -0,0 +1,40 @@ +package netconf + +import ( + "fmt" + "strings" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +// TestSetAliases ensures every network configuration key prefixed with "network" has an alias without the "network" prefix. +func TestSetAliases(t *testing.T) { + c := viper.New() + for _, key := range AllFlagNames() { + c.Set(fmt.Sprintf("network.%s", key), "not aliased") + c.Set(key, "aliased") + } + + // ensure network prefixed keys do not point to non-prefixed alias + for _, key := range c.AllKeys() { + parts := strings.Split(key, ".") + if len(parts) != 2 { + continue + } + require.NotEqual(t, c.GetString(parts[1]), c.GetString(key)) + } + + err := SetAliases(c) + require.NoError(t, err) + + // ensure each network prefixed key now points to the non-prefixed alias + for _, key := range c.AllKeys() { + parts := strings.Split(key, ".") + if len(parts) != 2 { + continue + } + require.Equal(t, c.GetString(parts[1]), c.GetString(key)) + } +} diff --git a/network/netconf/connection_manager.go b/network/netconf/connection_manager.go new file mode 100644 index 00000000000..333a3f8c6e5 --- /dev/null +++ b/network/netconf/connection_manager.go @@ -0,0 +1,25 @@ +package netconf + +import "time" + +type ConnectionManagerConfig struct { + // HighWatermark and LowWatermark govern the number of connections are maintained by the ConnManager. + // When the peer count exceeds the HighWatermark, as many peers will be pruned (and + // their connections terminated) until LowWatermark peers remain. In other words, whenever the + // peer count is x > HighWatermark, the ConnManager will prune x - LowWatermark peers. + // The pruning algorithm is as follows: + // 1. The ConnManager will not prune any peers that have been connected for less than GracePeriod. + // 2. The ConnManager will not prune any peers that are protected. + // 3. The ConnManager will sort the peers based on their number of streams and direction of connections, and + // prunes the peers with the least number of streams. If there are ties, the peer with the incoming connection + // will be pruned. If both peers have incoming connections, and there are still ties, one of the peers will be + // pruned at random. + // Algorithm implementation is in https://github.com/libp2p/go-libp2p/blob/master/p2p/net/connmgr/connmgr.go#L262-L318 + HighWatermark int `mapstructure:"libp2p-high-watermark"` // naming from libp2p + LowWatermark int `mapstructure:"libp2p-low-watermark"` // naming from libp2p + + // SilencePeriod is the time to wait before start pruning connections. + SilencePeriod time.Duration `mapstructure:"libp2p-silence-period"` // naming from libp2p + // GracePeriod is the time to wait before pruning a new connection. + GracePeriod time.Duration `mapstructure:"libp2p-grace-period"` // naming from libp2p +} diff --git a/network/netconf/flags.go b/network/netconf/flags.go new file mode 100644 index 00000000000..045755fde7b --- /dev/null +++ b/network/netconf/flags.go @@ -0,0 +1,188 @@ +package netconf + +import ( + "fmt" + "strings" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + + p2pmsg "github.com/onflow/flow-go/network/p2p/message" +) + +const ( + // All constant strings are used for CLI flag names and corresponding keys for config values. + // network configuration + networkingConnectionPruning = "networking-connection-pruning" + preferredUnicastsProtocols = "preferred-unicast-protocols" + receivedMessageCacheSize = "received-message-cache-size" + peerUpdateInterval = "peerupdate-interval" + unicastMessageTimeout = "unicast-message-timeout" + unicastCreateStreamRetryDelay = "unicast-create-stream-retry-delay" + dnsCacheTTL = "dns-cache-ttl" + disallowListNotificationCacheSize = "disallow-list-notification-cache-size" + // unicast rate limiters config + dryRun = "unicast-dry-run" + lockoutDuration = "unicast-lockout-duration" + messageRateLimit = "unicast-message-rate-limit" + bandwidthRateLimit = "unicast-bandwidth-rate-limit" + bandwidthBurstLimit = "unicast-bandwidth-burst-limit" + // resource manager config + memoryLimitRatio = "libp2p-memory-limit-ratio" + fileDescriptorsRatio = "libp2p-file-descriptors-ratio" + peerBaseLimitConnsInbound = "libp2p-peer-base-limits-conns-inbound" + // connection manager + highWatermark = "libp2p-high-watermark" + lowWatermark = "libp2p-low-watermark" + gracePeriod = "libp2p-grace-period" + silencePeriod = "libp2p-silence-period" + // gossipsub + peerScoring = "gossipsub-peer-scoring-enabled" + localMeshLogInterval = "gossipsub-local-mesh-logging-interval" + rpcSentTrackerCacheSize = "gossipsub-rpc-sent-tracker-cache-size" + rpcSentTrackerQueueCacheSize = "gossipsub-rpc-sent-tracker-queue-cache-size" + rpcSentTrackerNumOfWorkers = "gossipsub-rpc-sent-tracker-workers" + scoreTracerInterval = "gossipsub-score-tracer-interval" + // gossipsub validation inspector + gossipSubRPCInspectorNotificationCacheSize = "gossipsub-rpc-inspector-notification-cache-size" + validationInspectorNumberOfWorkers = "gossipsub-rpc-validation-inspector-workers" + validationInspectorInspectMessageQueueCacheSize = "gossipsub-rpc-validation-inspector-queue-cache-size" + validationInspectorClusterPrefixedTopicsReceivedCacheSize = "gossipsub-cluster-prefix-tracker-cache-size" + validationInspectorClusterPrefixedTopicsReceivedCacheDecay = "gossipsub-cluster-prefix-tracker-cache-decay" + validationInspectorClusterPrefixHardThreshold = "gossipsub-rpc-cluster-prefixed-hard-threshold" + + ihaveSyncSampleSizePercentage = "ihave-sync-inspection-sample-size-percentage" + ihaveAsyncSampleSizePercentage = "ihave-async-inspection-sample-size-percentage" + ihaveMaxSampleSize = "ihave-max-sample-size" + + // gossipsub metrics inspector + metricsInspectorNumberOfWorkers = "gossipsub-rpc-metrics-inspector-workers" + metricsInspectorCacheSize = "gossipsub-rpc-metrics-inspector-cache-size" + + alspDisabled = "alsp-disable-penalty" + alspSpamRecordCacheSize = "alsp-spam-record-cache-size" + alspSpamRecordQueueSize = "alsp-spam-report-queue-size" + alspHearBeatInterval = "alsp-heart-beat-interval" +) + +func AllFlagNames() []string { + return []string{ + networkingConnectionPruning, preferredUnicastsProtocols, receivedMessageCacheSize, peerUpdateInterval, unicastMessageTimeout, unicastCreateStreamRetryDelay, + dnsCacheTTL, disallowListNotificationCacheSize, dryRun, lockoutDuration, messageRateLimit, bandwidthRateLimit, bandwidthBurstLimit, memoryLimitRatio, + fileDescriptorsRatio, peerBaseLimitConnsInbound, highWatermark, lowWatermark, gracePeriod, silencePeriod, peerScoring, localMeshLogInterval, rpcSentTrackerCacheSize, rpcSentTrackerQueueCacheSize, rpcSentTrackerNumOfWorkers, + scoreTracerInterval, gossipSubRPCInspectorNotificationCacheSize, validationInspectorNumberOfWorkers, validationInspectorInspectMessageQueueCacheSize, validationInspectorClusterPrefixedTopicsReceivedCacheSize, + validationInspectorClusterPrefixedTopicsReceivedCacheDecay, validationInspectorClusterPrefixHardThreshold, ihaveSyncSampleSizePercentage, ihaveAsyncSampleSizePercentage, + ihaveMaxSampleSize, metricsInspectorNumberOfWorkers, metricsInspectorCacheSize, alspDisabled, alspSpamRecordCacheSize, alspSpamRecordQueueSize, alspHearBeatInterval, + } +} + +// InitializeNetworkFlags initializes all CLI flags for the Flow network configuration on the provided pflag set. +// Args: +// +// *pflag.FlagSet: the pflag set of the Flow node. +// *Config: the default network config used to set default values on the flags +func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { + initRpcInspectorValidationLimitsFlags(flags, config) + flags.Bool(networkingConnectionPruning, config.NetworkConnectionPruning, "enabling connection trimming") + flags.Duration(dnsCacheTTL, config.DNSCacheTTL, "time-to-live for dns cache") + flags.StringSlice(preferredUnicastsProtocols, config.PreferredUnicastProtocols, "preferred unicast protocols in ascending order of preference") + flags.Uint32(receivedMessageCacheSize, config.NetworkReceivedMessageCacheSize, "incoming message cache size at networking layer") + flags.Uint32(disallowListNotificationCacheSize, config.DisallowListNotificationCacheSize, "cache size for notification events from disallow list") + flags.Duration(peerUpdateInterval, config.PeerUpdateInterval, "how often to refresh the peer connections for the node") + flags.Duration(unicastMessageTimeout, config.UnicastMessageTimeout, "how long a unicast transmission can take to complete") + // unicast manager options + flags.Duration(unicastCreateStreamRetryDelay, config.UnicastCreateStreamRetryDelay, "Initial delay between failing to establish a connection with another node and retrying. This delay increases exponentially (exponential backoff) with the number of subsequent failures to establish a connection.") + // unicast stream handler rate limits + flags.Int(messageRateLimit, config.UnicastRateLimitersConfig.MessageRateLimit, "maximum number of unicast messages that a peer can send per second") + flags.Int(bandwidthRateLimit, config.UnicastRateLimitersConfig.BandwidthRateLimit, "bandwidth size in bytes a peer is allowed to send via unicast streams per second") + flags.Int(bandwidthBurstLimit, config.UnicastRateLimitersConfig.BandwidthBurstLimit, "bandwidth size in bytes a peer is allowed to send at one time") + flags.Duration(lockoutDuration, config.UnicastRateLimitersConfig.LockoutDuration, "the number of seconds a peer will be forced to wait before being allowed to successful reconnect to the node after being rate limited") + flags.Bool(dryRun, config.UnicastRateLimitersConfig.DryRun, "disable peer disconnects and connections gating when rate limiting peers") + // resource manager cli flags + flags.Float64(fileDescriptorsRatio, config.ResourceManagerConfig.FileDescriptorsRatio, "ratio of available file descriptors to be used by libp2p (in (0,1])") + flags.Float64(memoryLimitRatio, config.ResourceManagerConfig.MemoryLimitRatio, "ratio of available memory to be used by libp2p (in (0,1])") + flags.Int(peerBaseLimitConnsInbound, config.ResourceManagerConfig.PeerBaseLimitConnsInbound, "the maximum amount of allowed inbound connections per peer") + // connection manager + flags.Int(lowWatermark, config.ConnectionManagerConfig.LowWatermark, "low watermarking for libp2p connection manager") + flags.Int(highWatermark, config.ConnectionManagerConfig.HighWatermark, "high watermarking for libp2p connection manager") + flags.Duration(gracePeriod, config.ConnectionManagerConfig.GracePeriod, "grace period for libp2p connection manager") + flags.Duration(silencePeriod, config.ConnectionManagerConfig.SilencePeriod, "silence period for libp2p connection manager") + flags.Bool(peerScoring, config.GossipSubConfig.PeerScoring, "enabling peer scoring on pubsub network") + flags.Duration(localMeshLogInterval, config.GossipSubConfig.LocalMeshLogInterval, "logging interval for local mesh in gossipsub") + flags.Duration(scoreTracerInterval, config.GossipSubConfig.ScoreTracerInterval, "logging interval for peer score tracer in gossipsub, set to 0 to disable") + flags.Uint32(rpcSentTrackerCacheSize, config.GossipSubConfig.RPCSentTrackerCacheSize, "cache size of the rpc sent tracker used by the gossipsub mesh tracer.") + flags.Uint32(rpcSentTrackerQueueCacheSize, config.GossipSubConfig.RPCSentTrackerQueueCacheSize, "cache size of the rpc sent tracker worker queue.") + flags.Int(rpcSentTrackerNumOfWorkers, config.GossipSubConfig.RpcSentTrackerNumOfWorkers, "number of workers for the rpc sent tracker worker pool.") + // gossipsub RPC control message validation limits used for validation configuration and rate limiting + flags.Int(validationInspectorNumberOfWorkers, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.NumberOfWorkers, "number of gossupsub RPC control message validation inspector component workers") + flags.Uint32(validationInspectorInspectMessageQueueCacheSize, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.CacheSize, "cache size for gossipsub RPC validation inspector events worker pool queue.") + flags.Uint32(validationInspectorClusterPrefixedTopicsReceivedCacheSize, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.ClusterPrefixedControlMsgsReceivedCacheSize, "cache size for gossipsub RPC validation inspector cluster prefix received tracker.") + flags.Float64(validationInspectorClusterPrefixedTopicsReceivedCacheDecay, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.ClusterPrefixedControlMsgsReceivedCacheDecay, "the decay value used to decay cluster prefix received topics received cached counters.") + flags.Float64(validationInspectorClusterPrefixHardThreshold, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.ClusterPrefixHardThreshold, "the maximum number of cluster-prefixed control messages allowed to be processed when the active cluster id is unset or a mismatch is detected, exceeding this threshold will result in node penalization by gossipsub.") + // gossipsub RPC control message metrics observer inspector configuration + flags.Int(metricsInspectorNumberOfWorkers, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCMetricsInspectorConfigs.NumberOfWorkers, "cache size for gossipsub RPC metrics inspector events worker pool queue.") + flags.Uint32(metricsInspectorCacheSize, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCMetricsInspectorConfigs.CacheSize, "cache size for gossipsub RPC metrics inspector events worker pool.") + // networking event notifications + flags.Uint32(gossipSubRPCInspectorNotificationCacheSize, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCInspectorNotificationCacheSize, "cache size for notification events from gossipsub rpc inspector") + // application layer spam prevention (alsp) protocol + flags.Bool(alspDisabled, config.AlspConfig.DisablePenalty, "disable the penalty mechanism of the alsp protocol. default value (recommended) is false") + flags.Uint32(alspSpamRecordCacheSize, config.AlspConfig.SpamRecordCacheSize, "size of spam record cache, recommended to be 10x the number of authorized nodes") + flags.Uint32(alspSpamRecordQueueSize, config.AlspConfig.SpamReportQueueSize, "size of spam report queue, recommended to be 100x the number of authorized nodes") + flags.Duration(alspHearBeatInterval, config.AlspConfig.HearBeatInterval, "interval between two consecutive heartbeat events at alsp, recommended to leave it as default unless you know what you are doing.") + + flags.Float64(ihaveSyncSampleSizePercentage, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.IHaveSyncInspectSampleSizePercentage, "percentage of ihave messages to sample during synchronous validation") + flags.Float64(ihaveAsyncSampleSizePercentage, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.IHaveAsyncInspectSampleSizePercentage, "percentage of ihave messages to sample during asynchronous validation") + flags.Float64(ihaveMaxSampleSize, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.IHaveInspectionMaxSampleSize, "max number of ihaves to sample when performing validation") +} + +// rpcInspectorValidationLimits utility func that adds flags for each of the validation limits for each control message type. +func initRpcInspectorValidationLimitsFlags(flags *pflag.FlagSet, defaultNetConfig *Config) { + hardThresholdflagStrFmt := "gossipsub-rpc-%s-hard-threshold" + safetyThresholdflagStrFmt := "gossipsub-rpc-%s-safety-threshold" + rateLimitflagStrFmt := "gossipsub-rpc-%s-rate-limit" + validationInspectorConfig := defaultNetConfig.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs + + for _, ctrlMsgValidationConfig := range validationInspectorConfig.AllCtrlMsgValidationConfig() { + ctrlMsgType := ctrlMsgValidationConfig.ControlMsg + if ctrlMsgValidationConfig.ControlMsg == p2pmsg.CtrlMsgIWant { + continue + } + s := strings.ToLower(ctrlMsgType.String()) + flags.Uint64(fmt.Sprintf(hardThresholdflagStrFmt, s), ctrlMsgValidationConfig.HardThreshold, fmt.Sprintf("discard threshold limit for gossipsub RPC %s message validation", ctrlMsgType)) + flags.Uint64(fmt.Sprintf(safetyThresholdflagStrFmt, s), ctrlMsgValidationConfig.SafetyThreshold, fmt.Sprintf("safety threshold limit for gossipsub RPC %s message validation", ctrlMsgType)) + flags.Int(fmt.Sprintf(rateLimitflagStrFmt, s), ctrlMsgValidationConfig.RateLimit, fmt.Sprintf("rate limit for gossipsub RPC %s message validation", ctrlMsgType)) + } +} + +// SetAliases this func sets an aliases for each CLI flag defined for network config overrides to it's corresponding +// full key in the viper config store. This is required because in our config.yml file all configuration values for the +// Flow network are stored one level down on the network-config property. When the default config is bootstrapped viper will +// store these values with the "network-config." prefix on the config key, because we do not want to use CLI flags like --network-config.networking-connection-pruning +// to override default values we instead use cleans flags like --networking-connection-pruning and create an alias from networking-connection-pruning -> network-config.networking-connection-pruning +// to ensure overrides happen as expected. +// Args: +// *viper.Viper: instance of the viper store to register network config aliases on. +// Returns: +// error: if a flag does not have a corresponding key in the viper store. +func SetAliases(conf *viper.Viper) error { + m := make(map[string]string) + // create map of key -> full pathkey + // ie: "networking-connection-pruning" -> "network-config.networking-connection-pruning" + for _, key := range conf.AllKeys() { + s := strings.Split(key, ".") + // check len of s, we expect all network keys to have a single prefix "network-config" + // s should always contain only 2 elements + if len(s) == 2 { + m[s[1]] = key + } + } + // each flag name should correspond to exactly one key in our config store after it is loaded with the default config + for _, flagName := range AllFlagNames() { + fullKey, ok := m[flagName] + if !ok { + return fmt.Errorf("invalid network configuration missing configuration key flag name %s check config file and cli flags", flagName) + } + conf.RegisterAlias(fullKey, flagName) + } + return nil +} diff --git a/network/network.go b/network/network.go index 50c84887b72..38896633e4d 100644 --- a/network/network.go +++ b/network/network.go @@ -9,6 +9,30 @@ import ( "github.com/onflow/flow-go/network/channels" ) +// NetworkingType is the type of the Flow networking layer. It is used to differentiate between the public (i.e., unstaked) +// and private (i.e., staked) networks. +type NetworkingType uint8 + +func (t NetworkingType) String() string { + switch t { + case PrivateNetwork: + return "private" + case PublicNetwork: + return "public" + default: + return "unknown" + } +} + +const ( + // PrivateNetwork indicates that the staked private-side of the Flow blockchain that nodes can only join and leave + // with a staking requirement. + PrivateNetwork NetworkingType = iota + 1 + // PublicNetwork indicates that the unstaked public-side of the Flow blockchain that nodes can join and leave at will + // with no staking requirement. + PublicNetwork +) + // Network represents the network layer of the node. It allows processes that // work across the peer-to-peer network to register themselves as an engine with // a unique engine ID. The returned conduit allows the process to communicate to @@ -34,6 +58,7 @@ type Network interface { // Adapter is meant to be utilized by the Conduit interface to send messages to the Network layer to be // delivered to the remote targets. type Adapter interface { + MisbehaviorReportConsumer // UnicastOnChannel sends the message in a reliable way to the given recipient. UnicastOnChannel(channels.Channel, interface{}, flow.Identifier) error @@ -48,3 +73,16 @@ type Adapter interface { // receive messages from that channel. UnRegisterChannel(channel channels.Channel) error } + +// MisbehaviorReportConsumer set of funcs used to handle MisbehaviorReport disseminated from misbehavior reporters. +type MisbehaviorReportConsumer interface { + // ReportMisbehaviorOnChannel reports the misbehavior of a node on sending a message to the current node that appears + // valid based on the networking layer but is considered invalid by the current node based on the Flow protocol. + // The misbehavior report is sent to the current node's networking layer on the given channel to be processed. + // Args: + // - channel: The channel on which the misbehavior report is sent. + // - report: The misbehavior report to be sent. + // Returns: + // none + ReportMisbehaviorOnChannel(channel channels.Channel, report MisbehaviorReport) +} diff --git a/network/p2p/builder.go b/network/p2p/builder.go index ac1d2aeb978..b856f931a29 100644 --- a/network/p2p/builder.go +++ b/network/p2p/builder.go @@ -13,20 +13,21 @@ import ( madns "github.com/multiformats/go-multiaddr-dns" "github.com/rs/zerolog" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + flownet "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p/p2pconf" ) -// LibP2PFactoryFunc is a factory function type for generating libp2p Node instances. -type LibP2PFactoryFunc func() (LibP2PNode, error) -type GossipSubFactoryFunc func(context.Context, zerolog.Logger, host.Host, PubSubAdapterConfig) (PubSubAdapter, error) -type CreateNodeFunc func(zerolog.Logger, host.Host, ProtocolPeerCache, PeerManager) LibP2PNode +type GossipSubFactoryFunc func(context.Context, zerolog.Logger, host.Host, PubSubAdapterConfig, CollectionClusterChangesConsumer) (PubSubAdapter, error) +type CreateNodeFunc func(zerolog.Logger, host.Host, ProtocolPeerCache, PeerManager, *DisallowListCacheConfig) LibP2PNode type GossipSubAdapterConfigFunc func(*BasePubSubAdapterConfig) PubSubAdapterConfig // GossipSubBuilder provides a builder pattern for creating a GossipSub pubsub system. type GossipSubBuilder interface { - PeerScoringBuilder // SetHost sets the host of the builder. // If the host has already been set, a fatal error is logged. SetHost(host.Host) @@ -43,9 +44,16 @@ type GossipSubBuilder interface { // We expect the node to initialize with a default gossipsub config. Hence, this function overrides the default config. SetGossipSubConfigFunc(GossipSubAdapterConfigFunc) - // SetGossipSubPeerScoring sets the gossipsub peer scoring of the builder. - // If the gossipsub peer scoring flag has already been set, a fatal error is logged. - SetGossipSubPeerScoring(bool) + // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. + // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. + // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. + // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. + // Production Tip: use PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. + // Args: + // - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use PeerScoringConfigNoOverride for production. + // Returns: + // none + EnableGossipSubScoringWithOverride(*PeerScoringConfigOverride) // SetGossipSubScoreTracerInterval sets the gossipsub score tracer interval of the builder. // If the gossipsub score tracer interval has already been set, a fatal error is logged. @@ -55,16 +63,16 @@ type GossipSubBuilder interface { // If the gossipsub tracer has already been set, a fatal error is logged. SetGossipSubTracer(PubSubTracer) - // SetIDProvider sets the identity provider of the builder. - // If the identity provider has already been set, a fatal error is logged. - SetIDProvider(module.IdentityProvider) - // SetRoutingSystem sets the routing system of the builder. // If the routing system has already been set, a fatal error is logged. SetRoutingSystem(routing.Routing) - // SetGossipSubRPCInspectors sets the gossipsub rpc inspectors. - SetGossipSubRPCInspectors(inspectors ...GossipSubRPCInspector) + // OverrideDefaultRpcInspectorSuiteFactory overrides the default RPC inspector suite factory of the builder. + // A default RPC inspector suite factory is provided by the node. This function overrides the default factory. + // The purpose of override is to allow the node to provide a custom RPC inspector suite factory for sake of testing + // or experimentation. + // It is NOT recommended to override the default RPC inspector suite factory in production unless you know what you are doing. + OverrideDefaultRpcInspectorSuiteFactory(GossipSubRpcInspectorSuiteFactoryFunc) // Build creates a new GossipSub pubsub system. // It returns the newly created GossipSub pubsub system and any errors encountered during its creation. @@ -74,21 +82,33 @@ type GossipSubBuilder interface { // // Returns: // - PubSubAdapter: a GossipSub pubsub system for the libp2p node. - // - PeerScoreTracer: a peer score tracer for the GossipSub pubsub system (if enabled, otherwise nil). // - error: if an error occurs during the creation of the GossipSub pubsub system, it is returned. Otherwise, nil is returned. // Note that on happy path, the returned error is nil. Any error returned is unexpected and should be handled as irrecoverable. - Build(irrecoverable.SignalerContext) (PubSubAdapter, PeerScoreTracer, error) + Build(irrecoverable.SignalerContext) (PubSubAdapter, error) } -type PeerScoringBuilder interface { - // SetTopicScoreParams sets the topic score parameters for the given topic. - // If the topic score parameters have already been set for the given topic, it is overwritten. - SetTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) - - // SetAppSpecificScoreParams sets the application specific score parameters for the given topic. - // If the application specific score parameters have already been set for the given topic, it is overwritten. - SetAppSpecificScoreParams(func(peer.ID) float64) -} +// GossipSubRpcInspectorSuiteFactoryFunc is a function that creates a new RPC inspector suite. It is used to create +// RPC inspectors for the gossipsub protocol. The RPC inspectors are used to inspect and validate +// incoming RPC messages before they are processed by the gossipsub protocol. +// Args: +// - logger: logger to use +// - sporkID: spork ID of the node +// - cfg: configuration for the RPC inspectors +// - metrics: metrics to use for the RPC inspectors +// - heroCacheMetricsFactory: metrics factory for the hero cache +// - networkingType: networking type of the node, i.e., public or private +// - identityProvider: identity provider of the node +// Returns: +// - p2p.GossipSubInspectorSuite: new RPC inspector suite +// - error: error if any, any returned error is irrecoverable. +type GossipSubRpcInspectorSuiteFactoryFunc func( + zerolog.Logger, + flow.Identifier, + *p2pconf.GossipSubRPCInspectorsConfig, + module.GossipSubMetrics, + metrics.HeroCacheMetricsFactory, + flownet.NetworkingType, + module.IdentityProvider) (GossipSubInspectorSuite, error) // NodeBuilder is a builder pattern for creating a libp2p Node instance. type NodeBuilder interface { @@ -96,30 +116,54 @@ type NodeBuilder interface { SetSubscriptionFilter(pubsub.SubscriptionFilter) NodeBuilder SetResourceManager(network.ResourceManager) NodeBuilder SetConnectionManager(connmgr.ConnManager) NodeBuilder - SetConnectionGater(connmgr.ConnectionGater) NodeBuilder + SetConnectionGater(ConnectionGater) NodeBuilder SetRoutingSystem(func(context.Context, host.Host) (routing.Routing, error)) NodeBuilder - SetPeerManagerOptions(bool, time.Duration) NodeBuilder - // EnableGossipSubPeerScoring enables peer scoring for the GossipSub pubsub system. - // Arguments: - // - module.IdentityProvider: the identity provider for the node (must be set before calling this method). - // - *PeerScoringConfig: the peer scoring configuration for the GossipSub pubsub system. If nil, the default configuration is used. - EnableGossipSubPeerScoring(module.IdentityProvider, *PeerScoringConfig) NodeBuilder + // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. + // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. + // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. + // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. + // Production Tip: use PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. + // Args: + // - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use PeerScoringConfigNoOverride for production. + // Returns: + // none + EnableGossipSubScoringWithOverride(*PeerScoringConfigOverride) NodeBuilder SetCreateNode(CreateNodeFunc) NodeBuilder SetGossipSubFactory(GossipSubFactoryFunc, GossipSubAdapterConfigFunc) NodeBuilder SetStreamCreationRetryInterval(time.Duration) NodeBuilder SetRateLimiterDistributor(UnicastRateLimiterDistributor) NodeBuilder SetGossipSubTracer(PubSubTracer) NodeBuilder SetGossipSubScoreTracerInterval(time.Duration) NodeBuilder - // SetGossipSubRPCInspectors sets the gossipsub rpc inspectors. - SetGossipSubRPCInspectors(inspectors ...GossipSubRPCInspector) NodeBuilder + OverrideDefaultRpcInspectorSuiteFactory(GossipSubRpcInspectorSuiteFactoryFunc) NodeBuilder Build() (LibP2PNode, error) } -// PeerScoringConfig is a configuration for peer scoring parameters for a GossipSub pubsub system. -type PeerScoringConfig struct { +// PeerScoringConfigOverride is a structure that is used to carry over the override values for peer scoring configuration. +// Any attribute that is set in the override will override the default peer scoring config. +// Typically, we are not recommending to override the default peer scoring config in production unless you know what you are doing. +type PeerScoringConfigOverride struct { // TopicScoreParams is a map of topic score parameters for each topic. + // Override criteria: any topic (i.e., key in the map) will override the default topic score parameters for that topic and + // the corresponding value in the map will be used instead of the default value. + // If you don't want to override topic score params for a given topic, simply don't include that topic in the map. + // If the map is nil, the default topic score parameters are used for all topics. TopicScoreParams map[channels.Topic]*pubsub.TopicScoreParams + // AppSpecificScoreParams is a function that returns the application specific score parameters for a given peer. + // Override criteria: if the function is not nil, it will override the default application specific score parameters. + // If the function is nil, the default application specific score parameters are used. AppSpecificScoreParams func(peer.ID) float64 + + // DecayInterval is the interval over which we decay the effect of past behavior, so that + // a good or bad behavior will not have a permanent effect on the penalty. It is also the interval + // that GossipSub uses to refresh the scores of all peers. + // Override criteria: if the value is not zero, it will override the default decay interval. + // If the value is zero, the default decay interval is used. + DecayInterval time.Duration } + +// PeerScoringConfigNoOverride is a default peer scoring configuration for a GossipSub pubsub system. +// It is set to nil, which means that no override is done to the default peer scoring configuration. +// It is the recommended way to use the default peer scoring configuration. +var PeerScoringConfigNoOverride = (*PeerScoringConfigOverride)(nil) diff --git a/network/p2p/cache.go b/network/p2p/cache.go index b481ea67448..f764f1c6321 100644 --- a/network/p2p/cache.go +++ b/network/p2p/cache.go @@ -19,3 +19,67 @@ type ProtocolPeerCache interface { // GetPeers returns a copy of the set of peers that support the given protocol. GetPeers(pid protocol.ID) map[peer.ID]struct{} } + +// UpdateFunction is a function that adjusts the GossipSub spam record of a peer. +// Args: +// - record: the GossipSubSpamRecord of the peer. +// Returns: +// - *GossipSubSpamRecord: the adjusted GossipSubSpamRecord of the peer. +type UpdateFunction func(record GossipSubSpamRecord) GossipSubSpamRecord + +// GossipSubSpamRecordCache is a cache for storing the GossipSub spam records of peers. +// The spam records of peers is used to calculate the application specific score, which is part of the GossipSub score of a peer. +// Note that none of the spam records, application specific score, and GossipSub score are shared publicly with other peers. +// Rather they are solely used by the current peer to select the peers to which it will connect on a topic mesh. +// +// Implementation must be thread-safe. +type GossipSubSpamRecordCache interface { + // Add adds the GossipSubSpamRecord of a peer to the cache. + // Args: + // - peerID: the peer ID of the peer in the GossipSub protocol. + // - record: the GossipSubSpamRecord of the peer. + // + // Returns: + // - bool: true if the record was added successfully, false otherwise. + Add(peerId peer.ID, record GossipSubSpamRecord) bool + + // Get returns the GossipSubSpamRecord of a peer from the cache. + // Args: + // - peerID: the peer ID of the peer in the GossipSub protocol. + // Returns: + // - *GossipSubSpamRecord: the GossipSubSpamRecord of the peer. + // - error on failure to retrieve the record. The returned error is irrecoverable and indicates an exception. + // - bool: true if the record was retrieved successfully, false otherwise. + Get(peerID peer.ID) (*GossipSubSpamRecord, error, bool) + + // Update updates the GossipSub spam penalty of a peer in the cache using the given adjust function. + // Args: + // - peerID: the peer ID of the peer in the GossipSub protocol. + // - adjustFn: the adjust function to be applied to the record. + // Returns: + // - *GossipSubSpamRecord: the updated record. + // - error on failure to update the record. The returned error is irrecoverable and indicates an exception. + Update(peerID peer.ID, updateFunc UpdateFunction) (*GossipSubSpamRecord, error) + + // Has returns true if the cache contains the GossipSubSpamRecord of the given peer. + // Args: + // - peerID: the peer ID of the peer in the GossipSub protocol. + // Returns: + // - bool: true if the cache contains the GossipSubSpamRecord of the given peer, false otherwise. + Has(peerID peer.ID) bool +} + +// GossipSubSpamRecord represents spam record of a peer in the GossipSub protocol. +// It acts as a penalty card for a peer in the GossipSub protocol that keeps the +// spam penalty of the peer as well as its decay factor. +// GossipSubSpam record is used to calculate the application specific score of a peer in the GossipSub protocol. +type GossipSubSpamRecord struct { + // Decay factor of gossipsub spam penalty. + // The Penalty is multiplied by the Decay factor every time the Penalty is updated. + // This is to prevent the Penalty from being stuck at a negative value. + // Each peer has its own Decay factor based on its behavior. + // Valid decay value is in the range [0, 1]. + Decay float64 + // Penalty is the application specific Penalty of the peer. + Penalty float64 +} diff --git a/network/p2p/cache/gossipsub_spam_records.go b/network/p2p/cache/gossipsub_spam_records.go new file mode 100644 index 00000000000..61251e28bcc --- /dev/null +++ b/network/p2p/cache/gossipsub_spam_records.go @@ -0,0 +1,225 @@ +package cache + +import ( + "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" + "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" + "github.com/onflow/flow-go/module/mempool/stdmap" + "github.com/onflow/flow-go/network/p2p" +) + +// GossipSubSpamRecordCache is a cache for storing the gossipsub spam records of peers. It is thread-safe. +// The spam records of peers is used to calculate the application specific score, which is part of the GossipSub score of a peer. +// Note that neither of the spam records, application specific score, and GossipSub score are shared publicly with other peers. +// Rather they are solely used by the current peer to select the peers to which it will connect on a topic mesh. +type GossipSubSpamRecordCache struct { + // the in-memory and thread-safe cache for storing the spam records of peers. + c *stdmap.Backend + + // Optional: the pre-processors to be called upon reading or updating a record in the cache. + // The pre-processors are called in the order they are added to the cache. + // The pre-processors are used to perform any necessary pre-processing on the record before returning it. + // Primary use case is to perform decay operations on the record before reading or updating it. In this way, a + // record is only decayed when it is read or updated without the need to explicitly iterating over the cache. + preprocessFns []PreprocessorFunc +} + +var _ p2p.GossipSubSpamRecordCache = (*GossipSubSpamRecordCache)(nil) + +// PreprocessorFunc is a function that is called by the cache upon reading or updating a record in the cache. +// It is used to perform any necessary pre-processing on the record before returning it when reading or changing it when updating. +// The effect of the pre-processing is that the record is updated in the cache. +// If there are multiple pre-processors, they are called in the order they are added to the cache. +// Args: +// +// record: the record to be pre-processed. +// lastUpdated: the last time the record was updated. +// +// Returns: +// +// GossipSubSpamRecord: the pre-processed record. +// error: an error if the pre-processing failed. The error is considered irrecoverable (unless the parameters can be adjusted and the pre-processing can be retried). The caller is +// advised to crash the node upon an error if failure to read or update the record is not acceptable. +type PreprocessorFunc func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) + +// NewGossipSubSpamRecordCache returns a new HeroCache-based application specific Penalty cache. +// Args: +// +// sizeLimit: the maximum number of entries that can be stored in the cache. +// logger: the logger to be used by the cache. +// collector: the metrics collector to be used by the cache. +// +// Returns: +// +// *GossipSubSpamRecordCache: the newly created cache with a HeroCache-based backend. +func NewGossipSubSpamRecordCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, prFns ...PreprocessorFunc) *GossipSubSpamRecordCache { + backData := herocache.NewCache(sizeLimit, + herocache.DefaultOversizeFactor, + // we should not evict any record from the cache, + // eviction will open the node to spam attacks by malicious peers to erase their application specific penalty. + heropool.NoEjection, + logger.With().Str("mempool", "gossipsub-app-Penalty-cache").Logger(), + collector) + return &GossipSubSpamRecordCache{ + c: stdmap.NewBackend(stdmap.WithBackData(backData)), + preprocessFns: prFns, + } +} + +// Add adds the GossipSubSpamRecord of a peer to the cache. +// Args: +// - peerID: the peer ID of the peer in the GossipSub protocol. +// - record: the GossipSubSpamRecord of the peer. +// +// Returns: +// - bool: true if the record was added successfully, false otherwise. +// Note that a record is added successfully if the cache has enough space to store the record and no record exists for the peer in the cache. +// In other words, the entries are deduplicated by the peer ID. +func (a *GossipSubSpamRecordCache) Add(peerId peer.ID, record p2p.GossipSubSpamRecord) bool { + entityId := flow.HashToID([]byte(peerId)) // HeroCache uses hash of peer.ID as the unique identifier of the record. + return a.c.Add(gossipsubSpamRecordEntity{ + entityId: entityId, + peerID: peerId, + lastUpdated: time.Now(), + GossipSubSpamRecord: record, + }) +} + +// Update updates the GossipSub spam penalty of a peer in the cache. It assumes that a record already exists for the peer in the cache. +// It first reads the record from the cache, applies the pre-processing functions to the record, and then applies the update function to the record. +// The order of the pre-processing functions is the same as the order in which they were added to the cache. +// Args: +// - peerID: the peer ID of the peer in the GossipSub protocol. +// - updateFn: the update function to be applied to the record. +// Returns: +// - *GossipSubSpamRecord: the updated record. +// - error on failure to update the record. The returned error is irrecoverable and indicates an exception. +// Note that if any of the pre-processing functions returns an error, the record is reverted to its original state (prior to applying the update function). +func (a *GossipSubSpamRecordCache) Update(peerID peer.ID, updateFn p2p.UpdateFunction) (*p2p.GossipSubSpamRecord, error) { + // HeroCache uses flow.Identifier for keys, so reformat of the peer.ID + entityId := flow.HashToID([]byte(peerID)) + if !a.c.Has(entityId) { + return nil, fmt.Errorf("could not update spam records for peer %s, record not found", peerID.String()) + } + + var err error + record, updated := a.c.Adjust(entityId, func(entry flow.Entity) flow.Entity { + e := entry.(gossipsubSpamRecordEntity) + + currentRecord := e.GossipSubSpamRecord + // apply the pre-processing functions to the record. + for _, apply := range a.preprocessFns { + e.GossipSubSpamRecord, err = apply(e.GossipSubSpamRecord, e.lastUpdated) + if err != nil { + e.GossipSubSpamRecord = currentRecord + return e // return the original record if the pre-processing fails (atomic abort). + } + } + + // apply the update function to the record. + e.GossipSubSpamRecord = updateFn(e.GossipSubSpamRecord) + + if e.GossipSubSpamRecord != currentRecord { + e.lastUpdated = time.Now() + } + return e + }) + if err != nil { + return nil, fmt.Errorf("could not update spam records for peer %s, error: %w", peerID.String(), err) + } + if !updated { + // this happens when the underlying HeroCache fails to update the record. + return nil, fmt.Errorf("internal cache error for updating %s", peerID.String()) + } + + r := record.(gossipsubSpamRecordEntity).GossipSubSpamRecord + return &r, nil +} + +// Has returns true if the spam record of a peer is found in the cache, false otherwise. +// Args: +// - peerID: the peer ID of the peer in the GossipSub protocol. +// Returns: +// - true if the gossipsub spam record of the peer is found in the cache, false otherwise. +func (a *GossipSubSpamRecordCache) Has(peerID peer.ID) bool { + entityId := flow.HashToID([]byte(peerID)) // HeroCache uses hash of peer.ID as the unique identifier of the record. + return a.c.Has(entityId) +} + +// Get returns the spam record of a peer from the cache. +// Args: +// +// -peerID: the peer ID of the peer in the GossipSub protocol. +// +// Returns: +// - the application specific score record of the peer. +// - error if the underlying cache update fails, or any of the pre-processors fails. The error is considered irrecoverable, and +// the caller is advised to crash the node. +// - true if the record is found in the cache, false otherwise. +func (a *GossipSubSpamRecordCache) Get(peerID peer.ID) (*p2p.GossipSubSpamRecord, error, bool) { + entityId := flow.HashToID([]byte(peerID)) // HeroCache uses hash of peer.ID as the unique identifier of the record. + if !a.c.Has(entityId) { + return nil, nil, false + } + + var err error + record, updated := a.c.Adjust(entityId, func(entry flow.Entity) flow.Entity { + e := entry.(gossipsubSpamRecordEntity) + + currentRecord := e.GossipSubSpamRecord + for _, apply := range a.preprocessFns { + e.GossipSubSpamRecord, err = apply(e.GossipSubSpamRecord, e.lastUpdated) + if err != nil { + e.GossipSubSpamRecord = currentRecord + return e // return the original record if the pre-processing fails (atomic abort). + } + } + if e.GossipSubSpamRecord != currentRecord { + e.lastUpdated = time.Now() + } + return e + }) + if err != nil { + return nil, fmt.Errorf("error while applying pre-processing functions to cache record for peer %s: %w", peerID.String(), err), false + } + if !updated { + return nil, fmt.Errorf("could not decay cache record for peer %s", peerID.String()), false + } + + r := record.(gossipsubSpamRecordEntity).GossipSubSpamRecord + return &r, nil, true +} + +// GossipSubSpamRecord represents an Entity implementation GossipSubSpamRecord. +// It is internally used by the HeroCache to store the GossipSubSpamRecord. +type gossipsubSpamRecordEntity struct { + entityId flow.Identifier // the ID of the record (used to identify the record in the cache). + // lastUpdated is the time at which the record was last updated. + // the peer ID of the peer in the GossipSub protocol. + peerID peer.ID + lastUpdated time.Time + p2p.GossipSubSpamRecord +} + +// In order to use HeroCache, the gossipsubSpamRecordEntity must implement the flow.Entity interface. +var _ flow.Entity = (*gossipsubSpamRecordEntity)(nil) + +// ID returns the ID of the gossipsubSpamRecordEntity. As the ID is used to identify the record in the cache, it must be unique. +// Also, as the ID is used frequently in the cache, it is stored in the record to avoid recomputing it. +// ID is never exposed outside the cache. +func (a gossipsubSpamRecordEntity) ID() flow.Identifier { + return a.entityId +} + +// Checksum returns the same value as ID. Checksum is implemented to satisfy the flow.Entity interface. +// HeroCache does not use the checksum of the gossipsubSpamRecordEntity. +func (a gossipsubSpamRecordEntity) Checksum() flow.Identifier { + return a.entityId +} diff --git a/network/p2p/cache/gossipsub_spam_records_test.go b/network/p2p/cache/gossipsub_spam_records_test.go new file mode 100644 index 00000000000..166776b93ba --- /dev/null +++ b/network/p2p/cache/gossipsub_spam_records_test.go @@ -0,0 +1,481 @@ +package cache_test + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network/p2p" + netcache "github.com/onflow/flow-go/network/p2p/cache" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestGossipSubSpamRecordCache_Add tests the Add method of the GossipSubSpamRecordCache. It tests +// adding a new record to the cache. +func TestGossipSubSpamRecordCache_Add(t *testing.T) { + // create a new instance of GossipSubSpamRecordCache. + cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopCollector()) + + // tests adding a new record to the cache. + require.True(t, cache.Add("peer0", p2p.GossipSubSpamRecord{ + Decay: 0.1, + Penalty: 0.5, + })) + + // tests updating an existing record in the cache. + require.False(t, cache.Add("peer0", p2p.GossipSubSpamRecord{ + Decay: 0.1, + Penalty: 0.5, + })) + + // makes the cache full. + for i := 1; i < 100; i++ { + require.True(t, cache.Add(peer.ID(fmt.Sprintf("peer%d", i)), p2p.GossipSubSpamRecord{ + Decay: 0.1, + Penalty: 0.5, + })) + } + + // adding a new record to the cache should fail. + require.False(t, cache.Add("peer101", p2p.GossipSubSpamRecord{ + Decay: 0.1, + Penalty: 0.5, + })) + + // retrieving an existing record should work. + for i := 0; i < 100; i++ { + record, err, ok := cache.Get(peer.ID(fmt.Sprintf("peer%d", i))) + require.True(t, ok) + require.NoError(t, err) + + require.Equal(t, 0.1, record.Decay) + require.Equal(t, 0.5, record.Penalty) + } + + // yet attempting on adding an existing record should fail. + require.False(t, cache.Add("peer1", p2p.GossipSubSpamRecord{ + Decay: 0.2, + Penalty: 0.8, + })) +} + +// TestGossipSubSpamRecordCache_Concurrent_Add tests if the cache can be added and retrieved concurrently. +// It updates the cache with a number of records concurrently and then checks if the cache +// can retrieve all records. +func TestGossipSubSpamRecordCache_Concurrent_Add(t *testing.T) { + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) + + // defines the number of records to update. + numRecords := 100 + + // uses a wait group to wait for all goroutines to finish. + var wg sync.WaitGroup + wg.Add(numRecords) + + // adds the records concurrently. + for i := 0; i < numRecords; i++ { + go func(num int) { + defer wg.Done() + peerID := fmt.Sprintf("peer%d", num) + added := cache.Add(peer.ID(peerID), p2p.GossipSubSpamRecord{ + Decay: 0.1 * float64(num), + Penalty: float64(num), + }) + require.True(t, added) + }(i) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "could not update all records concurrently on time") + + // checks if the cache can retrieve all records. + for i := 0; i < numRecords; i++ { + peerID := fmt.Sprintf("peer%d", i) + record, err, found := cache.Get(peer.ID(peerID)) + require.True(t, found) + require.NoError(t, err) + + expectedPenalty := float64(i) + require.Equal(t, expectedPenalty, record.Penalty, + "Get() returned incorrect penalty for record %s: expected %f, got %f", peerID, expectedPenalty, record.Penalty) + expectedDecay := 0.1 * float64(i) + require.Equal(t, expectedDecay, record.Decay, + "Get() returned incorrect decay for record %s: expected %f, got %f", peerID, expectedDecay, record.Decay) + } +} + +// TestGossipSubSpamRecordCache_Update tests the Update method of the GossipSubSpamRecordCache. It tests if the cache can update +// the penalty of an existing record and fail to update the penalty of a non-existing record. +func TestGossipSubSpamRecordCache_Update(t *testing.T) { + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) + + peerID := "peer1" + + // tests updateing the penalty of an existing record. + require.True(t, cache.Add(peer.ID(peerID), p2p.GossipSubSpamRecord{ + Decay: 0.1, + Penalty: 0.5, + })) + record, err := cache.Update(peer.ID(peerID), func(record p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord { + record.Penalty = 0.7 + return record + }) + require.NoError(t, err) + require.Equal(t, 0.7, record.Penalty) // checks if the penalty is updateed correctly. + + // tests updating the penalty of a non-existing record. + record, err = cache.Update(peer.ID("peer2"), func(record p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord { + require.Fail(t, "the function should not be called for a non-existing record") + return record + }) + require.Error(t, err) + require.Nil(t, record) +} + +// TestGossipSubSpamRecordCache_Concurrent_Update tests if the cache can be updated concurrently. It updates the cache +// with a number of records concurrently and then checks if the cache can retrieve all records. +func TestGossipSubSpamRecordCache_Concurrent_Update(t *testing.T) { + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) + + // defines the number of records to update. + numRecords := 100 + + // adds all records to the cache, sequentially. + for i := 0; i < numRecords; i++ { + peerID := fmt.Sprintf("peer%d", i) + err := cache.Add(peer.ID(peerID), p2p.GossipSubSpamRecord{ + Decay: 0.1 * float64(i), + Penalty: float64(i), + }) + require.True(t, err) + } + + // uses a wait group to wait for all goroutines to finish. + var wg sync.WaitGroup + wg.Add(numRecords) + + // updates the records concurrently. + for i := 0; i < numRecords; i++ { + go func(num int) { + defer wg.Done() + peerID := fmt.Sprintf("peer%d", num) + _, err := cache.Update(peer.ID(peerID), func(record p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord { + record.Penalty = 0.7 * float64(num) + record.Decay = 0.1 * float64(num) + return record + }) + require.NoError(t, err) + }(i) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "could not update all records concurrently on time") + + // checks if the cache can retrieve all records. + for i := 0; i < numRecords; i++ { + peerID := fmt.Sprintf("peer%d", i) + record, err, found := cache.Get(peer.ID(peerID)) + require.True(t, found) + require.NoError(t, err) + + expectedPenalty := 0.7 * float64(i) + require.Equal(t, expectedPenalty, record.Penalty, + "Get() returned incorrect Penalty for record %s: expected %f, got %f", peerID, expectedPenalty, record.Penalty) + expectedDecay := 0.1 * float64(i) + require.Equal(t, expectedDecay, record.Decay, + "Get() returned incorrect Decay for record %s: expected %f, got %f", peerID, expectedDecay, record.Decay) + } +} + +// TestGossipSubSpamRecordCache_Update_With_Preprocess tests Update method of the GossipSubSpamRecordCache when the cache +// has preprocessor functions. +// It tests when the cache has preprocessor functions, all preprocessor functions are called prior to the update function. +// Also, it tests if the pre-processor functions are called in the order they are added. +func TestGossipSubSpamRecordCache_Update_With_Preprocess(t *testing.T) { + cache := netcache.NewGossipSubSpamRecordCache(200, + unittest.Logger(), + metrics.NewNoopCollector(), + func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { + record.Penalty += 1.5 + return record, nil + }, func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { + record.Penalty *= 2 + return record, nil + }) + + peerID := "peer1" + // adds a record to the cache. + require.True(t, cache.Add(peer.ID(peerID), p2p.GossipSubSpamRecord{ + Decay: 0.1, + Penalty: 0.5, + })) + + // tests updating the penalty of an existing record. + record, err := cache.Update(peer.ID(peerID), func(record p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord { + record.Penalty += 0.7 + return record + }) + require.NoError(t, err) + require.Equal(t, 4.7, record.Penalty) // (0.5+1.5) * 2 + 0.7 = 4.7 + require.Equal(t, 0.1, record.Decay) // checks if the decay is not changed. +} + +// TestGossipSubSpamRecordCache_Update_Preprocess_Error tests the Update method of the GossipSubSpamRecordCache. +// It tests if any of the preprocessor functions returns an error, the update function effect +// is reverted, and the error is returned. +func TestGossipSubSpamRecordCache_Update_Preprocess_Error(t *testing.T) { + secondPreprocessorCalled := false + cache := netcache.NewGossipSubSpamRecordCache(200, + unittest.Logger(), + metrics.NewNoopCollector(), + // the first preprocessor function does not return an error. + func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { + return record, nil + }, + // the second preprocessor function returns an error on the first call and nil on the second call onwards. + func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { + if !secondPreprocessorCalled { + secondPreprocessorCalled = true + return record, fmt.Errorf("error") + } + return record, nil + }) + + peerID := "peer1" + // adds a record to the cache. + require.True(t, cache.Add(peer.ID(peerID), p2p.GossipSubSpamRecord{ + Decay: 0.1, + Penalty: 0.5, + })) + + // tests updating the penalty of an existing record. + record, err := cache.Update(peer.ID(peerID), func(record p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord { + record.Penalty = 0.7 + return record + }) + // since the second preprocessor function returns an error, the update function effect should be reverted. + // the error should be returned. + require.Error(t, err) + require.Nil(t, record) + + // checks if the record is not changed. + record, err, found := cache.Get(peer.ID(peerID)) + require.True(t, found) + require.NoError(t, err) + require.Equal(t, 0.5, record.Penalty) // checks if the penalty is not changed. + require.Equal(t, 0.1, record.Decay) // checks if the decay is not changed. +} + +// TestGossipSubSpamRecordCache_ByValue tests if the cache stores the GossipSubSpamRecord by value. +// It updates the cache with a record and then modifies the record externally. +// It then checks if the record in the cache is still the original record. +// This is a desired behavior that is guaranteed by the underlying HeroCache library. +// In other words, we don't desire the records to be externally mutable after they are added to the cache (unless by a subsequent call to Update). +func TestGossipSubSpamRecordCache_ByValue(t *testing.T) { + cache := netcache.NewGossipSubSpamRecordCache(200, unittest.Logger(), metrics.NewNoopCollector()) + + peerID := "peer1" + added := cache.Add(peer.ID(peerID), p2p.GossipSubSpamRecord{ + Decay: 0.1, + Penalty: 0.5, + }) + require.True(t, added) + + // get the record from the cache + record, err, found := cache.Get(peer.ID(peerID)) + require.True(t, found) + require.NoError(t, err) + + // modify the record + record.Decay = 0.2 + record.Penalty = 0.8 + + // get the record from the cache again + record, err, found = cache.Get(peer.ID(peerID)) + require.True(t, found) + require.NoError(t, err) + + // check if the record is still the same + require.Equal(t, 0.1, record.Decay) + require.Equal(t, 0.5, record.Penalty) +} + +// TestGossipSubSpamRecordCache_Get_With_Preprocessors tests if the cache applies the preprocessors to the records +// before returning them. +func TestGossipSubSpamRecordCache_Get_With_Preprocessors(t *testing.T) { + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector(), + // first preprocessor: adds 1 to the penalty. + func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { + record.Penalty++ + return record, nil + }, + // second preprocessor: multiplies the penalty by 2 + func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { + record.Penalty *= 2 + return record, nil + }, + ) + + record := p2p.GossipSubSpamRecord{ + Decay: 0.5, + Penalty: 1, + } + added := cache.Add("peerA", record) + assert.True(t, added) + + // verifies that the preprocessors were called and the record was updated accordingly. + cachedRecord, err, ok := cache.Get("peerA") + assert.NoError(t, err) + assert.True(t, ok) + + // expected penalty is 4: the first preprocessor adds 1 to the penalty and the second preprocessor multiplies the penalty by 2. + // (1 + 1) * 2 = 4 + assert.Equal(t, 4.0, cachedRecord.Penalty) // penalty should be updated + assert.Equal(t, 0.5, cachedRecord.Decay) // decay should not be modified +} + +// TestGossipSubSpamRecordCache_Get_Preprocessor_Error tests if the cache returns an error if one of the preprocessors returns an error upon a Get. +// It adds a record to the cache and then checks if the cache returns an error upon a Get if one of the preprocessors returns an error. +// It also checks if a preprocessor is failed, the subsequent preprocessors are not called, and the original record is returned. +// In other words, the Get method acts atomically on the record for applying the preprocessors. If one of the preprocessors +// fails, the record is returned without applying the subsequent preprocessors. +func TestGossipSubSpamRecordCache_Get_Preprocessor_Error(t *testing.T) { + secondPreprocessorCalledCount := 0 + thirdPreprocessorCalledCount := 0 + + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector(), + // first preprocessor: adds 1 to the penalty. + func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { + record.Penalty++ + return record, nil + }, + // second preprocessor: multiplies the penalty by 2 (this preprocessor returns an error on the second call) + func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { + secondPreprocessorCalledCount++ + if secondPreprocessorCalledCount < 2 { + // on the first call, the preprocessor is successful + return record, nil + } else { + // on the second call, the preprocessor returns an error + return p2p.GossipSubSpamRecord{}, fmt.Errorf("error in preprocessor") + } + }, + // since second preprocessor returns an error on the second call, the third preprocessor should not be called more than once.. + func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { + thirdPreprocessorCalledCount++ + require.Less(t, secondPreprocessorCalledCount, 2) + return record, nil + }, + ) + + record := p2p.GossipSubSpamRecord{ + Decay: 0.5, + Penalty: 1, + } + added := cache.Add("peerA", record) + assert.True(t, added) + + // verifies that the preprocessors were called and the penalty was updated accordingly. + cachedRecord, err, ok := cache.Get("peerA") + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, 2.0, cachedRecord.Penalty) // penalty should be updated by the first preprocessor (1 + 1 = 2) + assert.Equal(t, 0.5, cachedRecord.Decay) + + // query the cache again that should trigger the second preprocessor to return an error. + cachedRecord, err, ok = cache.Get("peerA") + require.Error(t, err) + assert.False(t, ok) + assert.Nil(t, cachedRecord) + + // verifies that the third preprocessor was not called. + assert.Equal(t, 1, thirdPreprocessorCalledCount) + // verifies that the second preprocessor was called only twice (one success, and one failure). + assert.Equal(t, 2, secondPreprocessorCalledCount) +} + +// TestGossipSubSpamRecordCache_Get_Without_Preprocessors tests when no preprocessors are provided to the cache constructor +// that the cache returns the original record without any modifications. +func TestGossipSubSpamRecordCache_Get_Without_Preprocessors(t *testing.T) { + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector()) + + record := p2p.GossipSubSpamRecord{ + Decay: 0.5, + Penalty: 1, + } + added := cache.Add("peerA", record) + assert.True(t, added) + + // verifies that no preprocessors were called and the record was not updated. + cachedRecord, err, ok := cache.Get("peerA") + assert.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, 1.0, cachedRecord.Penalty) + assert.Equal(t, 0.5, cachedRecord.Decay) +} + +// TestGossipSubSpamRecordCache_Duplicate_Add_Sequential tests if the cache returns false when a duplicate record is added to the cache. +// This test evaluates that the cache de-duplicates the records based on their peer id and not content, and hence +// each peer id can only be added once to the cache. +func TestGossipSubSpamRecordCache_Duplicate_Add_Sequential(t *testing.T) { + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector()) + + record := p2p.GossipSubSpamRecord{ + Decay: 0.5, + Penalty: 1, + } + added := cache.Add("peerA", record) + assert.True(t, added) + + // verifies that the cache returns false when a duplicate record is added. + added = cache.Add("peerA", record) + assert.False(t, added) + + // verifies that the cache deduplicates the records based on their peer id and not content. + record.Penalty = 2 + added = cache.Add("peerA", record) + assert.False(t, added) +} + +// TestGossipSubSpamRecordCache_Duplicate_Add_Concurrent tests if the cache returns false when a duplicate record is added to the cache. +// Test is the concurrent version of TestAppScoreCache_DuplicateAdd_Sequential. +func TestGossipSubSpamRecordCache_Duplicate_Add_Concurrent(t *testing.T) { + cache := netcache.NewGossipSubSpamRecordCache(10, unittest.Logger(), metrics.NewNoopCollector()) + + successAdd := atomic.Int32{} + successAdd.Store(0) + + record1 := p2p.GossipSubSpamRecord{ + Decay: 0.5, + Penalty: 1, + } + + record2 := p2p.GossipSubSpamRecord{ + Decay: 0.5, + Penalty: 2, + } + + wg := sync.WaitGroup{} // wait group to wait for all goroutines to finish. + wg.Add(2) + // adds a record to the cache concurrently. + add := func(record p2p.GossipSubSpamRecord) { + added := cache.Add("peerA", record) + if added { + successAdd.Inc() + } + wg.Done() + } + + go add(record1) + go add(record2) + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "could not add records to the cache") + + // verifies that only one of the records was added to the cache. + assert.Equal(t, int32(1), successAdd.Load()) +} diff --git a/network/p2p/cache/node_blocklist_wrapper.go b/network/p2p/cache/node_blocklist_wrapper.go index ae045ecff62..f655215178a 100644 --- a/network/p2p/cache/node_blocklist_wrapper.go +++ b/network/p2p/cache/node_blocklist_wrapper.go @@ -10,7 +10,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/operation" ) @@ -24,84 +24,101 @@ func (s IdentifierSet) Contains(id flow.Identifier) bool { return found } -// NodeBlocklistWrapper is a wrapper for an `module.IdentityProvider` instance, where the -// wrapper overrides the `Ejected` flag to true for all NodeIDs in a `blocklist`. +// NodeDisallowListingWrapper is a wrapper for an `module.IdentityProvider` instance, where the +// wrapper overrides the `Ejected` flag to true for all NodeIDs in a `disallowList`. // To avoid modifying the source of the identities, the wrapper creates shallow copies // of the identities (whenever necessary) and modifies the `Ejected` flag only in // the copy. -// The `NodeBlocklistWrapper` internally represents the `blocklist` as a map, to enable +// The `NodeDisallowListingWrapper` internally represents the `disallowList` as a map, to enable // performant lookup. However, the exported API works with `flow.IdentifierList` for -// blocklist, as this is a broadly supported data structure which lends itself better +// disallowList, as this is a broadly supported data structure which lends itself better // to config or command-line inputs. -type NodeBlocklistWrapper struct { +// When a node is disallow-listed, the networking layer connection to that node is closed and no +// incoming or outgoing connections are established with that node. +// TODO: terminology change - rename `blocklist` to `disallowList` everywhere to be consistent with the code. +type NodeDisallowListingWrapper struct { m sync.RWMutex db *badger.DB identityProvider module.IdentityProvider - blocklist IdentifierSet // `IdentifierSet` is a map, hence efficient O(1) lookup - distributor p2p.DisallowListNotificationDistributor // distributor for the blocklist update notifications + disallowList IdentifierSet // `IdentifierSet` is a map, hence efficient O(1) lookup + + // updateConsumerOracle is called whenever the disallow-list is updated. + // Note that we do not use the `updateConsumer` directly due to the circular dependency between the + // middleware (i.e., updateConsumer), and the wrapper (i.e., NodeDisallowListingWrapper). + // Middleware needs identity provider to be initialized, and identity provider needs this wrapper to be initialized. + // Hence, if we pass the updateConsumer by the interface value, it will be nil at the time of initialization. + // Instead, we use the oracle function to get the updateConsumer whenever we need it. + updateConsumerOracle func() network.DisallowListNotificationConsumer } -var _ module.IdentityProvider = (*NodeBlocklistWrapper)(nil) +var _ module.IdentityProvider = (*NodeDisallowListingWrapper)(nil) -// NewNodeBlocklistWrapper wraps the given `IdentityProvider`. The blocklist is +// NewNodeDisallowListWrapper wraps the given `IdentityProvider`. The disallow-list is // loaded from the database (or assumed to be empty if no database entry is present). -func NewNodeBlocklistWrapper( +func NewNodeDisallowListWrapper( identityProvider module.IdentityProvider, db *badger.DB, - distributor p2p.DisallowListNotificationDistributor) (*NodeBlocklistWrapper, error) { + updateConsumerOracle func() network.DisallowListNotificationConsumer) (*NodeDisallowListingWrapper, error) { - blocklist, err := retrieveBlocklist(db) + disallowList, err := retrieveDisallowList(db) if err != nil { - return nil, fmt.Errorf("failed to read set of blocked node IDs from data base: %w", err) + return nil, fmt.Errorf("failed to read set of disallowed node IDs from data base: %w", err) } - return &NodeBlocklistWrapper{ - db: db, - identityProvider: identityProvider, - blocklist: blocklist, - distributor: distributor, + return &NodeDisallowListingWrapper{ + db: db, + identityProvider: identityProvider, + disallowList: disallowList, + updateConsumerOracle: updateConsumerOracle, }, nil } -// Update sets the wrapper's internal set of blocked nodes to `blocklist`. Empty list and `nil` -// (equivalent to empty list) are accepted inputs. To avoid legacy entries in the data base, this -// function purges the entire data base entry if `blocklist` is empty. -// This implementation is _eventually consistent_, where changes are written to the data base first +// Update sets the wrapper's internal set of blocked nodes to `disallowList`. Empty list and `nil` +// (equivalent to empty list) are accepted inputs. To avoid legacy entries in the database, this +// function purges the entire data base entry if `disallowList` is empty. +// This implementation is _eventually consistent_, where changes are written to the database first // and then (non-atomically!) the in-memory set of blocked nodes is updated. This strongly // benefits performance and modularity. No errors are expected during normal operations. -func (w *NodeBlocklistWrapper) Update(blocklist flow.IdentifierList) error { - b := blocklist.Lookup() // converts slice to map +// +// Args: +// - disallowList: list of node IDs to be disallow-listed from the networking layer, i.e., the existing connections +// to these nodes will be closed and no new connections will be established (neither incoming nor outgoing). +// +// Returns: +// - error: if the update fails, e.g., due to a database error. Any returned error is irrecoverable and the caller +// should abort the process. +func (w *NodeDisallowListingWrapper) Update(disallowList flow.IdentifierList) error { + b := disallowList.Lookup() // converts slice to map w.m.Lock() defer w.m.Unlock() - err := persistBlocklist(b, w.db) + err := persistDisallowList(b, w.db) if err != nil { return fmt.Errorf("failed to persist set of blocked nodes to the data base: %w", err) } - w.blocklist = b - err = w.distributor.DistributeBlockListNotification(blocklist) - - if err != nil { - return fmt.Errorf("failed to distribute blocklist update notification: %w", err) - } + w.disallowList = b + w.updateConsumerOracle().OnDisallowListNotification(&network.DisallowListingUpdate{ + FlowIds: disallowList, + Cause: network.DisallowListedCauseAdmin, + }) return nil } -// ClearBlocklist purges the set of blocked node IDs. Convenience function +// ClearDisallowList purges the set of blocked node IDs. Convenience function // equivalent to w.Update(nil). No errors are expected during normal operations. -func (w *NodeBlocklistWrapper) ClearBlocklist() error { +func (w *NodeDisallowListingWrapper) ClearDisallowList() error { return w.Update(nil) } -// GetBlocklist returns the set of blocked node IDs. -func (w *NodeBlocklistWrapper) GetBlocklist() flow.IdentifierList { +// GetDisallowList returns the set of blocked node IDs. +func (w *NodeDisallowListingWrapper) GetDisallowList() flow.IdentifierList { w.m.RLock() defer w.m.RUnlock() - identifiers := make(flow.IdentifierList, 0, len(w.blocklist)) - for i := range w.blocklist { + identifiers := make(flow.IdentifierList, 0, len(w.disallowList)) + for i := range w.disallowList { identifiers = append(identifiers, i) } return identifiers @@ -111,7 +128,7 @@ func (w *NodeBlocklistWrapper) GetBlocklist() flow.IdentifierList { // protocol that pass the provided filter. Caution, this includes ejected nodes. // Please check the `Ejected` flag in the returned identities (or provide a // filter for removing ejected nodes). -func (w *NodeBlocklistWrapper) Identities(filter flow.IdentityFilter) flow.IdentityList { +func (w *NodeDisallowListingWrapper) Identities(filter flow.IdentityFilter) flow.IdentityList { identities := w.identityProvider.Identities(filter) if len(identities) == 0 { return identities @@ -123,7 +140,7 @@ func (w *NodeBlocklistWrapper) Identities(filter flow.IdentityFilter) flow.Ident idtx := make(flow.IdentityList, 0, len(identities)) w.m.RLock() for _, identity := range identities { - if w.blocklist.Contains(identity.NodeID) { + if w.disallowList.Contains(identity.NodeID) { var i = *identity // shallow copy is sufficient, because `Ejected` flag is in top-level struct i.Ejected = true if filter(&i) { // we need to check the filter here again, because the filter might drop ejected nodes and we are modifying the ejected status here @@ -143,22 +160,22 @@ func (w *NodeBlocklistWrapper) Identities(filter flow.IdentityFilter) flow.Ident // true if and only if Identity has been found, i.e. `Identity` is not nil. // Caution: function returns include ejected nodes. Please check the `Ejected` // flag in the identity. -func (w *NodeBlocklistWrapper) ByNodeID(identifier flow.Identifier) (*flow.Identity, bool) { +func (w *NodeDisallowListingWrapper) ByNodeID(identifier flow.Identifier) (*flow.Identity, bool) { identity, b := w.identityProvider.ByNodeID(identifier) return w.setEjectedIfBlocked(identity), b } -// setEjectedIfBlocked checks whether the node with the given identity is on the `blocklist`. +// setEjectedIfBlocked checks whether the node with the given identity is on the `disallowList`. // Shortcuts: // - If the node's identity is nil, there is nothing to do because we don't generate identities here. -// - If the node is already ejected, we don't have to check the blocklist. -func (w *NodeBlocklistWrapper) setEjectedIfBlocked(identity *flow.Identity) *flow.Identity { +// - If the node is already ejected, we don't have to check the disallowList. +func (w *NodeDisallowListingWrapper) setEjectedIfBlocked(identity *flow.Identity) *flow.Identity { if identity == nil || identity.Ejected { return identity } w.m.RLock() - isBlocked := w.blocklist.Contains(identity.NodeID) + isBlocked := w.disallowList.Contains(identity.NodeID) w.m.RUnlock() if !isBlocked { return identity @@ -178,25 +195,25 @@ func (w *NodeBlocklistWrapper) setEjectedIfBlocked(identity *flow.Identity) *flo // true if and only if Identity has been found, i.e. `Identity` is not nil. // Caution: function returns include ejected nodes. Please check the `Ejected` // flag in the identity. -func (w *NodeBlocklistWrapper) ByPeerID(p peer.ID) (*flow.Identity, bool) { +func (w *NodeDisallowListingWrapper) ByPeerID(p peer.ID) (*flow.Identity, bool) { identity, b := w.identityProvider.ByPeerID(p) return w.setEjectedIfBlocked(identity), b } -// persistBlocklist writes the given blocklist to the database. To avoid legacy -// entries in the database, we prune the entire data base entry if `blocklist` is +// persistDisallowList writes the given disallowList to the database. To avoid legacy +// entries in the database, we prune the entire data base entry if `disallowList` is // empty. No errors are expected during normal operations. -func persistBlocklist(blocklist IdentifierSet, db *badger.DB) error { - if len(blocklist) == 0 { +func persistDisallowList(disallowList IdentifierSet, db *badger.DB) error { + if len(disallowList) == 0 { return db.Update(operation.PurgeBlocklist()) } - return db.Update(operation.PersistBlocklist(blocklist)) + return db.Update(operation.PersistBlocklist(disallowList)) } -// retrieveBlocklist reads the set of blocked nodes from the data base. +// retrieveDisallowList reads the set of blocked nodes from the data base. // In case no database entry exists, an empty set (nil map) is returned. // No errors are expected during normal operations. -func retrieveBlocklist(db *badger.DB) (IdentifierSet, error) { +func retrieveDisallowList(db *badger.DB) (IdentifierSet, error) { var blocklist map[flow.Identifier]struct{} err := db.View(operation.RetrieveBlocklist(&blocklist)) if err != nil && !errors.Is(err, storage.ErrNotFound) { diff --git a/network/p2p/cache/node_blocklist_wrapper_test.go b/network/p2p/cache/node_blocklist_wrapper_test.go index cdc32b546f5..929be0b066a 100644 --- a/network/p2p/cache/node_blocklist_wrapper_test.go +++ b/network/p2p/cache/node_blocklist_wrapper_test.go @@ -14,39 +14,42 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" mocks "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/cache" - mockp2p "github.com/onflow/flow-go/network/p2p/mock" "github.com/onflow/flow-go/utils/unittest" ) -type NodeBlocklistWrapperTestSuite struct { +type NodeDisallowListWrapperTestSuite struct { suite.Suite DB *badger.DB provider *mocks.IdentityProvider - wrapper *cache.NodeBlocklistWrapper - distributor *mockp2p.DisallowListNotificationDistributor + wrapper *cache.NodeDisallowListingWrapper + updateConsumer *mocknetwork.DisallowListNotificationConsumer } -func (s *NodeBlocklistWrapperTestSuite) SetupTest() { +func (s *NodeDisallowListWrapperTestSuite) SetupTest() { s.DB, _ = unittest.TempBadgerDB(s.T()) s.provider = new(mocks.IdentityProvider) var err error - s.distributor = mockp2p.NewDisallowListNotificationDistributor(s.T()) - s.wrapper, err = cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.distributor) + s.updateConsumer = mocknetwork.NewDisallowListNotificationConsumer(s.T()) + s.wrapper, err = cache.NewNodeDisallowListWrapper(s.provider, s.DB, func() network.DisallowListNotificationConsumer { + return s.updateConsumer + }) require.NoError(s.T(), err) } -func TestNodeBlocklistWrapperTestSuite(t *testing.T) { - suite.Run(t, new(NodeBlocklistWrapperTestSuite)) +func TestNodeDisallowListWrapperTestSuite(t *testing.T) { + suite.Run(t, new(NodeDisallowListWrapperTestSuite)) } // TestHonestNode verifies: -// For nodes _not_ on the blocklist, the `cache.NodeBlocklistWrapper` should forward +// For nodes _not_ on the disallowList, the `cache.NodeDisallowListingWrapper` should forward // the identities from the wrapped `IdentityProvider` without modification. -func (s *NodeBlocklistWrapperTestSuite) TestHonestNode() { +func (s *NodeDisallowListWrapperTestSuite) TestHonestNode() { s.Run("ByNodeID", func() { identity := unittest.IdentityFixture() s.provider.On("ByNodeID", identity.NodeID).Return(identity, true) @@ -78,8 +81,8 @@ func (s *NodeBlocklistWrapperTestSuite) TestHonestNode() { }) } -// TestDenylistedNode tests proper handling of identities _on_ the blocklist: -// - For any identity `i` with `i.NodeID ∈ blocklist`, the returned identity +// TestDisallowListNode tests proper handling of identities _on_ the disallowList: +// - For any identity `i` with `i.NodeID ∈ disallowList`, the returned identity // should have `i.Ejected` set to `true` (irrespective of the `Ejected` // flag's initial returned by the wrapped `IdentityProvider`). // - The wrapper should _copy_ the identity and _not_ write into the wrapped @@ -91,9 +94,12 @@ func (s *NodeBlocklistWrapperTestSuite) TestHonestNode() { // While returning (non-nil identity, false) is not a defined return value, // we expect the wrapper to nevertheless handle this case to increase its // generality. -func (s *NodeBlocklistWrapperTestSuite) TestDenylistedNode() { +func (s *NodeDisallowListWrapperTestSuite) TestDisallowListNode() { blocklist := unittest.IdentityListFixture(11) - s.distributor.On("DistributeBlockListNotification", blocklist.NodeIDs()).Return(nil).Once() + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: blocklist.NodeIDs(), + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() err := s.wrapper.Update(blocklist.NodeIDs()) require.NoError(s.T(), err) @@ -137,7 +143,8 @@ func (s *NodeBlocklistWrapperTestSuite) TestDenylistedNode() { blocklistLookup := blocklist.Lookup() honestIdentities := unittest.IdentityListFixture(8) combinedIdentities := honestIdentities.Union(blocklist) - combinedIdentities = combinedIdentities.DeterministicShuffle(1234) + combinedIdentities, err = combinedIdentities.Shuffle() + require.NoError(s.T(), err) numIdentities := len(combinedIdentities) s.provider.On("Identities", mock.Anything).Return(combinedIdentities) @@ -164,7 +171,8 @@ func (s *NodeBlocklistWrapperTestSuite) TestDenylistedNode() { blocklistLookup := blocklist.Lookup() honestIdentities := unittest.IdentityListFixture(8) combinedIdentities := honestIdentities.Union(blocklist) - combinedIdentities = combinedIdentities.DeterministicShuffle(1234) + combinedIdentities, err = combinedIdentities.Shuffle() + require.NoError(s.T(), err) numIdentities := len(combinedIdentities) s.provider.On("Identities", mock.Anything).Return(combinedIdentities) @@ -188,7 +196,7 @@ func (s *NodeBlocklistWrapperTestSuite) TestDenylistedNode() { // TestUnknownNode verifies that the wrapper forwards nil identities // irrespective of the boolean return values. -func (s *NodeBlocklistWrapperTestSuite) TestUnknownNode() { +func (s *NodeDisallowListWrapperTestSuite) TestUnknownNode() { for _, b := range []bool{true, false} { s.Run(fmt.Sprintf("IdentityProvider.ByNodeID returning (nil, %v)", b), func() { id := unittest.IdentifierFixture() @@ -210,13 +218,13 @@ func (s *NodeBlocklistWrapperTestSuite) TestUnknownNode() { } } -// TestBlocklistAddRemove checks that adding and subsequently removing a node from the blocklist +// TestDisallowListAddRemove checks that adding and subsequently removing a node from the disallowList // it in combination a no-op. We test two scenarious // - Node whose original `Identity` has `Ejected = false`: -// After adding the node to the blocklist and then removing it again, the `Ejected` should be false. +// After adding the node to the disallowList and then removing it again, the `Ejected` should be false. // - Node whose original `Identity` has `Ejected = true`: -// After adding the node to the blocklist and then removing it again, the `Ejected` should be still be true. -func (s *NodeBlocklistWrapperTestSuite) TestBlocklistAddRemove() { +// After adding the node to the disallowList and then removing it again, the `Ejected` should be still be true. +func (s *NodeDisallowListWrapperTestSuite) TestDisallowListAddRemove() { for _, originalEjected := range []bool{true, false} { s.Run(fmt.Sprintf("Add & remove node with Ejected = %v", originalEjected), func() { originalIdentity := unittest.IdentityFixture() @@ -225,7 +233,7 @@ func (s *NodeBlocklistWrapperTestSuite) TestBlocklistAddRemove() { s.provider.On("ByNodeID", originalIdentity.NodeID).Return(originalIdentity, true) s.provider.On("ByPeerID", peerID).Return(originalIdentity, true) - // step 1: before putting node on blocklist, + // step 1: before putting node on disallowList, // an Identity with `Ejected` equal to the original value should be returned i, found := s.wrapper.ByNodeID(originalIdentity.NodeID) require.True(s.T(), found) @@ -235,9 +243,12 @@ func (s *NodeBlocklistWrapperTestSuite) TestBlocklistAddRemove() { require.True(s.T(), found) require.Equal(s.T(), originalEjected, i.Ejected) - // step 2: _after_ putting node on blocklist, + // step 2: _after_ putting node on disallowList, // an Identity with `Ejected` equal to `true` should be returned - s.distributor.On("DistributeBlockListNotification", flow.IdentifierList{originalIdentity.NodeID}).Return(nil).Once() + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: flow.IdentifierList{originalIdentity.NodeID}, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() err := s.wrapper.Update(flow.IdentifierList{originalIdentity.NodeID}) require.NoError(s.T(), err) @@ -249,9 +260,12 @@ func (s *NodeBlocklistWrapperTestSuite) TestBlocklistAddRemove() { require.True(s.T(), found) require.True(s.T(), i.Ejected) - // step 3: after removing the node from the blocklist, + // step 3: after removing the node from the disallowList, // an Identity with `Ejected` equal to the original value should be returned - s.distributor.On("DistributeBlockListNotification", flow.IdentifierList{}).Return(nil).Once() + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: flow.IdentifierList{}, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() err = s.wrapper.Update(flow.IdentifierList{}) require.NoError(s.T(), err) @@ -266,125 +280,170 @@ func (s *NodeBlocklistWrapperTestSuite) TestBlocklistAddRemove() { } } -// TestUpdate tests updating, clearing and retrieving the blocklist. +// TestUpdate tests updating, clearing and retrieving the disallowList. // This test verifies that the wrapper updates _its own internal state_ correctly. // Note: -// conceptually, the blocklist is a set, i.e. not order dependent. +// conceptually, the disallowList is a set, i.e. not order dependent. // The wrapper internally converts the list to a set and vice versa. Therefore -// the order is not preserved by `GetBlocklist`. Consequently, we compare +// the order is not preserved by `GetDisallowList`. Consequently, we compare // map-based representations here. -func (s *NodeBlocklistWrapperTestSuite) TestUpdate() { - blocklist1 := unittest.IdentifierListFixture(8) - blocklist2 := unittest.IdentifierListFixture(11) - blocklist3 := unittest.IdentifierListFixture(5) - - s.distributor.On("DistributeBlockListNotification", blocklist1).Return(nil).Once() - err := s.wrapper.Update(blocklist1) +func (s *NodeDisallowListWrapperTestSuite) TestUpdate() { + disallowList1 := unittest.IdentifierListFixture(8) + disallowList2 := unittest.IdentifierListFixture(11) + disallowList3 := unittest.IdentifierListFixture(5) + + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: disallowList1, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err := s.wrapper.Update(disallowList1) require.NoError(s.T(), err) - require.Equal(s.T(), blocklist1.Lookup(), s.wrapper.GetBlocklist().Lookup()) + require.Equal(s.T(), disallowList1.Lookup(), s.wrapper.GetDisallowList().Lookup()) - s.distributor.On("DistributeBlockListNotification", blocklist2).Return(nil).Once() - err = s.wrapper.Update(blocklist2) + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: disallowList2, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err = s.wrapper.Update(disallowList2) require.NoError(s.T(), err) - require.Equal(s.T(), blocklist2.Lookup(), s.wrapper.GetBlocklist().Lookup()) + require.Equal(s.T(), disallowList2.Lookup(), s.wrapper.GetDisallowList().Lookup()) - s.distributor.On("DistributeBlockListNotification", (flow.IdentifierList)(nil)).Return(nil).Once() - err = s.wrapper.ClearBlocklist() + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: nil, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err = s.wrapper.ClearDisallowList() require.NoError(s.T(), err) - require.Empty(s.T(), s.wrapper.GetBlocklist()) + require.Empty(s.T(), s.wrapper.GetDisallowList()) - s.distributor.On("DistributeBlockListNotification", blocklist3).Return(nil).Once() - err = s.wrapper.Update(blocklist3) + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: disallowList3, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err = s.wrapper.Update(disallowList3) require.NoError(s.T(), err) - require.Equal(s.T(), blocklist3.Lookup(), s.wrapper.GetBlocklist().Lookup()) + require.Equal(s.T(), disallowList3.Lookup(), s.wrapper.GetDisallowList().Lookup()) } // TestDataBasePersist verifies database interactions of the wrapper with the data base. -// This test verifies that the blocklist updates are persisted across restarts. +// This test verifies that the disallowList updates are persisted across restarts. // To decouple this test from the lower-level data base design, we proceed as follows: -// - We do data-base operation through the exported methods from `NodeBlocklistWrapper` -// - Then, we create a new `NodeBlocklistWrapper` backed by the same data base. Since it is a +// - We do data-base operation through the exported methods from `NodeDisallowListingWrapper` +// - Then, we create a new `NodeDisallowListingWrapper` backed by the same data base. Since it is a // new wrapper, it must read its state from the data base. Hence, if the new wrapper returns // the correct data, we have strong evidence that data-base interactions are correct. // // Note: The wrapper internally converts the list to a set and vice versa. Therefore -// the order is not preserved by `GetBlocklist`. Consequently, we compare +// the order is not preserved by `GetDisallowList`. Consequently, we compare // map-based representations here. -func (s *NodeBlocklistWrapperTestSuite) TestDataBasePersist() { - blocklist := unittest.IdentifierListFixture(8) - blocklist2 := unittest.IdentifierListFixture(8) +func (s *NodeDisallowListWrapperTestSuite) TestDataBasePersist() { + disallowList1 := unittest.IdentifierListFixture(8) + disallowList2 := unittest.IdentifierListFixture(8) - s.Run("Get blocklist from empty database", func() { - require.Empty(s.T(), s.wrapper.GetBlocklist()) + s.Run("Get disallowList from empty database", func() { + require.Empty(s.T(), s.wrapper.GetDisallowList()) }) - s.Run("Clear blocklist on empty database", func() { - s.distributor.On("DistributeBlockListNotification", (flow.IdentifierList)(nil)).Return(nil).Once() - err := s.wrapper.ClearBlocklist() // No-op as data base does not contain any block list + s.Run("Clear disallow-list on empty database", func() { + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: nil, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err := s.wrapper.ClearDisallowList() // No-op as data base does not contain any block list require.NoError(s.T(), err) - require.Empty(s.T(), s.wrapper.GetBlocklist()) + require.Empty(s.T(), s.wrapper.GetDisallowList()) - // newly created wrapper should read `blocklist` from data base during initialization - w, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.distributor) + // newly created wrapper should read `disallowList` from data base during initialization + w, err := cache.NewNodeDisallowListWrapper(s.provider, s.DB, func() network.DisallowListNotificationConsumer { + return s.updateConsumer + }) require.NoError(s.T(), err) - require.Empty(s.T(), w.GetBlocklist()) + require.Empty(s.T(), w.GetDisallowList()) }) - s.Run("Update blocklist and init new wrapper from database", func() { - s.distributor.On("DistributeBlockListNotification", blocklist).Return(nil).Once() - err := s.wrapper.Update(blocklist) + s.Run("Update disallowList and init new wrapper from database", func() { + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: disallowList1, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err := s.wrapper.Update(disallowList1) require.NoError(s.T(), err) - // newly created wrapper should read `blocklist` from data base during initialization - w, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.distributor) + // newly created wrapper should read `disallowList` from data base during initialization + w, err := cache.NewNodeDisallowListWrapper(s.provider, s.DB, func() network.DisallowListNotificationConsumer { + return s.updateConsumer + }) require.NoError(s.T(), err) - require.Equal(s.T(), blocklist.Lookup(), w.GetBlocklist().Lookup()) + require.Equal(s.T(), disallowList1.Lookup(), w.GetDisallowList().Lookup()) }) - s.Run("Update and overwrite blocklist and then init new wrapper from database", func() { - s.distributor.On("DistributeBlockListNotification", blocklist).Return(nil).Once() - err := s.wrapper.Update(blocklist) + s.Run("Update and overwrite disallowList and then init new wrapper from database", func() { + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: disallowList1, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err := s.wrapper.Update(disallowList1) require.NoError(s.T(), err) - s.distributor.On("DistributeBlockListNotification", blocklist2).Return(nil).Once() - err = s.wrapper.Update(blocklist2) + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: disallowList2, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err = s.wrapper.Update(disallowList2) require.NoError(s.T(), err) // newly created wrapper should read initial state from data base - w, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.distributor) + w, err := cache.NewNodeDisallowListWrapper(s.provider, s.DB, func() network.DisallowListNotificationConsumer { + return s.updateConsumer + }) require.NoError(s.T(), err) - require.Equal(s.T(), blocklist2.Lookup(), w.GetBlocklist().Lookup()) + require.Equal(s.T(), disallowList2.Lookup(), w.GetDisallowList().Lookup()) }) s.Run("Update & clear & update and then init new wrapper from database", func() { - // set blocklist -> + // set disallowList -> // newly created wrapper should now read this list from data base during initialization - s.distributor.On("DistributeBlockListNotification", blocklist).Return(nil).Once() - err := s.wrapper.Update(blocklist) + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: disallowList1, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err := s.wrapper.Update(disallowList1) require.NoError(s.T(), err) - w0, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.distributor) + w0, err := cache.NewNodeDisallowListWrapper(s.provider, s.DB, func() network.DisallowListNotificationConsumer { + return s.updateConsumer + }) require.NoError(s.T(), err) - require.Equal(s.T(), blocklist.Lookup(), w0.GetBlocklist().Lookup()) - - // clear blocklist -> - // newly created wrapper should now read empty blocklist from data base during initialization - s.distributor.On("DistributeBlockListNotification", (flow.IdentifierList)(nil)).Return(nil).Once() - err = s.wrapper.ClearBlocklist() + require.Equal(s.T(), disallowList1.Lookup(), w0.GetDisallowList().Lookup()) + + // clear disallowList -> + // newly created wrapper should now read empty disallowList from data base during initialization + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: nil, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err = s.wrapper.ClearDisallowList() require.NoError(s.T(), err) - w1, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.distributor) + w1, err := cache.NewNodeDisallowListWrapper(s.provider, s.DB, func() network.DisallowListNotificationConsumer { + return s.updateConsumer + }) require.NoError(s.T(), err) - require.Empty(s.T(), w1.GetBlocklist()) + require.Empty(s.T(), w1.GetDisallowList()) - // set blocklist2 -> + // set disallowList2 -> // newly created wrapper should now read this list from data base during initialization - s.distributor.On("DistributeBlockListNotification", blocklist2).Return(nil).Once() - err = s.wrapper.Update(blocklist2) + s.updateConsumer.On("OnDisallowListNotification", &network.DisallowListingUpdate{ + FlowIds: disallowList2, + Cause: network.DisallowListedCauseAdmin, + }).Return().Once() + err = s.wrapper.Update(disallowList2) require.NoError(s.T(), err) - w2, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.distributor) + w2, err := cache.NewNodeDisallowListWrapper(s.provider, s.DB, func() network.DisallowListNotificationConsumer { + return s.updateConsumer + }) require.NoError(s.T(), err) - require.Equal(s.T(), blocklist2.Lookup(), w2.GetBlocklist().Lookup()) + require.Equal(s.T(), disallowList2.Lookup(), w2.GetDisallowList().Lookup()) }) } diff --git a/network/p2p/conduit/conduit.go b/network/p2p/conduit/conduit.go index 353e67c29fc..eef36cecbaa 100644 --- a/network/p2p/conduit/conduit.go +++ b/network/p2p/conduit/conduit.go @@ -5,8 +5,6 @@ import ( "fmt" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/component" - "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" ) @@ -15,23 +13,21 @@ import ( // It directly passes the incoming messages to the corresponding methods of the // network Adapter. type DefaultConduitFactory struct { - *component.ComponentManager adapter network.Adapter } -func NewDefaultConduitFactory() *DefaultConduitFactory { - d := &DefaultConduitFactory{} - // worker added so conduit factory doesn't immediately shut down when it's started - cm := component.NewComponentManagerBuilder(). - AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - ready() - - <-ctx.Done() - }).Build() +var _ network.ConduitFactory = (*DefaultConduitFactory)(nil) - d.ComponentManager = cm - - return d +// NewDefaultConduitFactory creates a new DefaultConduitFactory, this is the default conduit factory used by the node. +// Args: +// +// none +// +// Returns: +// +// a new instance of the DefaultConduitFactory. +func NewDefaultConduitFactory() *DefaultConduitFactory { + return &DefaultConduitFactory{} } // RegisterAdapter sets the Adapter component of the factory. @@ -74,6 +70,8 @@ type Conduit struct { adapter network.Adapter } +var _ network.Conduit = (*Conduit)(nil) + // Publish sends an event to the network layer for unreliable delivery // to subscribers of the given event on the network layer. It uses a // publish-subscribe layer and can thus not guarantee that the specified @@ -104,6 +102,14 @@ func (c *Conduit) Multicast(event interface{}, num uint, targetIDs ...flow.Ident return c.adapter.MulticastOnChannel(c.channel, event, num, targetIDs...) } +// ReportMisbehavior reports the misbehavior of a node on sending a message to the current node that appears valid +// based on the networking layer but is considered invalid by the current node based on the Flow protocol. +// The misbehavior is reported to the networking layer to penalize the misbehaving node. +// The implementation must be thread-safe and non-blocking. +func (c *Conduit) ReportMisbehavior(report network.MisbehaviorReport) { + c.adapter.ReportMisbehaviorOnChannel(c.channel, report) +} + func (c *Conduit) Close() error { if c.ctx.Err() != nil { return fmt.Errorf("conduit for channel %s already closed", c.channel) diff --git a/network/p2p/connection/connManager.go b/network/p2p/connection/connManager.go index 9483da30d75..54d85175cce 100644 --- a/network/p2p/connection/connManager.go +++ b/network/p2p/connection/connManager.go @@ -3,7 +3,6 @@ package connection import ( "context" "fmt" - "time" "github.com/libp2p/go-libp2p/core/connmgr" "github.com/libp2p/go-libp2p/core/network" @@ -12,35 +11,10 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p/connection/internal" ) -const ( - // defaultHighWatermark is the default value for the high watermark (i.e., max number of connections). - // We assume a complete topology graph with maximum of 500 nodes. - defaultHighWatermark = 500 - - // defaultLowWatermark is the default value for the low watermark (i.e., min number of connections). - // We assume a complete topology graph with minimum of 450 nodes. - defaultLowWatermark = 450 - - // defaultGracePeriod is the default value for the grace period (i.e., time to wait before pruning a new connection). - defaultGracePeriod = 1 * time.Minute - - // defaultSilencePeriod is the default value for the silence period (i.e., time to wait before start pruning connections). - defaultSilencePeriod = 10 * time.Second -) - -// DefaultConnManagerConfig returns the default configuration for the connection manager. -func DefaultConnManagerConfig() *ManagerConfig { - return &ManagerConfig{ - HighWatermark: defaultHighWatermark, - LowWatermark: defaultLowWatermark, - GracePeriod: defaultGracePeriod, - SilencePeriod: defaultSilencePeriod, - } -} - // ConnManager provides an implementation of Libp2p's ConnManager interface (https://pkg.go.dev/github.com/libp2p/go-libp2p/core/connmgr#ConnManager) // It is called back by libp2p when certain events occur such as opening/closing a stream, opening/closing connection etc. // Current implementation primarily acts as a wrapper around libp2p's BasicConnMgr (https://pkg.go.dev/github.com/libp2p/go-libp2p/p2p/net/connmgr#BasicConnMgr). @@ -53,33 +27,11 @@ type ConnManager struct { var _ connmgr.ConnManager = (*ConnManager)(nil) -type ManagerConfig struct { - // HighWatermark and LowWatermark govern the number of connections are maintained by the ConnManager. - // When the peer count exceeds the HighWatermark, as many peers will be pruned (and - // their connections terminated) until LowWatermark peers remain. In other words, whenever the - // peer count is x > HighWatermark, the ConnManager will prune x - LowWatermark peers. - // The pruning algorithm is as follows: - // 1. The ConnManager will not prune any peers that have been connected for less than GracePeriod. - // 2. The ConnManager will not prune any peers that are protected. - // 3. The ConnManager will sort the peers based on their number of streams and direction of connections, and - // prunes the peers with the least number of streams. If there are ties, the peer with the incoming connection - // will be pruned. If both peers have incoming connections, and there are still ties, one of the peers will be - // pruned at random. - // Algorithm implementation is in https://github.com/libp2p/go-libp2p/blob/master/p2p/net/connmgr/connmgr.go#L262-L318 - HighWatermark int // naming from libp2p - LowWatermark int // naming from libp2p - - // SilencePeriod is the time to wait before start pruning connections. - SilencePeriod time.Duration // naming from libp2p - // GracePeriod is the time to wait before pruning a new connection. - GracePeriod time.Duration // naming from libp2p -} - // NewConnManager creates a new connection manager. // It errors if creating the basic connection manager of libp2p fails. // The error is not benign, and we should crash the node if it happens. // It is a malpractice to start the node without connection manager. -func NewConnManager(logger zerolog.Logger, metric module.LibP2PConnectionMetrics, cfg *ManagerConfig) (*ConnManager, error) { +func NewConnManager(logger zerolog.Logger, metric module.LibP2PConnectionMetrics, cfg *netconf.ConnectionManagerConfig) (*ConnManager, error) { basic, err := libp2pconnmgr.NewConnManager( cfg.LowWatermark, cfg.HighWatermark, diff --git a/network/p2p/connection/connManager_test.go b/network/p2p/connection/connManager_test.go index 33808381de0..39016a09fc3 100644 --- a/network/p2p/connection/connManager_test.go +++ b/network/p2p/connection/connManager_test.go @@ -11,13 +11,15 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/config" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/internal/p2pfixtures" + "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p/connection" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/utils" - - "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/utils/unittest" ) @@ -54,8 +56,10 @@ var isNotProtected = fun{ func TestConnectionManagerProtection(t *testing.T) { log := zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) + flowConfig, err := config.DefaultConfig() + require.NoError(t, err) noopMetrics := metrics.NewNoopCollector() - connManager, err := connection.NewConnManager(log, noopMetrics, connection.DefaultConnManagerConfig()) + connManager, err := connection.NewConnManager(log, noopMetrics, &flowConfig.NetworkConfig.ConnectionManagerConfig) require.NoError(t, err) testCases := [][]fun{ @@ -102,7 +106,7 @@ func TestConnectionManager_Watermarking(t *testing.T) { signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) defer cancel() - cfg := &connection.ManagerConfig{ + cfg := &netconf.ConnectionManagerConfig{ HighWatermark: 4, // whenever the number of connections exceeds 4, connection manager prune connections. LowWatermark: 2, // connection manager prune connections until the number of connections is 2. GracePeriod: 500 * time.Millisecond, // extra connections will be pruned if they are older than a second (just for testing). @@ -113,14 +117,16 @@ func TestConnectionManager_Watermarking(t *testing.T) { metrics.NewNoopCollector(), cfg) require.NoError(t, err) - - thisNode, _ := p2ptest.NodeFixture( + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) + thisNode, identity := p2ptest.NodeFixture( t, sporkId, t.Name(), + idProvider, p2ptest.WithConnectionManager(thisConnMgr)) + idProvider.SetIdentities(flow.IdentityList{&identity}) - otherNodes, _ := p2ptest.NodesFixture(t, sporkId, t.Name(), 5) + otherNodes, _ := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, idProvider) nodes := append(otherNodes, thisNode) diff --git a/network/p2p/connection/connection_gater.go b/network/p2p/connection/connection_gater.go index 2ee0df16331..3603d15d227 100644 --- a/network/p2p/connection/connection_gater.go +++ b/network/p2p/connection/connection_gater.go @@ -1,9 +1,9 @@ package connection import ( + "fmt" "sync" - "github.com/libp2p/go-libp2p/core/connmgr" "github.com/libp2p/go-libp2p/core/control" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" @@ -15,7 +15,7 @@ import ( "github.com/onflow/flow-go/utils/logging" ) -var _ connmgr.ConnectionGater = (*ConnGater)(nil) +var _ p2p.ConnectionGater = (*ConnGater)(nil) // ConnGaterOption allow the connection gater to be configured with a list of PeerFilter funcs for a specific conn gater callback. // In the current implementation of the ConnGater the following callbacks can be configured with peer filters. @@ -44,6 +44,11 @@ type ConnGater struct { onInterceptPeerDialFilters []p2p.PeerFilter onInterceptSecuredFilters []p2p.PeerFilter + // disallowListOracle is consulted upon every incoming or outgoing connection attempt, and the connection is only + // allowed if the remote peer is not on the disallow list. + // A ConnGater must have a disallowListOracle set, and if one is not set the ConnGater will panic. + disallowListOracle p2p.DisallowListOracle + // identityProvider provides the identity of a node given its peer ID for logging purposes only. // It is not used for allowlisting or filtering. We use the onInterceptPeerDialFilters and onInterceptSecuredFilters // to determine if a node should be allowed to connect. @@ -68,6 +73,14 @@ func NewConnGater(log zerolog.Logger, identityProvider module.IdentityProvider, func (c *ConnGater) InterceptPeerDial(p peer.ID) bool { lg := c.log.With().Str("peer_id", p.String()).Logger() + disallowListCauses, disallowListed := c.disallowListOracle.IsDisallowListed(p) + if disallowListed { + lg.Warn(). + Str("disallow_list_causes", fmt.Sprintf("%v", disallowListCauses)). + Msg("outbound connection attempt to disallow listed peer is rejected") + return false + } + if len(c.onInterceptPeerDialFilters) == 0 { lg.Warn(). Msg("outbound connection established with no intercept peer dial filters") @@ -119,6 +132,14 @@ func (c *ConnGater) InterceptSecured(dir network.Direction, p peer.ID, addr netw Str("remote_address", addr.RemoteMultiaddr().String()). Logger() + disallowListCauses, disallowListed := c.disallowListOracle.IsDisallowListed(p) + if disallowListed { + lg.Warn(). + Str("disallow_list_causes", fmt.Sprintf("%v", disallowListCauses)). + Msg("inbound connection attempt to disallow listed peer is rejected") + return false + } + if len(c.onInterceptSecuredFilters) == 0 { lg.Warn().Msg("inbound connection established with no intercept secured filters") return true @@ -169,3 +190,28 @@ func (c *ConnGater) peerIDPassesAllFilters(p peer.ID, filters []p2p.PeerFilter) return nil } + +// SetDisallowListOracle sets the disallow list oracle for the connection gater. +// If one is set, the oracle is consulted upon every incoming or outgoing connection attempt, and +// the connection is only allowed if the remote peer is not on the disallow list. +// In Flow blockchain, it is not optional to dismiss the disallow list oracle, and if one is not set +// the connection gater will panic. +// Also, it follows a dependency injection pattern and does not allow to set the disallow list oracle more than once, +// any subsequent calls to this method will result in a panic. +// Args: +// +// oracle: the disallow list oracle to set. +// +// Returns: +// +// none +// +// Panics: +// +// if the disallow list oracle is already set. +func (c *ConnGater) SetDisallowListOracle(oracle p2p.DisallowListOracle) { + if c.disallowListOracle != nil { + panic("disallow list oracle already set") + } + c.disallowListOracle = oracle +} diff --git a/network/p2p/connection/connection_gater_test.go b/network/p2p/connection/connection_gater_test.go index f19c38ebd84..cd240b5293b 100644 --- a/network/p2p/connection/connection_gater_test.go +++ b/network/p2p/connection/connection_gater_test.go @@ -17,9 +17,10 @@ import ( mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" - "github.com/onflow/flow-go/network/internal/testutils" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/connection" mockp2p "github.com/onflow/flow-go/network/p2p/mock" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/unicast/stream" "github.com/onflow/flow-go/utils/unittest" @@ -38,7 +39,8 @@ func TestConnectionGating(t *testing.T) { t, sporkID, t.Name(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + idProvider, + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(p peer.ID) error { if !node1Peers.Has(p) { return fmt.Errorf("id not found: %s", p.String()) } @@ -51,7 +53,8 @@ func TestConnectionGating(t *testing.T) { t, sporkID, t.Name(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + idProvider, + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(p peer.ID) error { if !node2Peers.Has(p) { return fmt.Errorf("id not found: %s", p.String()) } @@ -117,6 +120,7 @@ func TestConnectionGating_ResourceAllocation_AllowListing(t *testing.T) { t, sporkID, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleConsensus)) node2Metrics := mockmodule.NewNetworkMetrics(t) @@ -144,11 +148,12 @@ func TestConnectionGating_ResourceAllocation_AllowListing(t *testing.T) { t, sporkID, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithMetricsCollector(node2Metrics), // we use default resource manager rather than the test resource manager to ensure that the metrics are called. p2ptest.WithDefaultResourceManager(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(p peer.ID) error { return nil // allow all connections. }))) idProvider.On("ByPeerID", node1.Host().ID()).Return(&node1Id, true).Maybe() @@ -179,6 +184,7 @@ func TestConnectionGating_ResourceAllocation_DisAllowListing(t *testing.T) { t, sporkID, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleConsensus)) node2Metrics := mockmodule.NewNetworkMetrics(t) @@ -187,11 +193,12 @@ func TestConnectionGating_ResourceAllocation_DisAllowListing(t *testing.T) { t, sporkID, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithMetricsCollector(node2Metrics), // we use default resource manager rather than the test resource manager to ensure that the metrics are called. p2ptest.WithDefaultResourceManager(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(p peer.ID) error { return fmt.Errorf("disallowed connection") // rejecting all connections. }))) idProvider.On("ByPeerID", node1.Host().ID()).Return(&node1Id, true).Maybe() @@ -231,18 +238,23 @@ func TestConnectionGater_InterceptUpgrade(t *testing.T) { disallowedPeerIds := unittest.NewProtectedMap[peer.ID, struct{}]() allPeerIds := make(peer.IDSlice, 0, count) - + idProvider := mockmodule.NewIdentityProvider(t) connectionGater := mockp2p.NewConnectionGater(t) for i := 0; i < count; i++ { handler, inbound := p2ptest.StreamHandlerFixture(t) - node, _ := p2ptest.NodeFixture( + node, id := p2ptest.NodeFixture( t, sporkId, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithDefaultStreamHandler(handler), // enable peer manager, with a 1-second refresh rate, and connection pruning enabled. - p2ptest.WithPeerManagerEnabled(true, 1*time.Second, func() peer.IDSlice { + p2ptest.WithPeerManagerEnabled(&p2pconfig.PeerManagerConfig{ + ConnectionPruning: true, + UpdateInterval: 1 * time.Second, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + }, func() peer.IDSlice { list := make(peer.IDSlice, 0) for _, pid := range allPeerIds { if !disallowedPeerIds.Has(pid) { @@ -252,7 +264,7 @@ func TestConnectionGater_InterceptUpgrade(t *testing.T) { return list }), p2ptest.WithConnectionGater(connectionGater)) - + idProvider.On("ByPeerID", node.Host().ID()).Return(&id, true).Maybe() nodes = append(nodes, node) allPeerIds = append(allPeerIds, node.Host().ID()) inbounds = append(inbounds, inbound) @@ -316,10 +328,15 @@ func TestConnectionGater_Disallow_Integration(t *testing.T) { t, sporkId, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithDefaultStreamHandler(handler), // enable peer manager, with a 1-second refresh rate, and connection pruning enabled. - p2ptest.WithPeerManagerEnabled(true, 1*time.Second, func() peer.IDSlice { + p2ptest.WithPeerManagerEnabled(&p2pconfig.PeerManagerConfig{ + ConnectionPruning: true, + UpdateInterval: 1 * time.Second, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + }, func() peer.IDSlice { list := make(peer.IDSlice, 0) for _, id := range ids { if disallowedList.Has(id) { @@ -333,7 +350,7 @@ func TestConnectionGater_Disallow_Integration(t *testing.T) { } return list }), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { return disallowedList.ForEach(func(id *flow.Identity, _ struct{}) error { bid, err := unittest.PeerIDFromFlowID(id) require.NoError(t, err) @@ -378,20 +395,21 @@ func TestConnectionGater_Disallow_Integration(t *testing.T) { // ensureCommunicationSilenceAmongGroups ensures no connection, unicast, or pubsub going to or coming from between the two groups of nodes. func ensureCommunicationSilenceAmongGroups(t *testing.T, ctx context.Context, sporkId flow.Identifier, groupA []p2p.LibP2PNode, groupB []p2p.LibP2PNode) { // ensures no connection, unicast, or pubsub going to the disallow-listed nodes - p2pfixtures.EnsureNotConnectedBetweenGroups(t, ctx, groupA, groupB) - p2pfixtures.EnsureNoPubsubExchangeBetweenGroups(t, ctx, groupA, groupB, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - return unittest.ProposalFixture(), blockTopic + p2ptest.EnsureNotConnectedBetweenGroups(t, ctx, groupA, groupB) + + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, groupA, groupB, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) p2pfixtures.EnsureNoStreamCreationBetweenGroups(t, ctx, groupA, groupB) } // ensureCommunicationOverAllProtocols ensures that all nodes are connected to each other, and they can exchange messages over the pubsub and unicast. func ensureCommunicationOverAllProtocols(t *testing.T, ctx context.Context, sporkId flow.Identifier, nodes []p2p.LibP2PNode, inbounds []chan string) { - p2ptest.EnsureConnected(t, ctx, nodes) - p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - return unittest.ProposalFixture(), blockTopic + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) p2pfixtures.EnsureMessageExchangeOverUnicast(t, ctx, nodes, inbounds, p2pfixtures.LongStringMessageFactoryFixture(t)) } diff --git a/network/p2p/connection/connector.go b/network/p2p/connection/connector.go index 5c25921a520..e185d38c69b 100644 --- a/network/p2p/connection/connector.go +++ b/network/p2p/connection/connector.go @@ -2,74 +2,66 @@ package connection import ( "context" - "errors" - "fmt" - "math/rand" - "time" - "github.com/hashicorp/go-multierror" - "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" - discoveryBackoff "github.com/libp2p/go-libp2p/p2p/discovery/backoff" "github.com/rs/zerolog" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/network/internal/p2putils" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/utils/logging" + "github.com/onflow/flow-go/utils/rand" ) const ( - ConnectionPruningEnabled = true - ConnectionPruningDisabled = false + // PruningEnabled is a boolean flag to enable pruning of connections to peers that are not part of + // the explicit update list. + // If set to true, the connector will prune connections to peers that are not part of the explicit update list. + PruningEnabled = true + + // PruningDisabled is a boolean flag to disable pruning of connections to peers that are not part of + // the explicit update list. + // If set to false, the connector will not prune connections to peers that are not part of the explicit update list. + PruningDisabled = false ) -// Libp2pConnector is a libp2p based Connector implementation to connect and disconnect from peers -type Libp2pConnector struct { - backoffConnector *discoveryBackoff.BackoffConnector - host host.Host +// PeerUpdater is a connector that connects to a list of peers and disconnects from any other connection that the libp2p node might have. +type PeerUpdater struct { + connector p2p.Connector + host p2p.ConnectorHost log zerolog.Logger pruneConnections bool } -var _ p2p.Connector = &Libp2pConnector{} +// PeerUpdaterConfig is the configuration for the libp2p based connector. +type PeerUpdaterConfig struct { + // PruneConnections is a boolean flag to enable pruning of connections to peers that are not part of the explicit update list. + PruneConnections bool -// UnconvertibleIdentitiesError is an error which reports all the flow.Identifiers that could not be converted to -// peer.AddrInfo -type UnconvertibleIdentitiesError struct { - errs map[flow.Identifier]error -} + // Logger is the logger to be used by the connector + Logger zerolog.Logger -func NewUnconvertableIdentitiesError(errs map[flow.Identifier]error) error { - return UnconvertibleIdentitiesError{ - errs: errs, - } -} - -func (e UnconvertibleIdentitiesError) Error() string { - multierr := new(multierror.Error) - for id, err := range e.errs { - multierr = multierror.Append(multierr, fmt.Errorf("failed to connect to %s: %w", id.String(), err)) - } - return multierr.Error() -} + // Host is the libp2p host to be used by the connector. + Host p2p.ConnectorHost -// IsUnconvertibleIdentitiesError returns whether the given error is an UnconvertibleIdentitiesError error -func IsUnconvertibleIdentitiesError(err error) bool { - var errUnconvertableIdentitiesError UnconvertibleIdentitiesError - return errors.As(err, &errUnconvertableIdentitiesError) + // ConnectorFactory is a factory function to create a new connector. + Connector p2p.Connector } -func NewLibp2pConnector(log zerolog.Logger, host host.Host, pruning bool) (*Libp2pConnector, error) { - connector, err := defaultLibp2pBackoffConnector(host) - if err != nil { - return nil, fmt.Errorf("failed to create libP2P connector: %w", err) - } - libP2PConnector := &Libp2pConnector{ - log: log, - backoffConnector: connector, - host: host, - pruneConnections: pruning, +var _ p2p.PeerUpdater = (*PeerUpdater)(nil) + +// NewPeerUpdater creates a new libp2p based connector +// Args: +// - cfg: configuration for the connector +// +// Returns: +// - *PeerUpdater: a new libp2p based connector +// - error: an error if there is any error while creating the connector. The errors are irrecoverable and unexpected. +func NewPeerUpdater(cfg *PeerUpdaterConfig) (*PeerUpdater, error) { + libP2PConnector := &PeerUpdater{ + log: cfg.Logger, + connector: cfg.Connector, + host: cfg.Host, + pruneConnections: cfg.PruneConnections, } return libP2PConnector, nil @@ -77,7 +69,7 @@ func NewLibp2pConnector(log zerolog.Logger, host host.Host, pruning bool) (*Libp // UpdatePeers is the implementation of the Connector.UpdatePeers function. It connects to all of the ids and // disconnects from any other connection that the libp2p node might have. -func (l *Libp2pConnector) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) { +func (l *PeerUpdater) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) { // connect to each of the peer.AddrInfo in pInfos l.connectToPeers(ctx, peerIDs) @@ -90,13 +82,26 @@ func (l *Libp2pConnector) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) } // connectToPeers connects each of the peer in pInfos -func (l *Libp2pConnector) connectToPeers(ctx context.Context, peerIDs peer.IDSlice) { +func (l *PeerUpdater) connectToPeers(ctx context.Context, peerIDs peer.IDSlice) { // create a channel of peer.AddrInfo as expected by the connector peerCh := make(chan peer.AddrInfo, len(peerIDs)) - // stuff all the peer.AddrInfo it into the channel + // first shuffle, and then stuff all the peer.AddrInfo it into the channel. + // shuffling is not in place. + err := rand.Shuffle(uint(len(peerIDs)), func(i, j uint) { + peerIDs[i], peerIDs[j] = peerIDs[j], peerIDs[i] + }) + if err != nil { + // this should never happen, but if it does, we should crash. + l.log.Fatal().Err(err).Msg("failed to shuffle peer IDs") + } + for _, peerID := range peerIDs { + if l.host.IsConnectedTo(peerID) { + l.log.Trace().Str("peer_id", peerID.String()).Msg("already connected to peer, skipping connection") + continue + } peerCh <- peer.AddrInfo{ID: peerID} } @@ -104,24 +109,21 @@ func (l *Libp2pConnector) connectToPeers(ctx context.Context, peerIDs peer.IDSli close(peerCh) // ask the connector to connect to all the peers - l.backoffConnector.Connect(ctx, peerCh) + l.connector.Connect(ctx, peerCh) } // pruneAllConnectionsExcept trims all connections of the node from peers not part of peerIDs. // A node would have created such extra connections earlier when the identity list may have been different, or // it may have been target of such connections from node which have now been excluded. -func (l *Libp2pConnector) pruneAllConnectionsExcept(peerIDs peer.IDSlice) { +func (l *PeerUpdater) pruneAllConnectionsExcept(peerIDs peer.IDSlice) { // convert the peerInfos to a peer.ID -> bool map peersToKeep := make(map[peer.ID]bool, len(peerIDs)) for _, pid := range peerIDs { peersToKeep[pid] = true } - // get all current node connections - allCurrentConns := l.host.Network().Conns() - // for each connection, check if that connection should be trimmed - for _, conn := range allCurrentConns { + for _, conn := range l.host.Connections() { // get the remote peer ID for this connection peerID := conn.RemotePeer() @@ -131,11 +133,11 @@ func (l *Libp2pConnector) pruneAllConnectionsExcept(peerIDs peer.IDSlice) { continue // skip pruning } - peerInfo := l.host.Network().Peerstore().PeerInfo(peerID) + peerInfo := l.host.PeerInfo(peerID) lg := l.log.With().Str("remote_peer", peerInfo.String()).Logger() // log the protected status of the connection - protected := l.host.ConnManager().IsProtected(peerID, "") + protected := l.host.IsProtected(peerID) lg = lg.With().Bool("protected", protected).Logger() // log if any stream is open on this connection. @@ -143,9 +145,14 @@ func (l *Libp2pConnector) pruneAllConnectionsExcept(peerIDs peer.IDSlice) { if flowStream != nil { lg = lg.With().Str("flow_stream", string(flowStream.Protocol())).Logger() } + for _, stream := range conn.GetStreams() { + if err := stream.Close(); err != nil { + lg.Warn().Err(err).Msg("failed to close stream, when pruning connections") + } + } // close the connection with the peer if it is not part of the current fanout - err := l.host.Network().ClosePeer(peerID) + err := l.host.ClosePeer(peerID) if err != nil { // logging with suspicious level as failure to disconnect from a peer can be a security issue. // e.g., failure to disconnect from a malicious peer can lead to a DoS attack. @@ -161,18 +168,3 @@ func (l *Libp2pConnector) pruneAllConnectionsExcept(peerIDs peer.IDSlice) { Msg("disconnected from peer") } } - -// defaultLibp2pBackoffConnector creates a default libp2p backoff connector similar to the one created by libp2p.pubsub -// (https://github.com/libp2p/go-libp2p-pubsub/blob/master/discovery.go#L34) -func defaultLibp2pBackoffConnector(host host.Host) (*discoveryBackoff.BackoffConnector, error) { - rngSrc := rand.NewSource(rand.Int63()) - minBackoff, maxBackoff := time.Second*10, time.Hour - cacheSize := 100 - dialTimeout := time.Minute * 2 - backoff := discoveryBackoff.NewExponentialBackoff(minBackoff, maxBackoff, discoveryBackoff.FullJitter, time.Second, 5.0, 0, rand.New(rngSrc)) - backoffConnector, err := discoveryBackoff.NewBackoffConnector(host, cacheSize, dialTimeout, backoff) - if err != nil { - return nil, fmt.Errorf("failed to create backoff connector: %w", err) - } - return backoffConnector, nil -} diff --git a/network/p2p/connection/connector_factory.go b/network/p2p/connection/connector_factory.go new file mode 100644 index 00000000000..10003895953 --- /dev/null +++ b/network/p2p/connection/connector_factory.go @@ -0,0 +1,101 @@ +package connection + +import ( + "crypto/rand" + "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/host" + discoveryBackoff "github.com/libp2p/go-libp2p/p2p/discovery/backoff" + + "github.com/onflow/flow-go/crypto/random" + "github.com/onflow/flow-go/network/p2p" +) + +const ( + // minBackoff is the minimum backoff duration for the backoff connector. + // We set it to 1 second as we want to let the LibP2PNode be in charge of connection establishment and can disconnect + // and reconnect to peers as soon as it needs. This is essential to ensure that the allow-listing and disallow-listing + // time intervals are working as expected. + minBackoff = 1 * time.Second + // maxBackoff is the maximum backoff duration for the backoff connector. When the backoff duration reaches this value, + // it will not increase any further. + maxBackoff = time.Hour + // timeUnit is the time unit for the backoff duration. The backoff duration will be a multiple of this value. + // As we use an exponential backoff, the backoff duration will be a multiple of this value multiplied by the exponential + // base raised to the exponential offset. + timeUnit = time.Second + // exponentialBackOffBase is the base for the exponential backoff. The backoff duration will be a multiple of the time unit + // multiplied by the exponential base raised to the exponential offset, i.e., exponentialBase^(timeUnit*attempt). + exponentialBackOffBase = 2.0 + // exponentialBackOffOffset is the offset for the exponential backoff. It acts as a constant that is added result + // of the exponential base raised to the exponential offset, i.e., exponentialBase^(timeUnit*attempt) + exponentialBackOffOffset. + // This is used to ensure that the backoff duration is always greater than the time unit. We set this to 0 as we want the + // backoff duration to be a multiple of the time unit. + exponentialBackOffOffset = 0 +) + +// DefaultLibp2pBackoffConnectorFactory is a factory function to create a new BackoffConnector. It uses the default +// values for the backoff connector. +// (https://github.com/libp2p/go-libp2p-pubsub/blob/master/discovery.go#L34) +func DefaultLibp2pBackoffConnectorFactory() p2p.ConnectorFactory { + return func(host host.Host) (p2p.Connector, error) { + rngSrc, err := newSource() + if err != nil { + return nil, fmt.Errorf("failed to generate a random source: %w", err) + } + + cacheSize := 100 + dialTimeout := time.Minute * 2 + backoff := discoveryBackoff.NewExponentialBackoff( + minBackoff, + maxBackoff, + discoveryBackoff.FullJitter, + timeUnit, + exponentialBackOffBase, + exponentialBackOffOffset, + rngSrc, + ) + backoffConnector, err := discoveryBackoff.NewBackoffConnector(host, cacheSize, dialTimeout, backoff) + if err != nil { + return nil, fmt.Errorf("failed to create backoff connector: %w", err) + } + return backoffConnector, nil + } +} + +// `source` implements math/rand.Source so it can be used +// by libp2p's `NewExponentialBackoff`. +// It is backed by a more secure randomness than math/rand's `NewSource`. +// `source` is only implemented to avoid using math/rand's `NewSource`. +type source struct { + prg random.Rand +} + +// Seed is not used by the backoff object from `NewExponentialBackoff` +func (src *source) Seed(seed int64) {} + +// Int63 is used by `NewExponentialBackoff` and is based on a crypto PRG +func (src *source) Int63() int64 { + return int64(src.prg.UintN(1 << 63)) +} + +// creates a source using a crypto PRG and secure random seed +// returned errors: +// - exception error if the system randomness fails (the system and other components would +// have many other issues if this happens) +// - exception error if the CSPRG (Chacha20) isn't initialized properly (should not happen in normal +// operations) +func newSource() (*source, error) { + seed := make([]byte, random.Chacha20SeedLen) + _, err := rand.Read(seed) // checking err only is enough + if err != nil { + return nil, fmt.Errorf("failed to generate a seed: %w", err) + } + prg, err := random.NewChacha20PRG(seed, nil) + if err != nil { + // should not happen in normal operations because `seed` has the correct length + return nil, fmt.Errorf("failed to generate a PRG: %w", err) + } + return &source{prg}, nil +} diff --git a/network/p2p/connection/connector_host.go b/network/p2p/connection/connector_host.go new file mode 100644 index 00000000000..04cfd56b28a --- /dev/null +++ b/network/p2p/connection/connector_host.go @@ -0,0 +1,86 @@ +package connection + +import ( + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/network/p2p" +) + +// ConnectorHost is a wrapper around the libp2p host.Host interface to provide the required functionality for the +// Connector interface. +type ConnectorHost struct { + h host.Host +} + +var _ p2p.ConnectorHost = (*ConnectorHost)(nil) + +func NewConnectorHost(h host.Host) *ConnectorHost { + return &ConnectorHost{ + h: h, + } +} + +// Connections returns all the connections of the underlying host. +func (c *ConnectorHost) Connections() []network.Conn { + return c.h.Network().Conns() +} + +// IsConnectedTo returns true if the given peer.ID is connected to the underlying host. +// Args: +// +// peerID: peer.ID for which the connection status is requested +// +// Returns: +// +// true if the given peer.ID is connected to the underlying host. +func (c *ConnectorHost) IsConnectedTo(peerID peer.ID) bool { + return c.h.Network().Connectedness(peerID) == network.Connected && len(c.h.Network().ConnsToPeer(peerID)) > 0 +} + +// PeerInfo returns the peer.AddrInfo for the given peer.ID. +// Args: +// +// id: peer.ID for which the peer.AddrInfo is requested +// +// Returns: +// +// peer.AddrInfo for the given peer.ID +func (c *ConnectorHost) PeerInfo(id peer.ID) peer.AddrInfo { + return c.h.Peerstore().PeerInfo(id) +} + +// IsProtected returns true if the given peer.ID is protected from pruning. +// Args: +// +// id: peer.ID for which the protection status is requested +// +// Returns: +// +// true if the given peer.ID is protected from pruning +func (c *ConnectorHost) IsProtected(id peer.ID) bool { + return c.h.ConnManager().IsProtected(id, "") +} + +// ClosePeer closes the connection to the given peer.ID. +// Args: +// +// id: peer.ID for which the connection is to be closed +// +// Returns: +// +// error if there is any error while closing the connection to the given peer.ID. All errors are benign. +func (c *ConnectorHost) ClosePeer(id peer.ID) error { + return c.h.Network().ClosePeer(id) +} + +// ID returns the peer.ID of the underlying host. +// Returns: +// +// peer.ID of the underlying host. +// +// ID returns the peer.ID of the underlying host. +func (c *ConnectorHost) ID() peer.ID { + return c.h.ID() +} diff --git a/network/p2p/connection/peerManager.go b/network/p2p/connection/peerManager.go index d82c5b779b6..11fe502a07c 100644 --- a/network/p2p/connection/peerManager.go +++ b/network/p2p/connection/peerManager.go @@ -3,7 +3,7 @@ package connection import ( "context" "fmt" - mrand "math/rand" + "sync" "time" @@ -14,10 +14,12 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/utils/logging" + "github.com/onflow/flow-go/utils/rand" ) -// DefaultPeerUpdateInterval is default duration for which the peer manager waits in between attempts to update peer connections -var DefaultPeerUpdateInterval = 10 * time.Minute +// DefaultPeerUpdateInterval is default duration for which the peer manager waits in between attempts to update peer connections. +// We set it to 1 second to be aligned with the heartbeat intervals of libp2p, alsp, and gossipsub. +var DefaultPeerUpdateInterval = time.Second var _ p2p.PeerManager = (*PeerManager)(nil) var _ component.Component = (*PeerManager)(nil) @@ -30,7 +32,7 @@ type PeerManager struct { logger zerolog.Logger peersProvider p2p.PeersProvider // callback to retrieve list of peers to connect to peerRequestQ chan struct{} // a channel to queue a peer update request - connector p2p.Connector // connector to connect or disconnect from peers + connector p2p.PeerUpdater // connector to connect or disconnect from peers peerUpdateInterval time.Duration // interval the peer manager runs on peersProviderMu sync.RWMutex @@ -38,9 +40,9 @@ type PeerManager struct { // NewPeerManager creates a new peer manager which calls the peersProvider callback to get a list of peers to connect to // and it uses the connector to actually connect or disconnect from peers. -func NewPeerManager(logger zerolog.Logger, updateInterval time.Duration, connector p2p.Connector) *PeerManager { +func NewPeerManager(logger zerolog.Logger, updateInterval time.Duration, connector p2p.PeerUpdater) *PeerManager { pm := &PeerManager{ - logger: logger, + logger: logger.With().Str("component", "peer-manager").Logger(), connector: connector, peerRequestQ: make(chan struct{}, 1), peerUpdateInterval: updateInterval, @@ -85,7 +87,11 @@ func (pm *PeerManager) updateLoop(ctx irrecoverable.SignalerContext) { func (pm *PeerManager) periodicLoop(ctx irrecoverable.SignalerContext) { // add a random delay to initial launch to avoid synchronizing this // potentially expensive operation across the network - delay := time.Duration(mrand.Int63n(pm.peerUpdateInterval.Nanoseconds())) + r, err := rand.Uint64n(uint64(pm.peerUpdateInterval.Nanoseconds())) + if err != nil { + ctx.Throw(fmt.Errorf("unable to generate random interval: %w", err)) + } + delay := time.Duration(r) ticker := time.NewTicker(pm.peerUpdateInterval) defer ticker.Stop() diff --git a/network/p2p/connection/peerManager_integration_test.go b/network/p2p/connection/peerManager_integration_test.go index b711c62ba65..684acfbdb8a 100644 --- a/network/p2p/connection/peerManager_integration_test.go +++ b/network/p2p/connection/peerManager_integration_test.go @@ -22,7 +22,7 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -// TestPeerManager_Integration tests the correctness of integration between PeerManager and Libp2pConnector over +// TestPeerManager_Integration tests the correctness of integration between PeerManager and PeerUpdater over // a fully connected topology. // PeerManager should be able to connect to all peers using the connector, and must also tear down the connection to // peers that are excluded from its identity provider. @@ -33,8 +33,9 @@ func TestPeerManager_Integration(t *testing.T) { signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) // create nodes - nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_peer_manager", count) - + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) + nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_peer_manager", count, idProvider) + idProvider.SetIdentities(identities) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -48,14 +49,21 @@ func TestPeerManager_Integration(t *testing.T) { thisNode.Host().Peerstore().SetAddrs(i.ID, i.Addrs, peerstore.PermanentAddrTTL) } + connector, err := connection.DefaultLibp2pBackoffConnectorFactory()(thisNode.Host()) + require.NoError(t, err) // setup - connector, err := connection.NewLibp2pConnector(unittest.Logger(), thisNode.Host(), connection.ConnectionPruningEnabled) + peerUpdater, err := connection.NewPeerUpdater(&connection.PeerUpdaterConfig{ + PruneConnections: connection.PruningEnabled, + Logger: unittest.Logger(), + Host: connection.NewConnectorHost(thisNode.Host()), + Connector: connector, + }) require.NoError(t, err) idTranslator, err := translator.NewFixedTableIdentityTranslator(identities) require.NoError(t, err) - peerManager := connection.NewPeerManager(unittest.Logger(), connection.DefaultPeerUpdateInterval, connector) + peerManager := connection.NewPeerManager(unittest.Logger(), connection.DefaultPeerUpdateInterval, peerUpdater) peerManager.SetPeersProvider(func() peer.IDSlice { // peerManager is furnished with a full topology that connects to all nodes // in the topologyPeers. diff --git a/network/p2p/connection/peerManager_test.go b/network/p2p/connection/peerManager_test.go index f2a9305c31b..2b4c8e367e9 100644 --- a/network/p2p/connection/peerManager_test.go +++ b/network/p2p/connection/peerManager_test.go @@ -18,9 +18,9 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/internal/p2pfixtures" - "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/keyutils" + mockp2p "github.com/onflow/flow-go/network/p2p/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -60,8 +60,8 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { pids := suite.generatePeerIDs(10) // create the connector mock to check ids requested for connect and disconnect - connector := new(mocknetwork.Connector) - connector.On("UpdatePeers", mock.Anything, mock.AnythingOfType("peer.IDSlice")). + peerUpdater := mockp2p.NewPeerUpdater(suite.T()) + peerUpdater.On("UpdatePeers", mock.Anything, mock.AnythingOfType("peer.IDSlice")). Run(func(args mock.Arguments) { idArg := args[1].(peer.IDSlice) assert.ElementsMatch(suite.T(), pids, idArg) @@ -69,7 +69,7 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { Return(nil) // create the peer manager (but don't start it) - pm := connection.NewPeerManager(suite.log, connection.DefaultPeerUpdateInterval, connector) + pm := connection.NewPeerManager(suite.log, connection.DefaultPeerUpdateInterval, peerUpdater) pm.SetPeersProvider(func() peer.IDSlice { return pids }) @@ -77,7 +77,7 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { // very first call to updatepeer suite.Run("updatePeers only connects to all peers the first time", func() { pm.ForceUpdatePeers(ctx) - connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 1) + peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 1) }) // a subsequent call to updatePeers should request a connector.UpdatePeers to existing ids and new ids @@ -87,7 +87,7 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { pids = append(pids, newPIDs...) pm.ForceUpdatePeers(ctx) - connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 2) + peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 2) }) // when ids are only excluded, connector.UpdatePeers should be called @@ -96,7 +96,7 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { pids = removeRandomElement(pids) pm.ForceUpdatePeers(ctx) - connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 3) + peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 3) }) // addition and deletion of ids should result in a call to connector.UpdatePeers @@ -111,7 +111,7 @@ func (suite *PeerManagerTestSuite) TestUpdatePeers() { pm.ForceUpdatePeers(ctx) - connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 4) + peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 4) }) } @@ -131,13 +131,13 @@ func (suite *PeerManagerTestSuite) TestPeriodicPeerUpdate() { // create some test ids pids := suite.generatePeerIDs(10) - connector := new(mocknetwork.Connector) + peerUpdater := mockp2p.NewPeerUpdater(suite.T()) wg := &sync.WaitGroup{} // keeps track of number of calls on `ConnectPeers` mu := &sync.Mutex{} // provides mutual exclusion on calls to `ConnectPeers` count := 0 times := 2 // we expect it to be called twice at least wg.Add(times) - connector.On("UpdatePeers", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + peerUpdater.On("UpdatePeers", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { mu.Lock() defer mu.Unlock() @@ -148,7 +148,7 @@ func (suite *PeerManagerTestSuite) TestPeriodicPeerUpdate() { }).Return(nil) peerUpdateInterval := 10 * time.Millisecond - pm := connection.NewPeerManager(suite.log, peerUpdateInterval, connector) + pm := connection.NewPeerManager(suite.log, peerUpdateInterval, peerUpdater) pm.SetPeersProvider(func() peer.IDSlice { return pids }) @@ -173,15 +173,15 @@ func (suite *PeerManagerTestSuite) TestOnDemandPeerUpdate() { // chooses peer interval rate deliberately long to capture on demand peer update peerUpdateInterval := time.Hour - // creates mock connector + // creates mock peerUpdater wg := &sync.WaitGroup{} // keeps track of number of calls on `ConnectPeers` mu := &sync.Mutex{} // provides mutual exclusion on calls to `ConnectPeers` count := 0 times := 2 // we expect it to be called twice overall wg.Add(1) // this accounts for one invocation, the other invocation is subsequent - connector := new(mocknetwork.Connector) + peerUpdater := mockp2p.NewPeerUpdater(suite.T()) // captures the first periodic update initiated after start to complete - connector.On("UpdatePeers", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + peerUpdater.On("UpdatePeers", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { mu.Lock() defer mu.Unlock() @@ -191,7 +191,7 @@ func (suite *PeerManagerTestSuite) TestOnDemandPeerUpdate() { } }).Return(nil) - pm := connection.NewPeerManager(suite.log, peerUpdateInterval, connector) + pm := connection.NewPeerManager(suite.log, peerUpdateInterval, peerUpdater) pm.SetPeersProvider(func() peer.IDSlice { return pids }) @@ -220,17 +220,17 @@ func (suite *PeerManagerTestSuite) TestConcurrentOnDemandPeerUpdate() { // create some test ids pids := suite.generatePeerIDs(10) - connector := new(mocknetwork.Connector) - // connectPeerGate channel gates the return of the connector + peerUpdater := mockp2p.NewPeerUpdater(suite.T()) + // connectPeerGate channel gates the return of the peerUpdater connectPeerGate := make(chan time.Time) defer close(connectPeerGate) // choose the periodic interval as a high value so that periodic runs don't interfere with this test peerUpdateInterval := time.Hour - connector.On("UpdatePeers", mock.Anything, mock.Anything).Return(nil). + peerUpdater.On("UpdatePeers", mock.Anything, mock.Anything).Return(nil). WaitUntil(connectPeerGate) // blocks call for connectPeerGate channel - pm := connection.NewPeerManager(suite.log, peerUpdateInterval, connector) + pm := connection.NewPeerManager(suite.log, peerUpdateInterval, peerUpdater) pm.SetPeersProvider(func() peer.IDSlice { return pids }) @@ -243,7 +243,7 @@ func (suite *PeerManagerTestSuite) TestConcurrentOnDemandPeerUpdate() { // assert that the first update started assert.Eventually(suite.T(), func() bool { - return connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 1) + return peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 1) }, 3*time.Second, 100*time.Millisecond) // makes 10 concurrent request for peer update @@ -255,6 +255,6 @@ func (suite *PeerManagerTestSuite) TestConcurrentOnDemandPeerUpdate() { // assert that only two calls to UpdatePeers were made (one by the periodic update and one by the on-demand update) assert.Eventually(suite.T(), func() bool { - return connector.AssertNumberOfCalls(suite.T(), "UpdatePeers", 2) + return peerUpdater.AssertNumberOfCalls(suite.T(), "UpdatePeers", 2) }, 10*time.Second, 100*time.Millisecond) } diff --git a/network/p2p/connectionGater.go b/network/p2p/connectionGater.go index d2732fbd713..212dea51102 100644 --- a/network/p2p/connectionGater.go +++ b/network/p2p/connectionGater.go @@ -1,23 +1,24 @@ package p2p -import ( - "github.com/libp2p/go-libp2p/core/control" - "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multiaddr" -) +import "github.com/libp2p/go-libp2p/core/connmgr" -// ConnectionGater is a copy of the libp2p ConnectionGater interface: -// https://github.com/libp2p/go-libp2p/blob/master/core/connmgr/gater.go#L54 -// We use it here to generate a mock for testing through testify mock. +// ConnectionGater the customized interface for the connection gater in the p2p package. +// It acts as a wrapper around the libp2p connmgr.ConnectionGater interface and adds some custom methods. type ConnectionGater interface { - InterceptPeerDial(p peer.ID) (allow bool) + connmgr.ConnectionGater - InterceptAddrDial(peer.ID, multiaddr.Multiaddr) (allow bool) - - InterceptAccept(network.ConnMultiaddrs) (allow bool) - - InterceptSecured(network.Direction, peer.ID, network.ConnMultiaddrs) (allow bool) - - InterceptUpgraded(network.Conn) (allow bool, reason control.DisconnectReason) + // SetDisallowListOracle sets the disallow list oracle for the connection gater. + // If one is set, the oracle is consulted upon every incoming or outgoing connection attempt, and + // the connection is only allowed if the remote peer is not on the disallow list. + // In Flow blockchain, it is not optional to dismiss the disallow list oracle, and if one is not set + // the connection gater will panic. + // Also, it follows a dependency injection pattern and does not allow to set the disallow list oracle more than once, + // any subsequent calls to this method will result in a panic. + // Args: + // oracle: the disallow list oracle to set. + // Returns: + // none + // Panics: + // if the disallow list oracle is already set. + SetDisallowListOracle(oracle DisallowListOracle) } diff --git a/network/p2p/connector.go b/network/p2p/connector.go index 3bc4dd3df74..f9c5897352c 100644 --- a/network/p2p/connector.go +++ b/network/p2p/connector.go @@ -3,11 +3,14 @@ package p2p import ( "context" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" ) -// Connector connects to peer and disconnects from peer using the underlying networking library -type Connector interface { +// PeerUpdater connects to the given peer.IDs. It also disconnects from any other peers with which it may have +// previously established connection. +type PeerUpdater interface { // UpdatePeers connects to the given peer.IDs. It also disconnects from any other peers with which it may have // previously established connection. // UpdatePeers implementation should be idempotent such that multiple calls to connect to the same peer should not @@ -15,6 +18,22 @@ type Connector interface { UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) } +// Connector is an interface that allows connecting to a peer.ID. +type Connector interface { + // Connect connects to the given peer.ID. + // Note that connection may be established asynchronously. Any error encountered while connecting to the peer.ID + // is benign and should not be returned. Also, Connect implementation should not cause any blocking or crash. + // Args: + // ctx: context.Context to be used for the connection + // peerChan: channel to which the peer.AddrInfo of the connected peer.ID is sent. + // Returns: + // none. + Connect(ctx context.Context, peerChan <-chan peer.AddrInfo) +} + +// ConnectorFactory is a factory function to create a new Connector. +type ConnectorFactory func(host host.Host) (Connector, error) + type PeerFilter func(peer.ID) error // AllowAllPeerFilter returns a peer filter that does not do any filtering. @@ -23,3 +42,43 @@ func AllowAllPeerFilter() PeerFilter { return nil } } + +// ConnectorHost is a wrapper around the libp2p host.Host interface to provide the required functionality for the +// Connector interface. +type ConnectorHost interface { + // Connections returns all the connections of the underlying host. + Connections() []network.Conn + + // IsConnectedTo returns true if the given peer.ID is connected to the underlying host. + // Args: + // peerID: peer.ID for which the connection status is requested + // Returns: + // true if the given peer.ID is connected to the underlying host. + IsConnectedTo(peerId peer.ID) bool + + // PeerInfo returns the peer.AddrInfo for the given peer.ID. + // Args: + // id: peer.ID for which the peer.AddrInfo is requested + // Returns: + // peer.AddrInfo for the given peer.ID + PeerInfo(peerId peer.ID) peer.AddrInfo + + // IsProtected returns true if the given peer.ID is protected from pruning. + // Args: + // id: peer.ID for which the protection status is requested + // Returns: + // true if the given peer.ID is protected from pruning + IsProtected(peerId peer.ID) bool + + // ClosePeer closes the connection to the given peer.ID. + // Args: + // id: peer.ID for which the connection is to be closed + // Returns: + // error if there is any error while closing the connection to the given peer.ID. All errors are benign. + ClosePeer(peerId peer.ID) error + + // ID returns the peer.ID of the underlying host. + // Returns: + // peer.ID of the underlying host. + ID() peer.ID +} diff --git a/network/p2p/consumer.go b/network/p2p/consumer.go deleted file mode 100644 index 4d9869b7111..00000000000 --- a/network/p2p/consumer.go +++ /dev/null @@ -1,110 +0,0 @@ -package p2p - -import ( - "github.com/libp2p/go-libp2p/core/peer" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/component" -) - -// DisallowListConsumer consumes notifications from the cache.NodeBlocklistWrapper whenever the block list is updated. -// Implementations must: -// - be concurrency safe -// - be non-blocking -type DisallowListConsumer interface { - // OnNodeDisallowListUpdate notifications whenever the node block list is updated. - // Prerequisites: - // Implementation must be concurrency safe; Non-blocking; - // and must handle repetition of the same events (with some processing overhead). - OnNodeDisallowListUpdate(list flow.IdentifierList) -} - -// ControlMessageType is the type of control message, as defined in the libp2p pubsub spec. -type ControlMessageType string - -const ( - CtrlMsgIHave ControlMessageType = "IHAVE" - CtrlMsgIWant ControlMessageType = "IWANT" - CtrlMsgGraft ControlMessageType = "GRAFT" - CtrlMsgPrune ControlMessageType = "PRUNE" -) - -// ControlMessageTypes returns list of all libp2p control message types. -func ControlMessageTypes() []ControlMessageType { - return []ControlMessageType{CtrlMsgIHave, CtrlMsgIWant, CtrlMsgGraft, CtrlMsgPrune} -} - -// DisallowListUpdateNotification is the event that is submitted to the distributor when the disallow list is updated. -type DisallowListUpdateNotification struct { - DisallowList flow.IdentifierList -} - -type DisallowListNotificationConsumer interface { - // OnDisallowListNotification is called when a new disallow list update notification is distributed. - // Any error on consuming event must handle internally. - // The implementation must be concurrency safe, but can be blocking. - OnDisallowListNotification(*DisallowListUpdateNotification) -} - -type DisallowListNotificationDistributor interface { - component.Component - // DistributeBlockListNotification distributes the event to all the consumers. - // Any error returned by the distributor is non-recoverable and will cause the node to crash. - // Implementation must be concurrency safe, and non-blocking. - DistributeBlockListNotification(list flow.IdentifierList) error - - // AddConsumer adds a consumer to the distributor. The consumer will be called the distributor distributes a new event. - // AddConsumer must be concurrency safe. Once a consumer is added, it must be called for all future events. - // There is no guarantee that the consumer will be called for events that were already received by the distributor. - AddConsumer(DisallowListNotificationConsumer) -} - -// GossipSubInspectorNotificationDistributor is the interface for the distributor that distributes gossip sub inspector notifications. -// It is used to distribute notifications to the consumers in an asynchronous manner and non-blocking manner. -// The implementation should guarantee that all registered consumers are called upon distribution of a new event. -type GossipSubInspectorNotificationDistributor interface { - component.Component - // DistributeInvalidControlMessageNotification distributes the event to all the consumers. - // Any error returned by the distributor is non-recoverable and will cause the node to crash. - // Implementation must be concurrency safe, and non-blocking. - DistributeInvalidControlMessageNotification(notification *InvalidControlMessageNotification) error - - // AddConsumer adds a consumer to the distributor. The consumer will be called the distributor distributes a new event. - // AddConsumer must be concurrency safe. Once a consumer is added, it must be called for all future events. - // There is no guarantee that the consumer will be called for events that were already received by the distributor. - AddConsumer(GossipSubInvalidControlMessageNotificationConsumer) -} - -// InvalidControlMessageNotification is the notification sent to the consumer when an invalid control message is received. -// It models the information that is available to the consumer about a misbehaving peer. -type InvalidControlMessageNotification struct { - // PeerID is the ID of the peer that sent the invalid control message. - PeerID peer.ID - // MsgType is the type of control message that was received. - MsgType ControlMessageType - // Count is the number of invalid control messages received from the peer that is reported in this notification. - Count uint64 - // Err any error associated with the invalid control message. - Err error -} - -// NewInvalidControlMessageNotification returns a new *InvalidControlMessageNotification -func NewInvalidControlMessageNotification(peerID peer.ID, msgType ControlMessageType, count uint64, err error) *InvalidControlMessageNotification { - return &InvalidControlMessageNotification{ - PeerID: peerID, - MsgType: msgType, - Count: count, - Err: err, - } -} - -// GossipSubInvalidControlMessageNotificationConsumer is the interface for the consumer that consumes gossip sub inspector notifications. -// It is used to consume notifications in an asynchronous manner. -// The implementation must be concurrency safe, but can be blocking. This is due to the fact that the consumer is called -// asynchronously by the distributor. -type GossipSubInvalidControlMessageNotificationConsumer interface { - // OnInvalidControlMessageNotification is called when a new invalid control message notification is distributed. - // Any error on consuming event must handle internally. - // The implementation must be concurrency safe, but can be blocking. - OnInvalidControlMessageNotification(*InvalidControlMessageNotification) -} diff --git a/network/p2p/consumers.go b/network/p2p/consumers.go new file mode 100644 index 00000000000..356bb410d81 --- /dev/null +++ b/network/p2p/consumers.go @@ -0,0 +1,76 @@ +package p2p + +import ( + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/module/component" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" +) + +// GossipSubInspectorNotifDistributor is the interface for the distributor that distributes gossip sub inspector notifications. +// It is used to distribute notifications to the consumers in an asynchronous manner and non-blocking manner. +// The implementation should guarantee that all registered consumers are called upon distribution of a new event. +type GossipSubInspectorNotifDistributor interface { + component.Component + // Distribute distributes the event to all the consumers. + // Any error returned by the distributor is non-recoverable and will cause the node to crash. + // Implementation must be concurrency safe, and non-blocking. + Distribute(notification *InvCtrlMsgNotif) error + + // AddConsumer adds a consumer to the distributor. The consumer will be called the distributor distributes a new event. + // AddConsumer must be concurrency safe. Once a consumer is added, it must be called for all future events. + // There is no guarantee that the consumer will be called for events that were already received by the distributor. + AddConsumer(GossipSubInvCtrlMsgNotifConsumer) +} + +// InvCtrlMsgNotif is the notification sent to the consumer when an invalid control message is received. +// It models the information that is available to the consumer about a misbehaving peer. +type InvCtrlMsgNotif struct { + // PeerID is the ID of the peer that sent the invalid control message. + PeerID peer.ID + // MsgType is the type of control message that was received. + MsgType p2pmsg.ControlMessageType + // Count is the number of invalid control messages received from the peer that is reported in this notification. + Count uint64 + // Err any error associated with the invalid control message. + Err error +} + +// NewInvalidControlMessageNotification returns a new *InvCtrlMsgNotif +func NewInvalidControlMessageNotification(peerID peer.ID, msgType p2pmsg.ControlMessageType, count uint64, err error) *InvCtrlMsgNotif { + return &InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: msgType, + Count: count, + Err: err, + } +} + +// GossipSubInvCtrlMsgNotifConsumer is the interface for the consumer that consumes gossip sub inspector notifications. +// It is used to consume notifications in an asynchronous manner. +// The implementation must be concurrency safe, but can be blocking. This is due to the fact that the consumer is called +// asynchronously by the distributor. +type GossipSubInvCtrlMsgNotifConsumer interface { + // OnInvalidControlMessageNotification is called when a new invalid control message notification is distributed. + // Any error on consuming event must handle internally. + // The implementation must be concurrency safe, but can be blocking. + OnInvalidControlMessageNotification(*InvCtrlMsgNotif) +} + +// GossipSubInspectorSuite is the interface for the GossipSub inspector suite. +// It encapsulates the rpc inspectors and the notification distributors. +type GossipSubInspectorSuite interface { + component.Component + CollectionClusterChangesConsumer + // InspectFunc returns the inspect function that is used to inspect the gossipsub rpc messages. + // This function follows a dependency injection pattern, where the inspect function is injected into the gossipsu, and + // is called whenever a gossipsub rpc message is received. + InspectFunc() func(peer.ID, *pubsub.RPC) error + + // AddInvCtrlMsgNotifConsumer adds a consumer to the invalid control message notification distributor. + // This consumer is notified when a misbehaving peer regarding gossipsub control messages is detected. This follows a pub/sub + // pattern where the consumer is notified when a new notification is published. + // A consumer is only notified once for each notification, and only receives notifications that were published after it was added. + AddInvalidControlMessageConsumer(GossipSubInvCtrlMsgNotifConsumer) +} diff --git a/network/p2p/dht/dht_test.go b/network/p2p/dht/dht_test.go index bc0cc970fd9..5ea0ee70e6a 100644 --- a/network/p2p/dht/dht_test.go +++ b/network/p2p/dht/dht_test.go @@ -11,8 +11,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/model/flow" libp2pmsg "github.com/onflow/flow-go/model/libp2p/message" "github.com/onflow/flow-go/module/irrecoverable" + mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/p2p" @@ -35,12 +37,14 @@ func TestFindPeerWithDHT(t *testing.T) { golog.SetAllLoggers(golog.LevelFatal) // change this to Debug if libp2p logs are needed sporkId := unittest.IdentifierFixture() - dhtServerNodes, _ := p2ptest.NodesFixture(t, sporkId, "dht_test", 2, p2ptest.WithDHTOptions(dht.AsServer())) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) + dhtServerNodes, serverIDs := p2ptest.NodesFixture(t, sporkId, "dht_test", 2, idProvider, p2ptest.WithDHTOptions(dht.AsServer())) require.Len(t, dhtServerNodes, 2) - dhtClientNodes, _ := p2ptest.NodesFixture(t, sporkId, "dht_test", count-2, p2ptest.WithDHTOptions(dht.AsClient())) + dhtClientNodes, clientIDs := p2ptest.NodesFixture(t, sporkId, "dht_test", count-2, idProvider, p2ptest.WithDHTOptions(dht.AsClient())) nodes := append(dhtServerNodes, dhtClientNodes...) + idProvider.SetIdentities(append(serverIDs, clientIDs...)) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -118,15 +122,21 @@ func TestPubSubWithDHTDiscovery(t *testing.T) { // N4 N5 N4-----N5 sporkId := unittest.IdentifierFixture() + idProvider := mockmodule.NewIdentityProvider(t) // create one node running the DHT Server (mimicking the staked AN) - dhtServerNodes, _ := p2ptest.NodesFixture(t, sporkId, "dht_test", 1, p2ptest.WithDHTOptions(dht.AsServer())) + dhtServerNodes, serverIDs := p2ptest.NodesFixture(t, sporkId, "dht_test", 1, idProvider, p2ptest.WithDHTOptions(dht.AsServer())) require.Len(t, dhtServerNodes, 1) dhtServerNode := dhtServerNodes[0] // crate other nodes running the DHT Client (mimicking the unstaked ANs) - dhtClientNodes, _ := p2ptest.NodesFixture(t, sporkId, "dht_test", count-1, p2ptest.WithDHTOptions(dht.AsClient())) + dhtClientNodes, clientIDs := p2ptest.NodesFixture(t, sporkId, "dht_test", count-1, idProvider, p2ptest.WithDHTOptions(dht.AsClient())) + ids := append(serverIDs, clientIDs...) nodes := append(dhtServerNodes, dhtClientNodes...) + for i, node := range nodes { + idProvider.On("ByPeerID", node.Host().ID()).Return(&ids[i], true).Maybe() + + } p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) diff --git a/network/p2p/disallowListCache.go b/network/p2p/disallowListCache.go new file mode 100644 index 00000000000..b153084b6cf --- /dev/null +++ b/network/p2p/disallowListCache.go @@ -0,0 +1,51 @@ +package p2p + +import ( + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network" +) + +// DisallowListCache is an interface for a cache that keeps the list of disallow-listed peers. +// It is designed to present a centralized interface for keeping track of disallow-listed peers for different reasons. +type DisallowListCache interface { + // IsDisallowListed determines whether the given peer is disallow-listed for any reason. + // Args: + // - peerID: the peer to check. + // Returns: + // - []network.DisallowListedCause: the list of causes for which the given peer is disallow-listed. If the peer is not disallow-listed for any reason, + // a nil slice is returned. + // - bool: true if the peer is disallow-listed for any reason, false otherwise. + IsDisallowListed(peerID peer.ID) ([]network.DisallowListedCause, bool) + + // DisallowFor disallow-lists a peer for a cause. + // Args: + // - peerID: the peerID of the peer to be disallow-listed. + // - cause: the cause for disallow-listing the peer. + // Returns: + // - the list of causes for which the peer is disallow-listed. + // - error if the operation fails, error is irrecoverable. + DisallowFor(peerID peer.ID, cause network.DisallowListedCause) ([]network.DisallowListedCause, error) + + // AllowFor removes a cause from the disallow list cache entity for the peerID. + // Args: + // - peerID: the peerID of the peer to be allow-listed. + // - cause: the cause for allow-listing the peer. + // Returns: + // - the list of causes for which the peer is disallow-listed. If the peer is not disallow-listed for any reason, + // an empty list is returned. + AllowFor(peerID peer.ID, cause network.DisallowListedCause) []network.DisallowListedCause +} + +// DisallowListCacheConfig is the configuration for the disallow-list cache. +// The disallow-list cache is used to temporarily disallow-list peers. +type DisallowListCacheConfig struct { + // MaxSize is the maximum number of peers that can be disallow-listed at any given time. + // When the cache is full, no further new peers can be disallow-listed. + // Recommended size is 100 * number of staked nodes. + MaxSize uint32 + + // Metrics is the HeroCache metrics collector to be used for the disallow-list cache. + Metrics module.HeroCacheMetrics +} diff --git a/network/p2p/distributor/disallow_list.go b/network/p2p/distributor/disallow_list.go deleted file mode 100644 index 848baa925bb..00000000000 --- a/network/p2p/distributor/disallow_list.go +++ /dev/null @@ -1,114 +0,0 @@ -package distributor - -import ( - "sync" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/common/worker" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/component" - "github.com/onflow/flow-go/module/mempool/queue" - "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/network/p2p" -) - -const ( - // DefaultDisallowListNotificationQueueCacheSize is the default size of the disallow list notification queue. - DefaultDisallowListNotificationQueueCacheSize = 100 -) - -// DisallowListNotificationDistributor is a component that distributes disallow list updates to registered consumers in an -// asynchronous, fan-out manner. It is thread-safe and can be used concurrently from multiple goroutines. -type DisallowListNotificationDistributor struct { - component.Component - cm *component.ComponentManager - logger zerolog.Logger - - consumerLock sync.RWMutex // protects the consumer field from concurrent updates - consumers []p2p.DisallowListNotificationConsumer - workerPool *worker.Pool[*p2p.DisallowListUpdateNotification] -} - -var _ p2p.DisallowListNotificationDistributor = (*DisallowListNotificationDistributor)(nil) - -// DefaultDisallowListNotificationDistributor creates a new disallow list notification distributor with default configuration. -func DefaultDisallowListNotificationDistributor(logger zerolog.Logger, opts ...queue.HeroStoreConfigOption) *DisallowListNotificationDistributor { - cfg := &queue.HeroStoreConfig{ - SizeLimit: DefaultDisallowListNotificationQueueCacheSize, - Collector: metrics.NewNoopCollector(), - } - - for _, opt := range opts { - opt(cfg) - } - - store := queue.NewHeroStore(cfg.SizeLimit, logger, cfg.Collector) - return NewDisallowListConsumer(logger, store) -} - -// NewDisallowListConsumer creates a new disallow list notification distributor. -// It takes a message store as a parameter, which is used to store the events that are distributed to the consumers. -// The message store is used to ensure that DistributeBlockListNotification is non-blocking. -func NewDisallowListConsumer(logger zerolog.Logger, store engine.MessageStore) *DisallowListNotificationDistributor { - lg := logger.With().Str("component", "node_disallow_distributor").Logger() - - d := &DisallowListNotificationDistributor{ - logger: lg, - } - - pool := worker.NewWorkerPoolBuilder[*p2p.DisallowListUpdateNotification]( - lg, - store, - d.distribute).Build() - - d.workerPool = pool - - cm := component.NewComponentManagerBuilder() - cm.AddWorker(d.workerPool.WorkerLogic()) - - d.cm = cm.Build() - d.Component = d.cm - - return d -} - -// distribute is called by the workers to process the event. It calls the OnDisallowListNotification method on all registered -// consumers. -// It does not return an error because the event is already in the store, so it will be retried. -func (d *DisallowListNotificationDistributor) distribute(notification *p2p.DisallowListUpdateNotification) error { - d.consumerLock.RLock() - defer d.consumerLock.RUnlock() - - for _, consumer := range d.consumers { - consumer.OnDisallowListNotification(notification) - } - - return nil -} - -// AddConsumer adds a consumer to the distributor. The consumer will be called the distributor distributes a new event. -// AddConsumer must be concurrency safe. Once a consumer is added, it must be called for all future events. -// There is no guarantee that the consumer will be called for events that were already received by the distributor. -func (d *DisallowListNotificationDistributor) AddConsumer(consumer p2p.DisallowListNotificationConsumer) { - d.consumerLock.Lock() - defer d.consumerLock.Unlock() - - d.consumers = append(d.consumers, consumer) -} - -// DistributeBlockListNotification distributes the event to all the consumers. -// Implementation is non-blocking, it submits the event to the worker pool and returns immediately. -// The event will be distributed to the consumers in the order it was submitted but asynchronously. -// If the worker pool is full, the event will be dropped and a warning will be logged. -// This implementation returns no error. -func (d *DisallowListNotificationDistributor) DistributeBlockListNotification(disallowList flow.IdentifierList) error { - ok := d.workerPool.Submit(&p2p.DisallowListUpdateNotification{DisallowList: disallowList}) - if !ok { - // we use a queue to buffer the events, so this may happen if the queue is full or the event is duplicate. In this case, we log a warning. - d.logger.Warn().Msg("node disallow list update notification queue is full or the event is duplicate, dropping event") - } - - return nil -} diff --git a/network/p2p/distributor/disallow_list_test.go b/network/p2p/distributor/disallow_list_test.go deleted file mode 100644 index 39cf9532f46..00000000000 --- a/network/p2p/distributor/disallow_list_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package distributor_test - -import ( - "context" - "math/rand" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/distributor" - mockp2p "github.com/onflow/flow-go/network/p2p/mock" - "github.com/onflow/flow-go/utils/unittest" -) - -// TestDisallowListNotificationDistributor tests the disallow list notification distributor by adding two consumers to the -// notification distributor component and sending a random set of notifications to the notification component. The test -// verifies that the consumers receive the notifications and that each consumer sees each notification only once. -func TestDisallowListNotificationDistributor(t *testing.T) { - d := distributor.DefaultDisallowListNotificationDistributor(unittest.Logger()) - - c1 := mockp2p.NewDisallowListNotificationConsumer(t) - c2 := mockp2p.NewDisallowListNotificationConsumer(t) - - d.AddConsumer(c1) - d.AddConsumer(c2) - - tt := disallowListUpdateNotificationsFixture(50) - - c1Done := sync.WaitGroup{} - c1Done.Add(len(tt)) - c1Seen := unittest.NewProtectedMap[flow.Identifier, struct{}]() - c1.On("OnDisallowListNotification", mock.Anything).Run(func(args mock.Arguments) { - n, ok := args.Get(0).(*p2p.DisallowListUpdateNotification) - require.True(t, ok) - - require.Contains(t, tt, n) - - // ensure consumer see each peer once - hash := flow.MerkleRoot(n.DisallowList...) - require.False(t, c1Seen.Has(hash)) - c1Seen.Add(hash, struct{}{}) - - c1Done.Done() - }).Return() - - c2Done := sync.WaitGroup{} - c2Done.Add(len(tt)) - c2Seen := unittest.NewProtectedMap[flow.Identifier, struct{}]() - c2.On("OnDisallowListNotification", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - n, ok := args.Get(0).(*p2p.DisallowListUpdateNotification) - require.True(t, ok) - - require.Contains(t, tt, n) - - // ensure consumer see each peer once - hash := flow.MerkleRoot(n.DisallowList...) - require.False(t, c2Seen.Has(hash)) - c2Seen.Add(hash, struct{}{}) - - c2Done.Done() - }).Return() - - cancelCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - ctx, _ := irrecoverable.WithSignaler(cancelCtx) - d.Start(ctx) - - unittest.RequireCloseBefore(t, d.Ready(), 100*time.Millisecond, "could not start distributor") - - for i := 0; i < len(tt); i++ { - go func(i int) { - require.NoError(t, d.DistributeBlockListNotification(tt[i].DisallowList)) - }(i) - } - - unittest.RequireReturnsBefore(t, c1Done.Wait, 1*time.Second, "events are not received by consumer 1") - unittest.RequireReturnsBefore(t, c2Done.Wait, 1*time.Second, "events are not received by consumer 2") - cancel() - unittest.RequireCloseBefore(t, d.Done(), 100*time.Millisecond, "could not stop distributor") -} - -func disallowListUpdateNotificationsFixture(n int) []*p2p.DisallowListUpdateNotification { - tt := make([]*p2p.DisallowListUpdateNotification, n) - for i := 0; i < n; i++ { - tt[i] = disallowListUpdateNotificationFixture() - } - return tt -} - -func disallowListUpdateNotificationFixture() *p2p.DisallowListUpdateNotification { - return &p2p.DisallowListUpdateNotification{ - DisallowList: unittest.IdentifierListFixture(rand.Int()%100 + 1), - } -} diff --git a/network/p2p/distributor/gossipsub_inspector.go b/network/p2p/distributor/gossipsub_inspector.go index 8ea7f2c0f2e..9c9eec28c61 100644 --- a/network/p2p/distributor/gossipsub_inspector.go +++ b/network/p2p/distributor/gossipsub_inspector.go @@ -20,25 +20,25 @@ const ( defaultGossipSubInspectorNotificationQueueWorkerCount = 1 ) -var _ p2p.GossipSubInspectorNotificationDistributor = (*GossipSubInspectorNotificationDistributor)(nil) +var _ p2p.GossipSubInspectorNotifDistributor = (*GossipSubInspectorNotifDistributor)(nil) -// GossipSubInspectorNotificationDistributor is a component that distributes gossipsub rpc inspector notifications to +// GossipSubInspectorNotifDistributor is a component that distributes gossipsub rpc inspector notifications to // registered consumers in a non-blocking manner and asynchronously. It is thread-safe and can be used concurrently from // multiple goroutines. The distribution is done by a worker pool. The worker pool is configured with a queue that has a // fixed size. If the queue is full, the notification is discarded. The queue size and the number of workers can be // configured. -type GossipSubInspectorNotificationDistributor struct { +type GossipSubInspectorNotifDistributor struct { component.Component cm *component.ComponentManager logger zerolog.Logger - workerPool *worker.Pool[*p2p.InvalidControlMessageNotification] + workerPool *worker.Pool[*p2p.InvCtrlMsgNotif] consumerLock sync.RWMutex // protects the consumer field from concurrent updates - consumers []p2p.GossipSubInvalidControlMessageNotificationConsumer + consumers []p2p.GossipSubInvCtrlMsgNotifConsumer } -// DefaultGossipSubInspectorNotificationDistributor returns a new GossipSubInspectorNotificationDistributor component with the default configuration. -func DefaultGossipSubInspectorNotificationDistributor(logger zerolog.Logger, opts ...queue.HeroStoreConfigOption) *GossipSubInspectorNotificationDistributor { +// DefaultGossipSubInspectorNotificationDistributor returns a new GossipSubInspectorNotifDistributor component with the default configuration. +func DefaultGossipSubInspectorNotificationDistributor(logger zerolog.Logger, opts ...queue.HeroStoreConfigOption) *GossipSubInspectorNotifDistributor { cfg := &queue.HeroStoreConfig{ SizeLimit: DefaultGossipSubInspectorNotificationQueueCacheSize, Collector: metrics.NewNoopCollector(), @@ -52,16 +52,16 @@ func DefaultGossipSubInspectorNotificationDistributor(logger zerolog.Logger, opt return NewGossipSubInspectorNotificationDistributor(logger, store) } -// NewGossipSubInspectorNotificationDistributor returns a new GossipSubInspectorNotificationDistributor component. +// NewGossipSubInspectorNotificationDistributor returns a new GossipSubInspectorNotifDistributor component. // It takes a message store to store the notifications in memory and process them asynchronously. -func NewGossipSubInspectorNotificationDistributor(log zerolog.Logger, store engine.MessageStore) *GossipSubInspectorNotificationDistributor { +func NewGossipSubInspectorNotificationDistributor(log zerolog.Logger, store engine.MessageStore) *GossipSubInspectorNotifDistributor { lg := log.With().Str("component", "gossipsub_rpc_inspector_distributor").Logger() - d := &GossipSubInspectorNotificationDistributor{ + d := &GossipSubInspectorNotifDistributor{ logger: lg, } - pool := worker.NewWorkerPoolBuilder[*p2p.InvalidControlMessageNotification](lg, store, d.distribute).Build() + pool := worker.NewWorkerPoolBuilder[*p2p.InvCtrlMsgNotif](lg, store, d.distribute).Build() d.workerPool = pool cm := component.NewComponentManagerBuilder() @@ -76,10 +76,10 @@ func NewGossipSubInspectorNotificationDistributor(log zerolog.Logger, store engi return d } -// DistributeInvalidControlMessageNotification distributes the gossipsub rpc inspector notification to all registered consumers. +// Distribute distributes the gossipsub rpc inspector notification to all registered consumers. // The distribution is done asynchronously and non-blocking. The notification is added to a queue and processed by a worker pool. // DistributeEvent in this implementation does not return an error, but it logs a warning if the queue is full. -func (g *GossipSubInspectorNotificationDistributor) DistributeInvalidControlMessageNotification(notification *p2p.InvalidControlMessageNotification) error { +func (g *GossipSubInspectorNotifDistributor) Distribute(notification *p2p.InvCtrlMsgNotif) error { if ok := g.workerPool.Submit(notification); !ok { // we use a queue with a fixed size, so this can happen when queue is full or when the notification is duplicate. g.logger.Warn().Msg("gossipsub rpc inspector notification queue is full or notification is duplicate, discarding notification") @@ -90,7 +90,7 @@ func (g *GossipSubInspectorNotificationDistributor) DistributeInvalidControlMess // AddConsumer adds a consumer to the distributor. The consumer will be called when distributor distributes a new event. // AddConsumer must be concurrency safe. Once a consumer is added, it must be called for all future events. // There is no guarantee that the consumer will be called for events that were already received by the distributor. -func (g *GossipSubInspectorNotificationDistributor) AddConsumer(consumer p2p.GossipSubInvalidControlMessageNotificationConsumer) { +func (g *GossipSubInspectorNotifDistributor) AddConsumer(consumer p2p.GossipSubInvCtrlMsgNotifConsumer) { g.consumerLock.Lock() defer g.consumerLock.Unlock() @@ -100,7 +100,7 @@ func (g *GossipSubInspectorNotificationDistributor) AddConsumer(consumer p2p.Gos // distribute calls the ConsumeEvent method of all registered consumers. It is called by the workers of the worker pool. // It is concurrency safe and can be called concurrently by multiple workers. However, the consumers may be blocking // on the ConsumeEvent method. -func (g *GossipSubInspectorNotificationDistributor) distribute(notification *p2p.InvalidControlMessageNotification) error { +func (g *GossipSubInspectorNotifDistributor) distribute(notification *p2p.InvCtrlMsgNotif) error { g.consumerLock.RLock() defer g.consumerLock.RUnlock() diff --git a/network/p2p/distributor/gossipsub_inspector_test.go b/network/p2p/distributor/gossipsub_inspector_test.go index fad17c8d026..23323d7282c 100644 --- a/network/p2p/distributor/gossipsub_inspector_test.go +++ b/network/p2p/distributor/gossipsub_inspector_test.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/distributor" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" mockp2p "github.com/onflow/flow-go/network/p2p/mock" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/utils/unittest" @@ -37,7 +38,7 @@ func TestGossipSubInspectorNotification(t *testing.T) { c1Done.Add(len(tt)) c1Seen := unittest.NewProtectedMap[peer.ID, struct{}]() c1.On("OnInvalidControlMessageNotification", mock.Anything).Run(func(args mock.Arguments) { - notification, ok := args.Get(0).(*p2p.InvalidControlMessageNotification) + notification, ok := args.Get(0).(*p2p.InvCtrlMsgNotif) require.True(t, ok) require.Contains(t, tt, notification) @@ -53,7 +54,7 @@ func TestGossipSubInspectorNotification(t *testing.T) { c2Done.Add(len(tt)) c2Seen := unittest.NewProtectedMap[peer.ID, struct{}]() c2.On("OnInvalidControlMessageNotification", mock.Anything).Run(func(args mock.Arguments) { - notification, ok := args.Get(0).(*p2p.InvalidControlMessageNotification) + notification, ok := args.Get(0).(*p2p.InvCtrlMsgNotif) require.True(t, ok) require.Contains(t, tt, notification) @@ -73,7 +74,7 @@ func TestGossipSubInspectorNotification(t *testing.T) { for i := 0; i < len(tt); i++ { go func(i int) { - require.NoError(t, g.DistributeInvalidControlMessageNotification(tt[i])) + require.NoError(t, g.Distribute(tt[i])) }(i) } @@ -83,18 +84,18 @@ func TestGossipSubInspectorNotification(t *testing.T) { unittest.RequireCloseBefore(t, g.Done(), 100*time.Millisecond, "could not stop distributor") } -func invalidControlMessageNotificationListFixture(t *testing.T, n int) []*p2p.InvalidControlMessageNotification { - list := make([]*p2p.InvalidControlMessageNotification, n) +func invalidControlMessageNotificationListFixture(t *testing.T, n int) []*p2p.InvCtrlMsgNotif { + list := make([]*p2p.InvCtrlMsgNotif, n) for i := 0; i < n; i++ { list[i] = invalidControlMessageNotificationFixture(t) } return list } -func invalidControlMessageNotificationFixture(t *testing.T) *p2p.InvalidControlMessageNotification { - return &p2p.InvalidControlMessageNotification{ +func invalidControlMessageNotificationFixture(t *testing.T) *p2p.InvCtrlMsgNotif { + return &p2p.InvCtrlMsgNotif{ PeerID: p2ptest.PeerIdFixture(t), - MsgType: []p2p.ControlMessageType{p2p.CtrlMsgGraft, p2p.CtrlMsgPrune, p2p.CtrlMsgIHave, p2p.CtrlMsgIWant}[rand.Intn(4)], + MsgType: []p2pmsg.ControlMessageType{p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune, p2pmsg.CtrlMsgIHave, p2pmsg.CtrlMsgIWant}[rand.Intn(4)], Count: rand.Uint64(), } } diff --git a/network/p2p/inspector/internal/cache/cache.go b/network/p2p/inspector/internal/cache/cache.go new file mode 100644 index 00000000000..82d8f781a98 --- /dev/null +++ b/network/p2p/inspector/internal/cache/cache.go @@ -0,0 +1,227 @@ +package cache + +import ( + "fmt" + "time" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" + "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" + "github.com/onflow/flow-go/module/mempool/stdmap" + "github.com/onflow/flow-go/network/p2p/scoring" +) + +type recordEntityFactory func(identifier flow.Identifier) ClusterPrefixedMessagesReceivedRecord + +type RecordCacheConfig struct { + sizeLimit uint32 + logger zerolog.Logger + collector module.HeroCacheMetrics + // recordDecay decay factor used by the cache to perform geometric decay on gauge values. + recordDecay float64 +} + +// RecordCache is a cache that stores ClusterPrefixedMessagesReceivedRecord by peer node ID. Each record +// contains a float64 Gauge field that indicates the current approximate number cluster prefixed control messages that were allowed to bypass +// validation due to some error that will prevent the message from being validated. +// Each record contains a float64 Gauge field that is decayed overtime back to 0. This ensures that nodes that fall +// behind in the protocol can catch up. +type RecordCache struct { + // recordEntityFactory is a factory function that creates a new ClusterPrefixedMessagesReceivedRecord. + recordEntityFactory recordEntityFactory + // c is the underlying cache. + c *stdmap.Backend + // decayFunc decay func used by the cache to perform decay on gauges. + decayFunc decayFunc +} + +// NewRecordCache creates a new *RecordCache. +// Args: +// - config: record cache config. +// - recordEntityFactory: a factory function that creates a new spam record. +// Returns: +// - *RecordCache, the created cache. +// Note that this cache is supposed to keep the cluster prefix control messages received record for the authorized (staked) nodes. Since the number of such nodes is +// expected to be small, we do not eject any records from the cache. The cache size must be large enough to hold all +// the records of the authorized nodes. Also, this cache is keeping at most one record per peer id, so the +// size of the cache must be at least the number of authorized nodes. +func NewRecordCache(config *RecordCacheConfig, recordEntityFactory recordEntityFactory) (*RecordCache, error) { + backData := herocache.NewCache(config.sizeLimit, + herocache.DefaultOversizeFactor, + // this cache is supposed to keep the cluster prefix control messages received record for the authorized (staked) nodes. Since the number of such nodes is + // expected to be small, we do not eject any records from the cache. The cache size must be large enough to hold all + // the records of the authorized nodes. Also, this cache is keeping at most one record per peer id, so the + // size of the cache must be at least the number of authorized nodes. + heropool.NoEjection, + config.logger.With().Str("mempool", "gossipsub=cluster-prefix-control-messages-received-records").Logger(), + config.collector) + return &RecordCache{ + recordEntityFactory: recordEntityFactory, + decayFunc: defaultDecayFunction(config.recordDecay), + c: stdmap.NewBackend(stdmap.WithBackData(backData)), + }, nil +} + +// Init initializes the record cache for the given peer id if it does not exist. +// Returns true if the record is initialized, false otherwise (i.e.: the record already exists). +// Args: +// - nodeID: the node ID of the sender of the control message. +// Returns: +// - true if the record is initialized, false otherwise (i.e.: the record already exists). +// Note that if Init is called multiple times for the same peer id, the record is initialized only once, and the +// subsequent calls return false and do not change the record (i.e.: the record is not re-initialized). +func (r *RecordCache) Init(nodeID flow.Identifier) bool { + entity := r.recordEntityFactory(nodeID) + return r.c.Add(entity) +} + +// ReceivedClusterPrefixedMessage applies an adjustment that increments the number of cluster prefixed control messages received by a peer. +// Returns number of cluster prefix control messages received after the adjustment. The record is initialized before +// the adjustment func is applied that will increment the Gauge. +// Args: +// - nodeID: the node ID of the sender of the control message. +// Returns: +// - The cluster prefix control messages received gauge value after the adjustment. +// - exception only in cases of internal data inconsistency or bugs. No errors are expected. +func (r *RecordCache) ReceivedClusterPrefixedMessage(nodeID flow.Identifier) (float64, error) { + var err error + optimisticAdjustFunc := func() (flow.Entity, bool) { + return r.c.Adjust(nodeID, func(entity flow.Entity) flow.Entity { + entity, err = r.decayAdjustment(entity) // first decay the record + if err != nil { + return entity + } + return r.incrementAdjustment(entity) // then increment the record + }) + } + + // optimisticAdjustFunc is called assuming the record exists; if the record does not exist, + // it means the record was not initialized. In this case, initialize the record and call optimisticAdjustFunc again. + // If the record was initialized, optimisticAdjustFunc will be called only once. + adjustedEntity, adjusted := optimisticAdjustFunc() + if err != nil { + return 0, fmt.Errorf("unexpected error while applying decay adjustment for node %s: %w", nodeID, err) + } + if !adjusted { + r.Init(nodeID) + adjustedEntity, adjusted = optimisticAdjustFunc() + if !adjusted { + return 0, fmt.Errorf("unexpected record not found for node ID %s, even after an init attempt", nodeID) + } + } + + return adjustedEntity.(ClusterPrefixedMessagesReceivedRecord).Gauge, nil +} + +// Get returns the current number of cluster prefixed control messages received from a peer. +// The record is initialized before the count is returned. +// Before the control messages received gauge value is returned it is decayed using the configured decay function. +// Returns the record and true if the record exists, nil and false otherwise. +// Args: +// - nodeID: the node ID of the sender of the control message. +// Returns: +// - The cluster prefixed control messages received gauge value after the decay and true if the record exists, 0 and false otherwise. +// No errors are expected during normal operation. +func (r *RecordCache) Get(nodeID flow.Identifier) (float64, bool, error) { + if r.Init(nodeID) { + return 0, true, nil + } + + var err error + adjustedEntity, adjusted := r.c.Adjust(nodeID, func(entity flow.Entity) flow.Entity { + // perform decay on gauge value + entity, err = r.decayAdjustment(entity) + return entity + }) + if err != nil { + return 0, false, fmt.Errorf("unexpected error while applying decay adjustment for node %s: %w", nodeID, err) + } + if !adjusted { + return 0, false, fmt.Errorf("unexpected error record not found for node ID %s, even after an init attempt", nodeID) + } + + record, ok := adjustedEntity.(ClusterPrefixedMessagesReceivedRecord) + if !ok { + // sanity check + // This should never happen, because the cache only contains ClusterPrefixedMessagesReceivedRecord entities. + panic(fmt.Sprintf("invalid entity type, expected ClusterPrefixedMessagesReceivedRecord type, got: %T", adjustedEntity)) + } + + return record.Gauge, true, nil +} + +// NodeIDs returns the list of identities of the nodes that have a spam record in the cache. +func (r *RecordCache) NodeIDs() []flow.Identifier { + return flow.GetIDs(r.c.All()) +} + +// Remove removes the record of the given peer id from the cache. +// Returns true if the record is removed, false otherwise (i.e., the record does not exist). +// Args: +// - nodeID: the node ID of the sender of the control message. +// Returns: +// - true if the record is removed, false otherwise (i.e., the record does not exist). +func (r *RecordCache) Remove(nodeID flow.Identifier) bool { + return r.c.Remove(nodeID) +} + +// Size returns the number of records in the cache. +func (r *RecordCache) Size() uint { + return r.c.Size() +} + +func (r *RecordCache) incrementAdjustment(entity flow.Entity) flow.Entity { + record, ok := entity.(ClusterPrefixedMessagesReceivedRecord) + if !ok { + // sanity check + // This should never happen, because the cache only contains ClusterPrefixedMessagesReceivedRecord entities. + panic(fmt.Sprintf("invalid entity type, expected ClusterPrefixedMessagesReceivedRecord type, got: %T", entity)) + } + record.Gauge++ + record.lastUpdated = time.Now() + // Return the adjusted record. + return record +} + +// All errors returned from this function are unexpected and irrecoverable. +func (r *RecordCache) decayAdjustment(entity flow.Entity) (flow.Entity, error) { + record, ok := entity.(ClusterPrefixedMessagesReceivedRecord) + if !ok { + // sanity check + // This should never happen, because the cache only contains ClusterPrefixedMessagesReceivedRecord entities. + panic(fmt.Sprintf("invalid entity type, expected ClusterPrefixedMessagesReceivedRecord type, got: %T", entity)) + } + var err error + record, err = r.decayFunc(record) + if err != nil { + return record, err + } + record.lastUpdated = time.Now() + // Return the adjusted record. + return record, nil +} + +// decayFunc the callback used to apply a decay method to the record. +// All errors returned from this callback are unexpected and irrecoverable. +type decayFunc func(recordEntity ClusterPrefixedMessagesReceivedRecord) (ClusterPrefixedMessagesReceivedRecord, error) + +// defaultDecayFunction is the default decay function that is used to decay the cluster prefixed control message received gauge of a peer. +// All errors returned are unexpected and irrecoverable. +func defaultDecayFunction(decay float64) decayFunc { + return func(recordEntity ClusterPrefixedMessagesReceivedRecord) (ClusterPrefixedMessagesReceivedRecord, error) { + received := recordEntity.Gauge + if received == 0 { + return recordEntity, nil + } + + decayedVal, err := scoring.GeometricDecay(received, decay, recordEntity.lastUpdated) + if err != nil { + return recordEntity, fmt.Errorf("could not decay cluster prefixed control messages received gauge: %w", err) + } + recordEntity.Gauge = decayedVal + return recordEntity, nil + } +} diff --git a/network/p2p/inspector/internal/cache/cache_test.go b/network/p2p/inspector/internal/cache/cache_test.go new file mode 100644 index 00000000000..2be9d4f2517 --- /dev/null +++ b/network/p2p/inspector/internal/cache/cache_test.go @@ -0,0 +1,503 @@ +package cache + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/utils/unittest" +) + +const defaultDecay = 0.99 + +// TestRecordCache_Init tests the Init method of the RecordCache. +// It ensures that the method returns true when a new record is initialized +// and false when an existing record is initialized. +func TestRecordCache_Init(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + nodeID1 := unittest.IdentifierFixture() + nodeID2 := unittest.IdentifierFixture() + + // test initializing a record for an node ID that doesn't exist in the cache + initialized := cache.Init(nodeID1) + require.True(t, initialized, "expected record to be initialized") + gauge, ok, err := cache.Get(nodeID1) + require.NoError(t, err) + require.True(t, ok, "expected record to exist") + require.Zerof(t, gauge, "expected gauge to be 0") + require.Equal(t, uint(1), cache.Size(), "expected cache to have one additional record") + + // test initializing a record for an node ID that already exists in the cache + initialized = cache.Init(nodeID1) + require.False(t, initialized, "expected record not to be initialized") + gaugeAgain, ok, err := cache.Get(nodeID1) + require.NoError(t, err) + require.True(t, ok, "expected record to still exist") + require.Zerof(t, gaugeAgain, "expected same gauge to be 0") + require.Equal(t, gauge, gaugeAgain, "expected records to be the same") + require.Equal(t, uint(1), cache.Size(), "expected cache to still have one additional record") + + // test initializing a record for another node ID + initialized = cache.Init(nodeID2) + require.True(t, initialized, "expected record to be initialized") + gauge2, ok, err := cache.Get(nodeID2) + require.NoError(t, err) + require.True(t, ok, "expected record to exist") + require.Zerof(t, gauge2, "expected second gauge to be 0") + require.Equal(t, uint(2), cache.Size(), "expected cache to have two additional records") +} + +// TestRecordCache_ConcurrentInit tests the concurrent initialization of records. +// The test covers the following scenarios: +// 1. Multiple goroutines initializing records for different node IDs. +// 2. Ensuring that all records are correctly initialized. +func TestRecordCache_ConcurrentInit(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + nodeIDs := unittest.IdentifierListFixture(10) + + var wg sync.WaitGroup + wg.Add(len(nodeIDs)) + + for _, nodeID := range nodeIDs { + go func(id flow.Identifier) { + defer wg.Done() + cache.Init(id) + }(nodeID) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that all records are correctly initialized + for _, nodeID := range nodeIDs { + gauge, found, err := cache.Get(nodeID) + require.NoError(t, err) + require.True(t, found) + require.Zerof(t, gauge, "expected all gauge values to be initialized to 0") + } +} + +// TestRecordCache_ConcurrentSameRecordInit tests the concurrent initialization of the same record. +// The test covers the following scenarios: +// 1. Multiple goroutines attempting to initialize the same record concurrently. +// 2. Only one goroutine successfully initializes the record, and others receive false on initialization. +// 3. The record is correctly initialized in the cache and can be retrieved using the Get method. +func TestRecordCache_ConcurrentSameRecordInit(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + nodeID := unittest.IdentifierFixture() + const concurrentAttempts = 10 + + var wg sync.WaitGroup + wg.Add(concurrentAttempts) + + successGauge := atomic.Int32{} + + for i := 0; i < concurrentAttempts; i++ { + go func() { + defer wg.Done() + initSuccess := cache.Init(nodeID) + if initSuccess { + successGauge.Inc() + } + }() + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that only one goroutine successfully initialized the record + require.Equal(t, int32(1), successGauge.Load()) + + // ensure that the record is correctly initialized in the cache + gauge, found, err := cache.Get(nodeID) + require.NoError(t, err) + require.True(t, found) + require.Zero(t, gauge) +} + +// TestRecordCache_ReceivedClusterPrefixedMessage tests the ReceivedClusterPrefixedMessage method of the RecordCache. +// The test covers the following scenarios: +// 1. Updating a record gauge for an existing node ID. +// 2. Attempting to update a record gauge for a non-existing node ID should not result in error. ReceivedClusterPrefixedMessage should always attempt to initialize the gauge. +// 3. Multiple updates on the same record only initialize the record once. +func TestRecordCache_ReceivedClusterPrefixedMessage(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + nodeID1 := unittest.IdentifierFixture() + nodeID2 := unittest.IdentifierFixture() + + // initialize spam records for nodeID1 and nodeID2 + require.True(t, cache.Init(nodeID1)) + require.True(t, cache.Init(nodeID2)) + + gauge, err := cache.ReceivedClusterPrefixedMessage(nodeID1) + require.NoError(t, err) + require.Equal(t, float64(1), gauge) + + // get will apply a slightl decay resulting + // in a gauge value less than gauge which is 1 but greater than 0.9 + currentGauge, ok, err := cache.Get(nodeID1) + require.NoError(t, err) + require.True(t, ok) + require.LessOrEqual(t, currentGauge, gauge) + require.Greater(t, currentGauge, 0.9) + + // test adjusting the spam record for a non-existing node ID + nodeID3 := unittest.IdentifierFixture() + gauge3, err := cache.ReceivedClusterPrefixedMessage(nodeID3) + require.NoError(t, err) + require.Equal(t, float64(1), gauge3) + + // when updated the value should be incremented from 1 -> 2 and slightly decayed resulting + // in a gauge value less than 2 but greater than 1.9 + gauge3, err = cache.ReceivedClusterPrefixedMessage(nodeID3) + require.NoError(t, err) + require.LessOrEqual(t, gauge3, 2.0) + require.Greater(t, gauge3, 1.9) +} + +// TestRecordCache_UpdateDecay ensures that a gauge in the record cache is eventually decayed back to 0 after some time. +func TestRecordCache_Decay(t *testing.T) { + cache := cacheFixture(t, 100, 0.09, zerolog.Nop(), metrics.NewNoopCollector()) + + nodeID1 := unittest.IdentifierFixture() + + // initialize spam records for nodeID1 and nodeID2 + require.True(t, cache.Init(nodeID1)) + gauge, err := cache.ReceivedClusterPrefixedMessage(nodeID1) + require.Equal(t, float64(1), gauge) + require.NoError(t, err) + gauge, ok, err := cache.Get(nodeID1) + require.True(t, ok) + require.NoError(t, err) + // gauge should have been delayed slightly + require.True(t, gauge < float64(1)) + + time.Sleep(time.Second) + + gauge, ok, err = cache.Get(nodeID1) + require.True(t, ok) + require.NoError(t, err) + // gauge should have been delayed slightly, but closer to 0 + require.Less(t, gauge, 0.1) +} + +// TestRecordCache_Identities tests the NodeIDs method of the RecordCache. +// The test covers the following scenarios: +// 1. Initializing the cache with multiple records. +// 2. Checking if the NodeIDs method returns the correct set of node IDs. +func TestRecordCache_Identities(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + // initialize spam records for a few node IDs + nodeID1 := unittest.IdentifierFixture() + nodeID2 := unittest.IdentifierFixture() + nodeID3 := unittest.IdentifierFixture() + + require.True(t, cache.Init(nodeID1)) + require.True(t, cache.Init(nodeID2)) + require.True(t, cache.Init(nodeID3)) + + // check if the NodeIDs method returns the correct set of node IDs + identities := cache.NodeIDs() + require.Equal(t, 3, len(identities)) + require.ElementsMatch(t, identities, []flow.Identifier{nodeID1, nodeID2, nodeID3}) +} + +// TestRecordCache_Remove tests the Remove method of the RecordCache. +// The test covers the following scenarios: +// 1. Initializing the cache with multiple records. +// 2. Removing a record and checking if it is removed correctly. +// 3. Ensuring the other records are still in the cache after removal. +// 4. Attempting to remove a non-existent node ID. +func TestRecordCache_Remove(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + // initialize spam records for a few node IDs + nodeID1 := unittest.IdentifierFixture() + nodeID2 := unittest.IdentifierFixture() + nodeID3 := unittest.IdentifierFixture() + + require.True(t, cache.Init(nodeID1)) + require.True(t, cache.Init(nodeID2)) + require.True(t, cache.Init(nodeID3)) + + numOfIds := uint(3) + require.Equal(t, numOfIds, cache.Size(), fmt.Sprintf("expected size of the cache to be %d", numOfIds)) + // remove nodeID1 and check if the record is removed + require.True(t, cache.Remove(nodeID1)) + require.NotContains(t, nodeID1, cache.NodeIDs()) + + // check if the other node IDs are still in the cache + _, exists, err := cache.Get(nodeID2) + require.NoError(t, err) + require.True(t, exists) + _, exists, err = cache.Get(nodeID3) + require.NoError(t, err) + require.True(t, exists) + + // attempt to remove a non-existent node ID + nodeID4 := unittest.IdentifierFixture() + require.False(t, cache.Remove(nodeID4)) +} + +// TestRecordCache_ConcurrentRemove tests the concurrent removal of records for different node IDs. +// The test covers the following scenarios: +// 1. Multiple goroutines removing records for different node IDs concurrently. +// 2. The records are correctly removed from the cache. +func TestRecordCache_ConcurrentRemove(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + nodeIDs := unittest.IdentifierListFixture(10) + for _, nodeID := range nodeIDs { + cache.Init(nodeID) + } + + var wg sync.WaitGroup + wg.Add(len(nodeIDs)) + + for _, nodeID := range nodeIDs { + go func(id flow.Identifier) { + defer wg.Done() + removed := cache.Remove(id) + require.True(t, removed) + require.NotContains(t, id, cache.NodeIDs()) + }(nodeID) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + require.Equal(t, uint(0), cache.Size()) +} + +// TestRecordCache_ConcurrentUpdatesAndReads tests the concurrent adjustments and reads of records for different +// node IDs. The test covers the following scenarios: +// 1. Multiple goroutines adjusting records for different node IDs concurrently. +// 2. Multiple goroutines getting records for different node IDs concurrently. +// 3. The adjusted records are correctly updated in the cache. +func TestRecordCache_ConcurrentUpdatesAndReads(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + nodeIDs := unittest.IdentifierListFixture(10) + for _, nodeID := range nodeIDs { + cache.Init(nodeID) + } + + var wg sync.WaitGroup + wg.Add(len(nodeIDs) * 2) + + for _, nodeID := range nodeIDs { + // adjust spam records concurrently + go func(id flow.Identifier) { + defer wg.Done() + _, err := cache.ReceivedClusterPrefixedMessage(id) + require.NoError(t, err) + }(nodeID) + + // get spam records concurrently + go func(id flow.Identifier) { + defer wg.Done() + _, found, err := cache.Get(id) + require.NoError(t, err) + require.True(t, found) + }(nodeID) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that the records are correctly updated in the cache + for _, nodeID := range nodeIDs { + gauge, found, err := cache.Get(nodeID) + require.NoError(t, err) + require.True(t, found) + // slight decay will result in 0.9 < gauge < 1 + require.LessOrEqual(t, gauge, 1.0) + require.Greater(t, gauge, 0.9) + } +} + +// TestRecordCache_ConcurrentInitAndRemove tests the concurrent initialization and removal of records for different +// node IDs. The test covers the following scenarios: +// 1. Multiple goroutines initializing records for different node IDs concurrently. +// 2. Multiple goroutines removing records for different node IDs concurrently. +// 3. The initialized records are correctly added to the cache. +// 4. The removed records are correctly removed from the cache. +func TestRecordCache_ConcurrentInitAndRemove(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + nodeIDs := unittest.IdentifierListFixture(20) + nodeIDsToAdd := nodeIDs[:10] + nodeIDsToRemove := nodeIDs[10:] + + for _, nodeID := range nodeIDsToRemove { + cache.Init(nodeID) + } + + var wg sync.WaitGroup + wg.Add(len(nodeIDs)) + + // initialize spam records concurrently + for _, nodeID := range nodeIDsToAdd { + go func(id flow.Identifier) { + defer wg.Done() + cache.Init(id) + }(nodeID) + } + + // remove spam records concurrently + for _, nodeID := range nodeIDsToRemove { + go func(id flow.Identifier) { + defer wg.Done() + cache.Remove(id) + require.NotContains(t, id, cache.NodeIDs()) + }(nodeID) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that the initialized records are correctly added to the cache + // and removed records are correctly removed from the cache + require.ElementsMatch(t, nodeIDsToAdd, cache.NodeIDs()) +} + +// TestRecordCache_ConcurrentInitRemoveUpdate tests the concurrent initialization, removal, and adjustment of +// records for different node IDs. The test covers the following scenarios: +// 1. Multiple goroutines initializing records for different node IDs concurrently. +// 2. Multiple goroutines removing records for different node IDs concurrently. +// 3. Multiple goroutines adjusting records for different node IDs concurrently. +func TestRecordCache_ConcurrentInitRemoveUpdate(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + nodeIDs := unittest.IdentifierListFixture(30) + nodeIDsToAdd := nodeIDs[:10] + nodeIDsToRemove := nodeIDs[10:20] + nodeIDsToAdjust := nodeIDs[20:] + + for _, nodeID := range nodeIDsToRemove { + cache.Init(nodeID) + } + + var wg sync.WaitGroup + wg.Add(len(nodeIDs)) + + // Initialize spam records concurrently + for _, nodeID := range nodeIDsToAdd { + go func(id flow.Identifier) { + defer wg.Done() + cache.Init(id) + }(nodeID) + } + + // Remove spam records concurrently + for _, nodeID := range nodeIDsToRemove { + go func(id flow.Identifier) { + defer wg.Done() + cache.Remove(id) + require.NotContains(t, id, cache.NodeIDs()) + }(nodeID) + } + + // Adjust spam records concurrently + for _, nodeID := range nodeIDsToAdjust { + go func(id flow.Identifier) { + defer wg.Done() + _, _ = cache.ReceivedClusterPrefixedMessage(id) + }(nodeID) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + require.ElementsMatch(t, append(nodeIDsToAdd, nodeIDsToAdjust...), cache.NodeIDs()) +} + +// TestRecordCache_EdgeCasesAndInvalidInputs tests the edge cases and invalid inputs for RecordCache methods. +// The test covers the following scenarios: +// 1. Initializing a record multiple times. +// 2. Adjusting a non-existent record. +// 3. Removing a record multiple times. +func TestRecordCache_EdgeCasesAndInvalidInputs(t *testing.T) { + cache := cacheFixture(t, 100, defaultDecay, zerolog.Nop(), metrics.NewNoopCollector()) + + nodeIDs := unittest.IdentifierListFixture(20) + nodeIDsToAdd := nodeIDs[:10] + nodeIDsToRemove := nodeIDs[10:20] + + for _, nodeID := range nodeIDsToRemove { + cache.Init(nodeID) + } + + var wg sync.WaitGroup + wg.Add(len(nodeIDs) + 10) + + // initialize spam records concurrently + for _, nodeID := range nodeIDsToAdd { + go func(id flow.Identifier) { + defer wg.Done() + require.True(t, cache.Init(id)) + retrieved, ok, err := cache.Get(id) + require.NoError(t, err) + require.True(t, ok) + require.Zero(t, retrieved) + }(nodeID) + } + + // remove spam records concurrently + for _, nodeID := range nodeIDsToRemove { + go func(id flow.Identifier) { + defer wg.Done() + require.True(t, cache.Remove(id)) + require.NotContains(t, id, cache.NodeIDs()) + }(nodeID) + } + + // call NodeIDs method concurrently + for i := 0; i < 10; i++ { + go func() { + defer wg.Done() + ids := cache.NodeIDs() + // the number of returned IDs should be less than or equal to the number of node IDs + require.True(t, len(ids) <= len(nodeIDs)) + // the returned IDs should be a subset of the node IDs + for _, id := range ids { + require.Contains(t, nodeIDs, id) + } + }() + } + unittest.RequireReturnsBefore(t, wg.Wait, 1*time.Second, "timed out waiting for goroutines to finish") +} + +// recordFixture creates a new record entity with the given node id. +// Args: +// - id: the node id of the record. +// Returns: +// - RecordEntity: the created record entity. +func recordEntityFixture(id flow.Identifier) ClusterPrefixedMessagesReceivedRecord { + return ClusterPrefixedMessagesReceivedRecord{NodeID: id, Gauge: 0.0, lastUpdated: time.Now()} +} + +// cacheFixture returns a new *RecordCache. +func cacheFixture(t *testing.T, sizeLimit uint32, recordDecay float64, logger zerolog.Logger, collector module.HeroCacheMetrics) *RecordCache { + recordFactory := func(id flow.Identifier) ClusterPrefixedMessagesReceivedRecord { + return recordEntityFixture(id) + } + config := &RecordCacheConfig{ + sizeLimit: sizeLimit, + logger: logger, + collector: collector, + recordDecay: recordDecay, + } + r, err := NewRecordCache(config, recordFactory) + require.NoError(t, err) + // expect cache to be empty + require.Equalf(t, uint(0), r.Size(), "cache size must be 0") + require.NotNil(t, r) + return r +} diff --git a/network/p2p/inspector/internal/cache/cluster_prefixed_received_tracker.go b/network/p2p/inspector/internal/cache/cluster_prefixed_received_tracker.go new file mode 100644 index 00000000000..b112b7d7a7c --- /dev/null +++ b/network/p2p/inspector/internal/cache/cluster_prefixed_received_tracker.go @@ -0,0 +1,63 @@ +package cache + +import ( + "fmt" + + "github.com/rs/zerolog" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" +) + +// ClusterPrefixedMessagesReceivedTracker struct that keeps track of the amount of cluster prefixed control messages received by a peer. +type ClusterPrefixedMessagesReceivedTracker struct { + cache *RecordCache + // activeClusterIds atomic pointer that stores the current active cluster IDs. This ensures safe concurrent access to the activeClusterIds internal flow.ChainIDList. + activeClusterIds *atomic.Pointer[flow.ChainIDList] +} + +// NewClusterPrefixedMessagesReceivedTracker returns a new *ClusterPrefixedMessagesReceivedTracker. +func NewClusterPrefixedMessagesReceivedTracker(logger zerolog.Logger, sizeLimit uint32, clusterPrefixedCacheCollector module.HeroCacheMetrics, decay float64) (*ClusterPrefixedMessagesReceivedTracker, error) { + config := &RecordCacheConfig{ + sizeLimit: sizeLimit, + logger: logger, + collector: clusterPrefixedCacheCollector, + recordDecay: decay, + } + recordCache, err := NewRecordCache(config, NewClusterPrefixedMessagesReceivedRecord) + if err != nil { + return nil, fmt.Errorf("failed to create new record cahe: %w", err) + } + return &ClusterPrefixedMessagesReceivedTracker{cache: recordCache, activeClusterIds: atomic.NewPointer[flow.ChainIDList](&flow.ChainIDList{})}, nil +} + +// Inc increments the cluster prefixed control messages received Gauge for the peer. +// All errors returned from this func are unexpected and irrecoverable. +func (c *ClusterPrefixedMessagesReceivedTracker) Inc(nodeID flow.Identifier) (float64, error) { + count, err := c.cache.ReceivedClusterPrefixedMessage(nodeID) + if err != nil { + return 0, fmt.Errorf("failed to increment cluster prefixed received tracker gauge value for peer %s: %w", nodeID, err) + } + return count, nil +} + +// Load loads the current number of cluster prefixed control messages received by a peer. +// All errors returned from this func are unexpected and irrecoverable. +func (c *ClusterPrefixedMessagesReceivedTracker) Load(nodeID flow.Identifier) (float64, error) { + count, _, err := c.cache.Get(nodeID) + if err != nil { + return 0, fmt.Errorf("failed to get cluster prefixed received tracker gauge value for peer %s: %w", nodeID, err) + } + return count, nil +} + +// StoreActiveClusterIds stores the active cluster Ids in the underlying record cache. +func (c *ClusterPrefixedMessagesReceivedTracker) StoreActiveClusterIds(clusterIdList flow.ChainIDList) { + c.activeClusterIds.Store(&clusterIdList) +} + +// GetActiveClusterIds gets the active cluster Ids from the underlying record cache. +func (c *ClusterPrefixedMessagesReceivedTracker) GetActiveClusterIds() flow.ChainIDList { + return *c.activeClusterIds.Load() +} diff --git a/network/p2p/inspector/internal/cache/record.go b/network/p2p/inspector/internal/cache/record.go new file mode 100644 index 00000000000..3fcf1fec80d --- /dev/null +++ b/network/p2p/inspector/internal/cache/record.go @@ -0,0 +1,40 @@ +package cache + +import ( + "time" + + "github.com/onflow/flow-go/model/flow" +) + +// ClusterPrefixedMessagesReceivedRecord cache record that keeps track of the amount of cluster prefixed control messages received from a peer. +// This struct implements the flow.Entity interface and uses node ID of the sender for deduplication. +type ClusterPrefixedMessagesReceivedRecord struct { + // NodeID the node ID of the sender. + NodeID flow.Identifier + // Gauge represents the approximate amount of cluster prefixed messages received by a peer, this + // value is decayed back to 0 after some time. + Gauge float64 + lastUpdated time.Time +} + +func NewClusterPrefixedMessagesReceivedRecord(nodeID flow.Identifier) ClusterPrefixedMessagesReceivedRecord { + return ClusterPrefixedMessagesReceivedRecord{ + NodeID: nodeID, + Gauge: 0.0, + lastUpdated: time.Now(), + } +} + +var _ flow.Entity = (*ClusterPrefixedMessagesReceivedRecord)(nil) + +// ID returns the node ID of the sender, which is used as the unique identifier of the entity for maintenance and +// deduplication purposes in the cache. +func (c ClusterPrefixedMessagesReceivedRecord) ID() flow.Identifier { + return c.NodeID +} + +// Checksum returns the node ID of the sender, it does not have any purpose in the cache. +// It is implemented to satisfy the flow.Entity interface. +func (c ClusterPrefixedMessagesReceivedRecord) Checksum() flow.Identifier { + return c.NodeID +} diff --git a/network/p2p/inspector/internal/cache/tracker_test.go b/network/p2p/inspector/internal/cache/tracker_test.go new file mode 100644 index 00000000000..4918efc0d1f --- /dev/null +++ b/network/p2p/inspector/internal/cache/tracker_test.go @@ -0,0 +1,138 @@ +package cache + +import ( + "sync" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestClusterPrefixedMessagesReceivedTracker_Inc ensures cluster prefixed received tracker increments a cluster prefixed control messages received gauge value correctly. +func TestClusterPrefixedMessagesReceivedTracker_Inc(t *testing.T) { + tracker := mockTracker(t) + id := unittest.IdentifierFixture() + n := float64(5) + prevGuage := 0.0 + for i := float64(1); i <= n; i++ { + guage, err := tracker.Inc(id) + require.NoError(t, err) + // on each increment the current gauge value should + // always be greater than the previous gauge value but + // slightly less than i due to the decay. + require.LessOrEqual(t, guage, i) + require.Greater(t, guage, prevGuage) + } +} + +// TestClusterPrefixedMessagesReceivedTracker_IncConcurrent ensures cluster prefixed received tracker increments a cluster prefixed control messages received gauge value correctly concurrently. +func TestClusterPrefixedMessagesReceivedTracker_IncConcurrent(t *testing.T) { + tracker := mockTracker(t) + n := float64(5) + id := unittest.IdentifierFixture() + var wg sync.WaitGroup + wg.Add(5) + for i := float64(0); i < n; i++ { + go func() { + defer wg.Done() + _, err := tracker.Inc(id) + require.NoError(t, err) + }() + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + // after each decay is applied the gauge value result should be slightly less than n + gaugeVal, err := tracker.Load(id) + require.NoError(t, err) + require.InDelta(t, n, gaugeVal, .2) +} + +// TestClusterPrefixedMessagesReceivedTracker_ConcurrentIncAndLoad ensures cluster prefixed received tracker increments/loads the cluster prefixed control messages received gauge value correctly concurrently. +func TestClusterPrefixedMessagesReceivedTracker_ConcurrentIncAndLoad(t *testing.T) { + tracker := mockTracker(t) + n := float64(5) + id := unittest.IdentifierFixture() + var wg sync.WaitGroup + wg.Add(10) + go func() { + for i := float64(0); i < n; i++ { + go func() { + defer wg.Done() + _, err := tracker.Inc(id) + require.NoError(t, err) + }() + } + }() + go func() { + for i := float64(0); i < n; i++ { + go func() { + defer wg.Done() + gaugeVal, err := tracker.Load(id) + require.NoError(t, err) + require.Greater(t, gaugeVal, float64(0)) + require.LessOrEqual(t, gaugeVal, n) + }() + } + }() + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + gaugeVal, err := tracker.Load(id) + require.NoError(t, err) + // after each decay is applied the gauge value result should be slightly less than n + require.InDelta(t, n, gaugeVal, .2) +} + +func TestClusterPrefixedMessagesReceivedTracker_StoreAndGetActiveClusterIds(t *testing.T) { + tracker := mockTracker(t) + activeClusterIds := []flow.ChainIDList{chainIDListFixture(), chainIDListFixture(), chainIDListFixture()} + for _, chainIDList := range activeClusterIds { + tracker.StoreActiveClusterIds(chainIDList) + actualChainIdList := tracker.GetActiveClusterIds() + require.Equal(t, chainIDList, actualChainIdList) + } +} + +func TestClusterPrefixedMessagesReceivedTracker_StoreAndGetActiveClusterIdsConcurrent(t *testing.T) { + tracker := mockTracker(t) + activeClusterIds := []flow.ChainIDList{chainIDListFixture(), chainIDListFixture(), chainIDListFixture()} + expectedLen := len(activeClusterIds[0]) + var wg sync.WaitGroup + wg.Add(len(activeClusterIds)) + for _, chainIDList := range activeClusterIds { + go func(ids flow.ChainIDList) { + defer wg.Done() + tracker.StoreActiveClusterIds(ids) + actualChainIdList := tracker.GetActiveClusterIds() + require.NotNil(t, actualChainIdList) + require.Equal(t, expectedLen, len(actualChainIdList)) // each fixture is of the same len + }(chainIDList) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + actualChainIdList := tracker.GetActiveClusterIds() + require.NotNil(t, actualChainIdList) + require.Equal(t, expectedLen, len(actualChainIdList)) // each fixture is of the same len +} + +func mockTracker(t *testing.T) *ClusterPrefixedMessagesReceivedTracker { + logger := zerolog.Nop() + sizeLimit := uint32(100) + collector := metrics.NewNoopCollector() + decay := defaultDecay + tracker, err := NewClusterPrefixedMessagesReceivedTracker(logger, sizeLimit, collector, decay) + require.NoError(t, err) + return tracker +} + +func chainIDListFixture() flow.ChainIDList { + return flow.ChainIDList{ + flow.ChainID(unittest.IdentifierFixture().String()), + flow.ChainID(unittest.IdentifierFixture().String()), + flow.ChainID(unittest.IdentifierFixture().String()), + flow.ChainID(unittest.IdentifierFixture().String()), + } +} diff --git a/network/p2p/inspector/internal/ratelimit/control_message_rate_limiter.go b/network/p2p/inspector/internal/ratelimit/control_message_rate_limiter.go index 6a43c87ff96..e3d135383e5 100644 --- a/network/p2p/inspector/internal/ratelimit/control_message_rate_limiter.go +++ b/network/p2p/inspector/internal/ratelimit/control_message_rate_limiter.go @@ -4,9 +4,11 @@ import ( "time" "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog" "golang.org/x/time/rate" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/network/p2p/utils/ratelimiter" ) @@ -19,7 +21,13 @@ var _ p2p.BasicRateLimiter = (*ControlMessageRateLimiter)(nil) // NewControlMessageRateLimiter returns a new ControlMessageRateLimiter. The cleanup loop will be started in a // separate goroutine and should be stopped by calling Close. -func NewControlMessageRateLimiter(limit rate.Limit, burst int) p2p.BasicRateLimiter { +func NewControlMessageRateLimiter(logger zerolog.Logger, limit rate.Limit, burst int) p2p.BasicRateLimiter { + if limit == 0 { + logger.Warn().Msg("control message rate limit set to 0 using noop rate limiter") + // setup noop rate limiter if rate limiting is disabled + return ratelimit.NewNoopRateLimiter() + } + // NOTE: we use a lockout duration of 0 because we only need to expose the basic functionality of the // rate limiter and not the lockout feature. lockoutDuration := time.Duration(0) diff --git a/network/p2p/inspector/validation/control_message_validation.go b/network/p2p/inspector/validation/control_message_validation.go deleted file mode 100644 index 86ec4bd7e57..00000000000 --- a/network/p2p/inspector/validation/control_message_validation.go +++ /dev/null @@ -1,318 +0,0 @@ -package validation - -import ( - "fmt" - - pubsub "github.com/libp2p/go-libp2p-pubsub" - pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/engine/common/worker" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/component" - "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/mempool/queue" - "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/network/channels" - "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/inspector/internal" - "github.com/onflow/flow-go/utils/logging" -) - -const ( - // DefaultNumberOfWorkers default number of workers for the inspector component. - DefaultNumberOfWorkers = 5 - // DefaultControlMsgValidationInspectorQueueCacheSize is the default size of the inspect message queue. - DefaultControlMsgValidationInspectorQueueCacheSize = 100 - // rpcInspectorComponentName the rpc inspector component name. - rpcInspectorComponentName = "gossipsub_rpc_validation_inspector" -) - -// InspectMsgRequest represents a short digest of an RPC control message. It is used for further message inspection by component workers. -type InspectMsgRequest struct { - // Nonce adds random value so that when msg req is stored on hero store a unique ID can be created from the struct fields. - Nonce []byte - // Peer sender of the message. - Peer peer.ID - // CtrlMsg the control message that will be inspected. - ctrlMsg *pubsub_pb.ControlMessage - validationConfig *CtrlMsgValidationConfig -} - -// ControlMsgValidationInspectorConfig validation configuration for each type of RPC control message. -type ControlMsgValidationInspectorConfig struct { - // NumberOfWorkers number of component workers to start for processing RPC messages. - NumberOfWorkers int - // InspectMsgStoreOpts options used to configure the underlying herocache message store. - InspectMsgStoreOpts []queue.HeroStoreConfigOption - // GraftValidationCfg validation configuration for GRAFT control messages. - GraftValidationCfg *CtrlMsgValidationConfig - // PruneValidationCfg validation configuration for PRUNE control messages. - PruneValidationCfg *CtrlMsgValidationConfig -} - -// getCtrlMsgValidationConfig returns the CtrlMsgValidationConfig for the specified p2p.ControlMessageType. -func (conf *ControlMsgValidationInspectorConfig) getCtrlMsgValidationConfig(controlMsg p2p.ControlMessageType) (*CtrlMsgValidationConfig, bool) { - switch controlMsg { - case p2p.CtrlMsgGraft: - return conf.GraftValidationCfg, true - case p2p.CtrlMsgPrune: - return conf.PruneValidationCfg, true - default: - return nil, false - } -} - -// allCtrlMsgValidationConfig returns all control message validation configs in a list. -func (conf *ControlMsgValidationInspectorConfig) allCtrlMsgValidationConfig() CtrlMsgValidationConfigs { - return CtrlMsgValidationConfigs{conf.GraftValidationCfg, conf.PruneValidationCfg} -} - -// ControlMsgValidationInspector RPC message inspector that inspects control messages and performs some validation on them, -// when some validation rule is broken feedback is given via the Peer scoring notifier. -type ControlMsgValidationInspector struct { - component.Component - logger zerolog.Logger - sporkID flow.Identifier - // config control message validation configurations. - config *ControlMsgValidationInspectorConfig - // distributor used to disseminate invalid RPC message notifications. - distributor p2p.GossipSubInspectorNotificationDistributor - // workerPool queue that stores *InspectMsgRequest that will be processed by component workers. - workerPool *worker.Pool[*InspectMsgRequest] -} - -var _ component.Component = (*ControlMsgValidationInspector)(nil) -var _ p2p.GossipSubRPCInspector = (*ControlMsgValidationInspector)(nil) - -// NewInspectMsgRequest returns a new *InspectMsgRequest. -func NewInspectMsgRequest(from peer.ID, validationConfig *CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage) (*InspectMsgRequest, error) { - nonce, err := internal.Nonce() - if err != nil { - return nil, fmt.Errorf("failed to get inspect message request nonce: %w", err) - } - return &InspectMsgRequest{Nonce: nonce, Peer: from, validationConfig: validationConfig, ctrlMsg: ctrlMsg}, nil -} - -// NewControlMsgValidationInspector returns new ControlMsgValidationInspector -func NewControlMsgValidationInspector( - logger zerolog.Logger, - sporkID flow.Identifier, - config *ControlMsgValidationInspectorConfig, - distributor p2p.GossipSubInspectorNotificationDistributor, -) *ControlMsgValidationInspector { - lg := logger.With().Str("component", "gossip_sub_rpc_validation_inspector").Logger() - c := &ControlMsgValidationInspector{ - logger: lg, - sporkID: sporkID, - config: config, - distributor: distributor, - } - - cfg := &queue.HeroStoreConfig{ - SizeLimit: DefaultControlMsgValidationInspectorQueueCacheSize, - Collector: metrics.NewNoopCollector(), - } - - for _, opt := range config.InspectMsgStoreOpts { - opt(cfg) - } - - store := queue.NewHeroStore(cfg.SizeLimit, logger, cfg.Collector) - pool := worker.NewWorkerPoolBuilder[*InspectMsgRequest](lg, store, c.processInspectMsgReq).Build() - - c.workerPool = pool - - builder := component.NewComponentManagerBuilder() - // start rate limiters cleanup loop in workers - for _, conf := range c.config.allCtrlMsgValidationConfig() { - validationConfig := conf - builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - ready() - validationConfig.RateLimiter.Start(ctx) - }) - } - for i := 0; i < c.config.NumberOfWorkers; i++ { - builder.AddWorker(pool.WorkerLogic()) - } - c.Component = builder.Build() - return c -} - -// Inspect inspects the rpc received and returns an error if any validation rule is broken. -// For each control message type an initial inspection is done synchronously to check the amount -// of messages in the control message. Further inspection is done asynchronously to check rate limits -// and validate topic IDS each control message if initial validation is passed. -// All errors returned from this function can be considered benign. -// errors returned: -// -// ErrDiscardThreshold - if the message count for the control message type exceeds the discard threshold. -func (c *ControlMsgValidationInspector) Inspect(from peer.ID, rpc *pubsub.RPC) error { - control := rpc.GetControl() - for _, ctrlMsgType := range p2p.ControlMessageTypes() { - lg := c.logger.With(). - Str("peer_id", from.String()). - Str("ctrl_msg_type", string(ctrlMsgType)).Logger() - validationConfig, ok := c.config.getCtrlMsgValidationConfig(ctrlMsgType) - if !ok { - lg.Trace().Msg("validation configuration for control type does not exists skipping") - continue - } - - // mandatory blocking pre-processing of RPC to check discard threshold. - err := c.blockingPreprocessingRpc(from, validationConfig, control) - if err != nil { - lg.Error(). - Err(err). - Str("peer_id", from.String()). - Str("ctrl_msg_type", string(ctrlMsgType)). - Msg("could not pre-process rpc, aborting") - return fmt.Errorf("could not pre-process rpc, aborting: %w", err) - } - - // queue further async inspection - req, err := NewInspectMsgRequest(from, validationConfig, control) - if err != nil { - lg.Error(). - Err(err). - Str("peer_id", from.String()). - Str("ctrl_msg_type", string(ctrlMsgType)). - Msg("failed to get inspect message request") - return fmt.Errorf("failed to get inspect message request: %w", err) - } - c.workerPool.Submit(req) - } - - return nil -} - -// Name returns the name of the rpc inspector. -func (c *ControlMsgValidationInspector) Name() string { - return rpcInspectorComponentName -} - -// blockingPreprocessingRpc ensures the RPC control message count does not exceed the configured discard threshold. -func (c *ControlMsgValidationInspector) blockingPreprocessingRpc(from peer.ID, validationConfig *CtrlMsgValidationConfig, controlMessage *pubsub_pb.ControlMessage) error { - lg := c.logger.With(). - Str("peer_id", from.String()). - Str("ctrl_msg_type", string(validationConfig.ControlMsg)).Logger() - - count := c.getCtrlMsgCount(validationConfig.ControlMsg, controlMessage) - // if Count greater than discard threshold drop message and penalize - if count > validationConfig.DiscardThreshold { - discardThresholdErr := NewDiscardThresholdErr(validationConfig.ControlMsg, count, validationConfig.DiscardThreshold) - lg.Warn(). - Err(discardThresholdErr). - Uint64("ctrl_msg_count", count). - Uint64("upper_threshold", discardThresholdErr.discardThreshold). - Bool(logging.KeySuspicious, true). - Msg("rejecting rpc control message") - err := c.distributor.DistributeInvalidControlMessageNotification(p2p.NewInvalidControlMessageNotification(from, validationConfig.ControlMsg, count, discardThresholdErr)) - if err != nil { - lg.Error(). - Err(err). - Bool(logging.KeySuspicious, true). - Msg("failed to distribute invalid control message notification") - return err - } - return discardThresholdErr - } - - return nil -} - -// processInspectMsgReq func used by component workers to perform further inspection of control messages that will check if the messages are rate limited -// and ensure all topic IDS are valid when the amount of messages is above the configured safety threshold. -func (c *ControlMsgValidationInspector) processInspectMsgReq(req *InspectMsgRequest) error { - count := c.getCtrlMsgCount(req.validationConfig.ControlMsg, req.ctrlMsg) - lg := c.logger.With(). - Str("peer_id", req.Peer.String()). - Str("ctrl_msg_type", string(req.validationConfig.ControlMsg)). - Uint64("ctrl_msg_count", count).Logger() - var validationErr error - switch { - case !req.validationConfig.RateLimiter.Allow(req.Peer, int(count)): // check if Peer RPC messages are rate limited - validationErr = NewRateLimitedControlMsgErr(req.validationConfig.ControlMsg) - case count > req.validationConfig.SafetyThreshold: // check if Peer RPC messages Count greater than safety threshold further inspect each message individually - validationErr = c.validateTopics(req.validationConfig.ControlMsg, req.ctrlMsg) - default: - lg.Trace(). - Uint64("upper_threshold", req.validationConfig.DiscardThreshold). - Uint64("safety_threshold", req.validationConfig.SafetyThreshold). - Msg(fmt.Sprintf("control message %s inspection passed %d is below configured safety threshold", req.validationConfig.ControlMsg, count)) - return nil - } - if validationErr != nil { - lg.Error(). - Err(validationErr). - Bool(logging.KeySuspicious, true). - Msg("rpc control message async inspection failed") - err := c.distributor.DistributeInvalidControlMessageNotification(p2p.NewInvalidControlMessageNotification(req.Peer, req.validationConfig.ControlMsg, count, validationErr)) - if err != nil { - lg.Error(). - Err(err). - Bool(logging.KeySuspicious, true). - Msg("failed to distribute invalid control message notification") - } - } - return nil -} - -// getCtrlMsgCount returns the amount of specified control message type in the rpc ControlMessage. -func (c *ControlMsgValidationInspector) getCtrlMsgCount(ctrlMsgType p2p.ControlMessageType, ctrlMsg *pubsub_pb.ControlMessage) uint64 { - switch ctrlMsgType { - case p2p.CtrlMsgGraft: - return uint64(len(ctrlMsg.GetGraft())) - case p2p.CtrlMsgPrune: - return uint64(len(ctrlMsg.GetPrune())) - default: - return 0 - } -} - -// validateTopics ensures all topics in the specified control message are valid flow topic/channel and no duplicate topics exist. -// All errors returned from this function can be considered benign. -func (c *ControlMsgValidationInspector) validateTopics(ctrlMsgType p2p.ControlMessageType, ctrlMsg *pubsub_pb.ControlMessage) error { - seen := make(map[channels.Topic]struct{}) - validateTopic := func(topic channels.Topic) error { - if _, ok := seen[topic]; ok { - return NewIDuplicateTopicErr(topic) - } - seen[topic] = struct{}{} - err := c.validateTopic(topic) - if err != nil { - return err - } - return nil - } - switch ctrlMsgType { - case p2p.CtrlMsgGraft: - for _, graft := range ctrlMsg.GetGraft() { - topic := channels.Topic(graft.GetTopicID()) - err := validateTopic(topic) - if err != nil { - return err - } - } - case p2p.CtrlMsgPrune: - for _, prune := range ctrlMsg.GetPrune() { - topic := channels.Topic(prune.GetTopicID()) - err := validateTopic(topic) - if err != nil { - return err - } - } - } - return nil -} - -// validateTopic the topic is a valid flow topic/channel. -// All errors returned from this function can be considered benign. -func (c *ControlMsgValidationInspector) validateTopic(topic channels.Topic) error { - err := channels.IsValidFlowTopic(topic, c.sporkID) - if err != nil { - return NewInvalidTopicErr(topic, err) - } - return nil -} diff --git a/network/p2p/inspector/validation/control_message_validation_config.go b/network/p2p/inspector/validation/control_message_validation_config.go deleted file mode 100644 index 61162207f4e..00000000000 --- a/network/p2p/inspector/validation/control_message_validation_config.go +++ /dev/null @@ -1,95 +0,0 @@ -package validation - -import ( - "golang.org/x/time/rate" - - "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/inspector/internal/ratelimit" -) - -const ( - // DiscardThresholdMapKey key used to set the discard threshold config limit. - DiscardThresholdMapKey = "discardthreshold" - // SafetyThresholdMapKey key used to set the safety threshold config limit. - SafetyThresholdMapKey = "safetythreshold" - // RateLimitMapKey key used to set the rate limit config limit. - RateLimitMapKey = "ratelimit" - - // DefaultGraftDiscardThreshold upper bound for graft messages, RPC control messages with a count - // above the discard threshold are automatically discarded. - DefaultGraftDiscardThreshold = 30 - // DefaultGraftSafetyThreshold a lower bound for graft messages, RPC control messages with a message count - // lower than the safety threshold bypass validation. - DefaultGraftSafetyThreshold = .5 * DefaultGraftDiscardThreshold - // DefaultGraftRateLimit the rate limit for graft control messages. - // Currently, the default rate limit is equal to the discard threshold amount. - // This will result in a rate limit of 30 grafts/sec. - DefaultGraftRateLimit = DefaultGraftDiscardThreshold - - // DefaultPruneDiscardThreshold upper bound for prune messages, RPC control messages with a count - // above the discard threshold are automatically discarded. - DefaultPruneDiscardThreshold = 30 - // DefaultPruneSafetyThreshold a lower bound for prune messages, RPC control messages with a message count - // lower than the safety threshold bypass validation. - DefaultPruneSafetyThreshold = .5 * DefaultPruneDiscardThreshold - // DefaultPruneRateLimit the rate limit for prune control messages. - // Currently, the default rate limit is equal to the discard threshold amount. - // This will result in a rate limit of 30 prunes/sec. - DefaultPruneRateLimit = DefaultPruneDiscardThreshold -) - -// CtrlMsgValidationLimits limits used to construct control message validation configuration. -type CtrlMsgValidationLimits map[string]int - -func (c CtrlMsgValidationLimits) DiscardThreshold() uint64 { - return uint64(c[DiscardThresholdMapKey]) -} - -func (c CtrlMsgValidationLimits) SafetyThreshold() uint64 { - return uint64(c[SafetyThresholdMapKey]) -} - -func (c CtrlMsgValidationLimits) RateLimit() int { - return c[RateLimitMapKey] -} - -// CtrlMsgValidationConfigs list of *CtrlMsgValidationConfig -type CtrlMsgValidationConfigs []*CtrlMsgValidationConfig - -// CtrlMsgValidationConfig configuration values for upper, lower threshold and rate limit. -type CtrlMsgValidationConfig struct { - // ControlMsg the type of RPC control message. - ControlMsg p2p.ControlMessageType - // DiscardThreshold indicates the hard limit for size of the RPC control message - // any RPC messages with size > DiscardThreshold should be dropped. - DiscardThreshold uint64 - // SafetyThreshold lower limit for the size of the RPC control message, any RPC messages - // with a size < SafetyThreshold can skip validation step to avoid resource wasting. - SafetyThreshold uint64 - - // RateLimiter basic limiter without lockout duration. - RateLimiter p2p.BasicRateLimiter -} - -// NewCtrlMsgValidationConfig ensures each config limit value is greater than 0 before returning a new CtrlMsgValidationConfig. -// errors returned: -// -// ErrValidationLimit - if any of the validation limits provided are less than 0. This error is non-recoverable -// and the node should crash if this error is encountered. -func NewCtrlMsgValidationConfig(controlMsg p2p.ControlMessageType, cfgLimitValues CtrlMsgValidationLimits) (*CtrlMsgValidationConfig, error) { - switch { - case cfgLimitValues.RateLimit() <= 0: - return nil, NewInvalidLimitConfigErr(controlMsg, RateLimitMapKey, uint64(cfgLimitValues.RateLimit())) - case cfgLimitValues.DiscardThreshold() <= 0: - return nil, NewInvalidLimitConfigErr(controlMsg, DiscardThresholdMapKey, cfgLimitValues.DiscardThreshold()) - case cfgLimitValues.RateLimit() <= 0: - return nil, NewInvalidLimitConfigErr(controlMsg, SafetyThresholdMapKey, cfgLimitValues.SafetyThreshold()) - default: - return &CtrlMsgValidationConfig{ - ControlMsg: controlMsg, - DiscardThreshold: cfgLimitValues.DiscardThreshold(), - SafetyThreshold: cfgLimitValues.SafetyThreshold(), - RateLimiter: ratelimit.NewControlMessageRateLimiter(rate.Limit(cfgLimitValues.RateLimit()), cfgLimitValues.RateLimit()), - }, nil - } -} diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go new file mode 100644 index 00000000000..7daf3f38599 --- /dev/null +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -0,0 +1,601 @@ +package validation + +import ( + "fmt" + "time" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog" + "golang.org/x/time/rate" + + "github.com/onflow/flow-go/engine/common/worker" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mempool/queue" + "github.com/onflow/flow-go/module/util" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/inspector/internal/cache" + "github.com/onflow/flow-go/network/p2p/inspector/internal/ratelimit" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" + "github.com/onflow/flow-go/network/p2p/p2pconf" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/events" + "github.com/onflow/flow-go/utils/logging" + flowrand "github.com/onflow/flow-go/utils/rand" +) + +// ControlMsgValidationInspector RPC message inspector that inspects control messages and performs some validation on them, +// when some validation rule is broken feedback is given via the Peer scoring notifier. +type ControlMsgValidationInspector struct { + component.Component + events.Noop + logger zerolog.Logger + sporkID flow.Identifier + metrics module.GossipSubRpcValidationInspectorMetrics + // config control message validation configurations. + config *p2pconf.GossipSubRPCValidationInspectorConfigs + // distributor used to disseminate invalid RPC message notifications. + distributor p2p.GossipSubInspectorNotifDistributor + // workerPool queue that stores *InspectMsgRequest that will be processed by component workers. + workerPool *worker.Pool[*InspectMsgRequest] + // tracker is a map that associates the hash of a peer's ID with the + // number of cluster-prefix topic control messages received from that peer. It helps in tracking + // and managing the rate of incoming control messages from each peer, ensuring that the system + // stays performant and resilient against potential spam or abuse. + // The counter is incremented in the following scenarios: + // 1. The cluster prefix topic is received while the inspector waits for the cluster IDs provider to be set (this can happen during the startup or epoch transitions). + // 2. The node sends a cluster prefix topic where the cluster prefix does not match any of the active cluster IDs. + // In such cases, the inspector will allow a configured number of these messages from the corresponding peer. + tracker *cache.ClusterPrefixedMessagesReceivedTracker + idProvider module.IdentityProvider + rateLimiters map[p2pmsg.ControlMessageType]p2p.BasicRateLimiter +} + +var _ component.Component = (*ControlMsgValidationInspector)(nil) +var _ p2p.GossipSubRPCInspector = (*ControlMsgValidationInspector)(nil) +var _ protocol.Consumer = (*ControlMsgValidationInspector)(nil) + +// NewControlMsgValidationInspector returns new ControlMsgValidationInspector +// Args: +// - logger: the logger used by the inspector. +// - sporkID: the current spork ID. +// - config: inspector configuration. +// - distributor: gossipsub inspector notification distributor. +// - clusterPrefixedCacheCollector: metrics collector for the underlying cluster prefix received tracker cache. +// - idProvider: identity provider is used to get the flow identifier for a peer. +// +// Returns: +// - *ControlMsgValidationInspector: a new control message validation inspector. +// - error: an error if there is any error while creating the inspector. All errors are irrecoverable and unexpected. +func NewControlMsgValidationInspector( + logger zerolog.Logger, + sporkID flow.Identifier, + config *p2pconf.GossipSubRPCValidationInspectorConfigs, + distributor p2p.GossipSubInspectorNotifDistributor, + inspectMsgQueueCacheCollector module.HeroCacheMetrics, + clusterPrefixedCacheCollector module.HeroCacheMetrics, + idProvider module.IdentityProvider, + inspectorMetrics module.GossipSubRpcValidationInspectorMetrics) (*ControlMsgValidationInspector, error) { + lg := logger.With().Str("component", "gossip_sub_rpc_validation_inspector").Logger() + + tracker, err := cache.NewClusterPrefixedMessagesReceivedTracker(logger, config.ClusterPrefixedControlMsgsReceivedCacheSize, clusterPrefixedCacheCollector, config.ClusterPrefixedControlMsgsReceivedCacheDecay) + if err != nil { + return nil, fmt.Errorf("failed to create cluster prefix topics received tracker") + } + + c := &ControlMsgValidationInspector{ + logger: lg, + sporkID: sporkID, + config: config, + distributor: distributor, + tracker: tracker, + idProvider: idProvider, + metrics: inspectorMetrics, + rateLimiters: make(map[p2pmsg.ControlMessageType]p2p.BasicRateLimiter), + } + + store := queue.NewHeroStore(config.CacheSize, logger, inspectMsgQueueCacheCollector) + pool := worker.NewWorkerPoolBuilder[*InspectMsgRequest](lg, store, c.processInspectMsgReq).Build() + + c.workerPool = pool + + builder := component.NewComponentManagerBuilder() + builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + distributor.Start(ctx) + select { + case <-ctx.Done(): + case <-distributor.Ready(): + ready() + } + <-distributor.Done() + }) + // start rate limiters cleanup loop in workers + for _, conf := range c.config.AllCtrlMsgValidationConfig() { + l := logger.With().Str("control_message_type", conf.ControlMsg.String()).Logger() + limiter := ratelimit.NewControlMessageRateLimiter(l, rate.Limit(conf.RateLimit), conf.RateLimit) + c.rateLimiters[conf.ControlMsg] = limiter + builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + limiter.Start(ctx) + }) + } + for i := 0; i < c.config.NumberOfWorkers; i++ { + builder.AddWorker(pool.WorkerLogic()) + } + c.Component = builder.Build() + return c, nil +} + +// Inspect is called by gossipsub upon reception of an rpc from a remote node. +// It examines the provided message to ensure it adheres to the expected +// format and conventions. If the message passes validation, the method returns +// a nil error. If an issue is found, the method returns an error detailing +// the specific issue encountered. +// The returned error can be of two types: +// 1. Expected errors: These are issues that are expected to occur during normal +// operation, such as invalid messages or messages that don't follow the +// conventions. These errors should be handled gracefully by the caller. +// 2. Exceptions: These are unexpected issues, such as internal system errors +// or misconfigurations, that may require immediate attention or a change in +// the system's behavior. The caller should log and handle these errors +// accordingly. +// +// The returned error is returned to the gossipsub node which causes the rejection of rpc (for non-nil errors). +func (c *ControlMsgValidationInspector) Inspect(from peer.ID, rpc *pubsub.RPC) error { + control := rpc.GetControl() + for _, ctrlMsgType := range p2pmsg.ControlMessageTypes() { + lg := c.logger.With(). + Str("peer_id", from.String()). + Str("ctrl_msg_type", string(ctrlMsgType)).Logger() + validationConfig, ok := c.config.GetCtrlMsgValidationConfig(ctrlMsgType) + if !ok { + lg.Trace().Msg("validation configuration for control type does not exists skipping") + continue + } + + switch ctrlMsgType { + case p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune: + // normal pre-processing + err := c.blockingPreprocessingRpc(from, validationConfig, control) + if err != nil { + lg.Error(). + Err(err). + Msg("could not pre-process rpc, aborting") + return fmt.Errorf("could not pre-process rpc, aborting: %w", err) + } + case p2pmsg.CtrlMsgIHave: + // iHave specific pre-processing + sampleSize := util.SampleN(len(control.GetIhave()), c.config.IHaveInspectionMaxSampleSize, c.config.IHaveSyncInspectSampleSizePercentage) + err := c.blockingIHaveSamplePreprocessing(from, validationConfig, control, sampleSize) + if err != nil { + lg.Error(). + Err(err). + Msg("could not pre-process rpc, aborting") + return fmt.Errorf("could not pre-process rpc, aborting: %w", err) + } + } + + // queue further async inspection + req, err := NewInspectMsgRequest(from, validationConfig, control) + if err != nil { + lg.Error(). + Err(err). + Str("peer_id", from.String()). + Str("ctrl_msg_type", string(ctrlMsgType)). + Msg("failed to get inspect message request") + return fmt.Errorf("failed to get inspect message request: %w", err) + } + c.workerPool.Submit(req) + } + + return nil +} + +// Name returns the name of the rpc inspector. +func (c *ControlMsgValidationInspector) Name() string { + return rpcInspectorComponentName +} + +// ActiveClustersChanged consumes cluster ID update protocol events. +func (c *ControlMsgValidationInspector) ActiveClustersChanged(clusterIDList flow.ChainIDList) { + c.tracker.StoreActiveClusterIds(clusterIDList) +} + +// blockingPreprocessingRpc ensures the RPC control message count does not exceed the configured discard threshold. +// Expected error returns during normal operations: +// - ErrDiscardThreshold: if control message count exceeds the configured discard threshold. +// +// blockingPreprocessingRpc generic pre-processing validation func that ensures the RPC control message count does not exceed the configured hard threshold. +func (c *ControlMsgValidationInspector) blockingPreprocessingRpc(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, controlMessage *pubsub_pb.ControlMessage) error { + if validationConfig.ControlMsg != p2pmsg.CtrlMsgGraft && validationConfig.ControlMsg != p2pmsg.CtrlMsgPrune { + return fmt.Errorf("unexpected control message type %s encountered during blocking pre-processing rpc, expected %s or %s", validationConfig.ControlMsg, p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune) + } + count := c.getCtrlMsgCount(validationConfig.ControlMsg, controlMessage) + lg := c.logger.With(). + Uint64("ctrl_msg_count", count). + Str("peer_id", from.String()). + Str("ctrl_msg_type", string(validationConfig.ControlMsg)).Logger() + + c.metrics.BlockingPreProcessingStarted(validationConfig.ControlMsg.String(), uint(count)) + start := time.Now() + defer func() { + c.metrics.BlockingPreProcessingFinished(validationConfig.ControlMsg.String(), uint(count), time.Since(start)) + }() + + // if Count greater than hard threshold drop message and penalize + if count > validationConfig.HardThreshold { + hardThresholdErr := NewHardThresholdErr(validationConfig.ControlMsg, count, validationConfig.HardThreshold) + lg.Warn(). + Err(hardThresholdErr). + Uint64("upper_threshold", hardThresholdErr.hardThreshold). + Bool(logging.KeySuspicious, true). + Msg("rejecting rpc control message") + err := c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(from, validationConfig.ControlMsg, count, hardThresholdErr)) + if err != nil { + lg.Error(). + Err(err). + Bool(logging.KeySuspicious, true). + Msg("failed to distribute invalid control message notification") + return err + } + return hardThresholdErr + } + + return nil +} + +// blockingPreprocessingSampleRpc blocking pre-processing of a sample of iHave control messages. +func (c *ControlMsgValidationInspector) blockingIHaveSamplePreprocessing(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, controlMessage *pubsub_pb.ControlMessage, sampleSize uint) error { + c.metrics.BlockingPreProcessingStarted(p2pmsg.CtrlMsgIHave.String(), sampleSize) + start := time.Now() + defer func() { + c.metrics.BlockingPreProcessingFinished(p2pmsg.CtrlMsgIHave.String(), sampleSize, time.Since(start)) + }() + err := c.blockingPreprocessingSampleRpc(from, validationConfig, controlMessage, sampleSize) + if err != nil { + return fmt.Errorf("failed to pre-process a sample of iHave messages: %w", err) + } + return nil +} + +// blockingPreprocessingSampleRpc blocking pre-processing validation func that performs some pre-validation of RPC control messages. +// If the RPC control message count exceeds the configured hard threshold we perform synchronous topic validation on a subset +// of the control messages. This is used for control message types that do not have an upper bound on the amount of messages a node can send. +func (c *ControlMsgValidationInspector) blockingPreprocessingSampleRpc(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, controlMessage *pubsub_pb.ControlMessage, sampleSize uint) error { + if validationConfig.ControlMsg != p2pmsg.CtrlMsgIHave && validationConfig.ControlMsg != p2pmsg.CtrlMsgIWant { + return fmt.Errorf("unexpected control message type %s encountered during blocking pre-processing sample rpc, expected %s or %s", validationConfig.ControlMsg, p2pmsg.CtrlMsgIHave, p2pmsg.CtrlMsgIWant) + } + activeClusterIDS := c.tracker.GetActiveClusterIds() + count := c.getCtrlMsgCount(validationConfig.ControlMsg, controlMessage) + lg := c.logger.With(). + Uint64("ctrl_msg_count", count). + Str("peer_id", from.String()). + Str("ctrl_msg_type", string(validationConfig.ControlMsg)).Logger() + // if count greater than hard threshold perform synchronous topic validation on random subset of the iHave messages + if count > validationConfig.HardThreshold { + // for iHave control message topic validation we only validate a random subset of the messages + // shuffle the ihave messages to perform random validation on a subset of size sampleSize + err := c.sampleCtrlMessages(p2pmsg.CtrlMsgIHave, controlMessage, sampleSize) + if err != nil { + return fmt.Errorf("failed to sample ihave messages: %w", err) + } + err = c.validateTopicsSample(from, validationConfig, controlMessage, activeClusterIDS, sampleSize) + if err != nil { + lg.Warn(). + Err(err). + Bool(logging.KeySuspicious, true). + Msg("topic validation pre-processing failed rejecting rpc control message") + disErr := c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(from, validationConfig.ControlMsg, count, err)) + if disErr != nil { + lg.Error(). + Err(disErr). + Bool(logging.KeySuspicious, true). + Msg("failed to distribute invalid control message notification") + return disErr + } + return err + } + } + + // pre-processing validation passed, perform ihave sampling again + // to randomize async validation to avoid data race that can occur when + // performing the sampling asynchronously. + // for iHave control message topic validation we only validate a random subset of the messages + err := c.sampleCtrlMessages(p2pmsg.CtrlMsgIHave, controlMessage, sampleSize) + if err != nil { + return fmt.Errorf("failed to sample ihave messages: %w", err) + } + return nil +} + +// sampleCtrlMessages performs sampling on the specified control message that will randomize +// the items in the control message slice up to index sampleSize-1. +func (c *ControlMsgValidationInspector) sampleCtrlMessages(ctrlMsgType p2pmsg.ControlMessageType, ctrlMsg *pubsub_pb.ControlMessage, sampleSize uint) error { + switch ctrlMsgType { + case p2pmsg.CtrlMsgIHave: + iHaves := ctrlMsg.GetIhave() + swap := func(i, j uint) { + iHaves[i], iHaves[j] = iHaves[j], iHaves[i] + } + err := flowrand.Samples(uint(len(iHaves)), sampleSize, swap) + if err != nil { + return fmt.Errorf("failed to get random sample of ihave control messages: %w", err) + } + } + return nil +} + +// processInspectMsgReq func used by component workers to perform further inspection of control messages that will check if the messages are rate limited +// and ensure all topic IDS are valid when the amount of messages is above the configured safety threshold. +func (c *ControlMsgValidationInspector) processInspectMsgReq(req *InspectMsgRequest) error { + c.metrics.AsyncProcessingStarted(req.validationConfig.ControlMsg.String()) + start := time.Now() + defer func() { + c.metrics.AsyncProcessingFinished(req.validationConfig.ControlMsg.String(), time.Since(start)) + }() + + count := c.getCtrlMsgCount(req.validationConfig.ControlMsg, req.ctrlMsg) + lg := c.logger.With(). + Str("peer_id", req.Peer.String()). + Str("ctrl_msg_type", string(req.validationConfig.ControlMsg)). + Uint64("ctrl_msg_count", count).Logger() + + var validationErr error + switch { + case !c.rateLimiters[req.validationConfig.ControlMsg].Allow(req.Peer, int(count)): // check if Peer RPC messages are rate limited + validationErr = NewRateLimitedControlMsgErr(req.validationConfig.ControlMsg) + case count > req.validationConfig.SafetyThreshold: + // check if Peer RPC messages Count greater than safety threshold further inspect each message individually + validationErr = c.validateTopics(req.Peer, req.validationConfig, req.ctrlMsg) + default: + lg.Trace(). + Uint64("hard_threshold", req.validationConfig.HardThreshold). + Uint64("safety_threshold", req.validationConfig.SafetyThreshold). + Msg(fmt.Sprintf("control message %s inspection passed %d is below configured safety threshold", req.validationConfig.ControlMsg, count)) + return nil + } + if validationErr != nil { + lg.Error(). + Err(validationErr). + Bool(logging.KeySuspicious, true). + Msg("rpc control message async inspection failed") + err := c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(req.Peer, req.validationConfig.ControlMsg, count, validationErr)) + if err != nil { + lg.Error(). + Err(err). + Bool(logging.KeySuspicious, true). + Msg("failed to distribute invalid control message notification") + } + } + return nil +} + +// getCtrlMsgCount returns the amount of specified control message type in the rpc ControlMessage. +func (c *ControlMsgValidationInspector) getCtrlMsgCount(ctrlMsgType p2pmsg.ControlMessageType, ctrlMsg *pubsub_pb.ControlMessage) uint64 { + switch ctrlMsgType { + case p2pmsg.CtrlMsgGraft: + return uint64(len(ctrlMsg.GetGraft())) + case p2pmsg.CtrlMsgPrune: + return uint64(len(ctrlMsg.GetPrune())) + case p2pmsg.CtrlMsgIHave: + return uint64(len(ctrlMsg.GetIhave())) + default: + return 0 + } +} + +// validateTopics ensures all topics in the specified control message are valid flow topic/channel and no duplicate topics exist. +// Expected error returns during normal operations: +// - channels.InvalidTopicErr: if topic is invalid. +// - ErrDuplicateTopic: if a duplicate topic ID is encountered. +func (c *ControlMsgValidationInspector) validateTopics(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage) error { + activeClusterIDS := c.tracker.GetActiveClusterIds() + switch validationConfig.ControlMsg { + case p2pmsg.CtrlMsgGraft: + return c.validateGrafts(from, ctrlMsg, activeClusterIDS) + case p2pmsg.CtrlMsgPrune: + return c.validatePrunes(from, ctrlMsg, activeClusterIDS) + case p2pmsg.CtrlMsgIHave: + return c.validateIhaves(from, validationConfig, ctrlMsg, activeClusterIDS) + default: + // sanity check + // This should never happen validateTopics is only used to validate GRAFT and PRUNE control message types + // if any other control message type is encountered here this indicates invalid state irrecoverable error. + c.logger.Fatal().Msg(fmt.Sprintf("encountered invalid control message type in validate topics expected %s, %s or %s got %s", p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune, p2pmsg.CtrlMsgIHave, validationConfig.ControlMsg)) + } + return nil +} + +// validateGrafts performs topic validation on all grafts in the control message using the provided validateTopic func while tracking duplicates. +func (c *ControlMsgValidationInspector) validateGrafts(from peer.ID, ctrlMsg *pubsub_pb.ControlMessage, activeClusterIDS flow.ChainIDList) error { + tracker := make(duplicateTopicTracker) + for _, graft := range ctrlMsg.GetGraft() { + topic := channels.Topic(graft.GetTopicID()) + if tracker.isDuplicate(topic) { + return NewDuplicateTopicErr(topic) + } + tracker.set(topic) + err := c.validateTopic(from, topic, activeClusterIDS) + if err != nil { + return err + } + } + return nil +} + +// validatePrunes performs topic validation on all prunes in the control message using the provided validateTopic func while tracking duplicates. +func (c *ControlMsgValidationInspector) validatePrunes(from peer.ID, ctrlMsg *pubsub_pb.ControlMessage, activeClusterIDS flow.ChainIDList) error { + tracker := make(duplicateTopicTracker) + for _, prune := range ctrlMsg.GetPrune() { + topic := channels.Topic(prune.GetTopicID()) + if tracker.isDuplicate(topic) { + return NewDuplicateTopicErr(topic) + } + tracker.set(topic) + err := c.validateTopic(from, topic, activeClusterIDS) + if err != nil { + return err + } + } + return nil +} + +// validateIhaves performs topic validation on all ihaves in the control message using the provided validateTopic func while tracking duplicates. +func (c *ControlMsgValidationInspector) validateIhaves(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage, activeClusterIDS flow.ChainIDList) error { + sampleSize := util.SampleN(len(ctrlMsg.GetIhave()), c.config.IHaveInspectionMaxSampleSize, c.config.IHaveAsyncInspectSampleSizePercentage) + return c.validateTopicsSample(from, validationConfig, ctrlMsg, activeClusterIDS, sampleSize) +} + +// validateTopicsSample samples a subset of topics from the specified control message and ensures the sample contains only valid flow topic/channel and no duplicate topics exist. +// Sample size ensures liveness of the network when validating messages with no upper bound on the amount of messages that may be received. +// All errors returned from this function can be considered benign. +func (c *ControlMsgValidationInspector) validateTopicsSample(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage, activeClusterIDS flow.ChainIDList, sampleSize uint) error { + tracker := make(duplicateTopicTracker) + switch validationConfig.ControlMsg { + case p2pmsg.CtrlMsgIHave: + for i := uint(0); i < sampleSize; i++ { + topic := channels.Topic(ctrlMsg.Ihave[i].GetTopicID()) + if tracker.isDuplicate(topic) { + return NewDuplicateTopicErr(topic) + } + tracker.set(topic) + err := c.validateTopic(from, topic, activeClusterIDS) + if err != nil { + return err + } + } + default: + // sanity check + // This should never happen validateTopicsSample is only used to validate IHAVE control message types + // if any other control message type is encountered here this indicates invalid state irrecoverable error. + c.logger.Fatal().Msg(fmt.Sprintf("encountered invalid control message type in validate topics sample expected %s got %s", p2pmsg.CtrlMsgIHave, validationConfig.ControlMsg)) + } + return nil +} + +// validateTopic ensures the topic is a valid flow topic/channel. +// Expected error returns during normal operations: +// - channels.InvalidTopicErr: if topic is invalid. +// - ErrActiveClusterIdsNotSet: if the cluster ID provider is not set. +// - channels.UnknownClusterIDErr: if the topic contains a cluster ID prefix that is not in the active cluster IDs list. +// +// This func returns an exception in case of unexpected bug or state corruption if cluster prefixed topic validation +// fails due to unexpected error returned when getting the active cluster IDS. +func (c *ControlMsgValidationInspector) validateTopic(from peer.ID, topic channels.Topic, activeClusterIds flow.ChainIDList) error { + channel, ok := channels.ChannelFromTopic(topic) + if !ok { + return channels.NewInvalidTopicErr(topic, fmt.Errorf("failed to get channel from topic")) + } + + // handle cluster prefixed topics + if channels.IsClusterChannel(channel) { + return c.validateClusterPrefixedTopic(from, topic, activeClusterIds) + } + + // non cluster prefixed topic validation + err := channels.IsValidNonClusterFlowTopic(topic, c.sporkID) + if err != nil { + return err + } + return nil +} + +// validateClusterPrefixedTopic validates cluster prefixed topics. +// Expected error returns during normal operations: +// - ErrActiveClusterIdsNotSet: if the cluster ID provider is not set. +// - channels.InvalidTopicErr: if topic is invalid. +// - channels.UnknownClusterIDErr: if the topic contains a cluster ID prefix that is not in the active cluster IDs list. +// +// In the case where an ErrActiveClusterIdsNotSet or UnknownClusterIDErr is encountered and the cluster prefixed topic received +// tracker for the peer is less than or equal to the configured ClusterPrefixHardThreshold an error will only be logged and not returned. +// At the point where the hard threshold is crossed the error will be returned and the sender will start to be penalized. +// Any errors encountered while incrementing or loading the cluster prefixed control message gauge for a peer will result in a fatal log, these +// errors are unexpected and irrecoverable indicating a bug. +func (c *ControlMsgValidationInspector) validateClusterPrefixedTopic(from peer.ID, topic channels.Topic, activeClusterIds flow.ChainIDList) error { + lg := c.logger.With(). + Str("from", from.String()). + Logger() + // reject messages from unstaked nodes for cluster prefixed topics + nodeID, err := c.getFlowIdentifier(from) + if err != nil { + return err + } + + if len(activeClusterIds) == 0 { + // cluster IDs have not been updated yet + _, err = c.tracker.Inc(nodeID) + if err != nil { + return err + } + + // if the amount of messages received is below our hard threshold log the error and return nil. + if c.checkClusterPrefixHardThreshold(nodeID) { + lg.Warn(). + Err(err). + Str("topic", topic.String()). + Msg("failed to validate cluster prefixed control message with cluster pre-fixed topic active cluster ids not set") + return nil + } + + return NewActiveClusterIdsNotSetErr(topic) + } + + err = channels.IsValidFlowClusterTopic(topic, activeClusterIds) + if err != nil { + if channels.IsUnknownClusterIDErr(err) { + // unknown cluster ID error could indicate that a node has fallen + // behind and needs to catchup increment to topics received cache. + _, incErr := c.tracker.Inc(nodeID) + if incErr != nil { + // irrecoverable error encountered + c.logger.Fatal().Err(incErr). + Str("node_id", nodeID.String()). + Msg("unexpected irrecoverable error encountered while incrementing the cluster prefixed control message gauge") + } + // if the amount of messages received is below our hard threshold log the error and return nil. + if c.checkClusterPrefixHardThreshold(nodeID) { + lg.Warn(). + Err(err). + Str("topic", topic.String()). + Msg("processing unknown cluster prefixed topic received below cluster prefixed discard threshold peer may be behind in the protocol") + return nil + } + } + return err + } + + return nil +} + +// getFlowIdentifier returns the flow identity identifier for a peer. +// Args: +// - peerID: the peer id of the sender. +// +// The returned error indicates that the peer is un-staked. +func (c *ControlMsgValidationInspector) getFlowIdentifier(peerID peer.ID) (flow.Identifier, error) { + id, ok := c.idProvider.ByPeerID(peerID) + if !ok { + return flow.ZeroID, NewUnstakedPeerErr(fmt.Errorf("failed to get flow identity for peer: %s", peerID)) + } + return id.ID(), nil +} + +// checkClusterPrefixHardThreshold returns true if the cluster prefix received tracker count is less than +// the configured ClusterPrefixHardThreshold, false otherwise. +// If any error is encountered while loading from the tracker this func will emit a fatal level log, these errors +// are unexpected and irrecoverable indicating a bug. +func (c *ControlMsgValidationInspector) checkClusterPrefixHardThreshold(nodeID flow.Identifier) bool { + gauge, err := c.tracker.Load(nodeID) + if err != nil { + // irrecoverable error encountered + c.logger.Fatal().Err(err). + Str("node_id", nodeID.String()). + Msg("unexpected irrecoverable error encountered while loading the cluster prefixed control message gauge during hard threshold check") + } + return gauge <= c.config.ClusterPrefixHardThreshold +} diff --git a/network/p2p/inspector/validation/errors.go b/network/p2p/inspector/validation/errors.go index ab1cb4be11e..231fe979498 100644 --- a/network/p2p/inspector/validation/errors.go +++ b/network/p2p/inspector/validation/errors.go @@ -5,62 +5,37 @@ import ( "fmt" "github.com/onflow/flow-go/network/channels" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) -// ErrDiscardThreshold indicates that the amount of RPC messages received exceeds discard threshold. -type ErrDiscardThreshold struct { +// ErrHardThreshold indicates that the amount of RPC messages received exceeds hard threshold. +type ErrHardThreshold struct { // controlMsg the control message type. - controlMsg p2p.ControlMessageType + controlMsg p2pmsg.ControlMessageType // amount the amount of control messages. amount uint64 - // discardThreshold configured discard threshold. - discardThreshold uint64 + // hardThreshold configured hard threshold. + hardThreshold uint64 } -func (e ErrDiscardThreshold) Error() string { - return fmt.Sprintf("number of %s messges received exceeds the configured discard threshold: received %d discard threshold %d", e.controlMsg, e.amount, e.discardThreshold) +func (e ErrHardThreshold) Error() string { + return fmt.Sprintf("number of %s messges received exceeds the configured hard threshold: received %d hard threshold %d", e.controlMsg, e.amount, e.hardThreshold) } -// NewDiscardThresholdErr returns a new ErrDiscardThreshold. -func NewDiscardThresholdErr(controlMsg p2p.ControlMessageType, amount, discardThreshold uint64) ErrDiscardThreshold { - return ErrDiscardThreshold{controlMsg: controlMsg, amount: amount, discardThreshold: discardThreshold} +// NewHardThresholdErr returns a new ErrHardThreshold. +func NewHardThresholdErr(controlMsg p2pmsg.ControlMessageType, amount, hardThreshold uint64) ErrHardThreshold { + return ErrHardThreshold{controlMsg: controlMsg, amount: amount, hardThreshold: hardThreshold} } -// IsErrDiscardThreshold returns true if an error is ErrDiscardThreshold -func IsErrDiscardThreshold(err error) bool { - var e ErrDiscardThreshold - return errors.As(err, &e) -} - -// ErrInvalidLimitConfig indicates the validation limit is < 0. -type ErrInvalidLimitConfig struct { - // controlMsg the control message type. - controlMsg p2p.ControlMessageType - // limit the value of the configuration limit. - limit uint64 - // limitStr the string representation of the config limit. - limitStr string -} - -func (e ErrInvalidLimitConfig) Error() string { - return fmt.Sprintf("invalid rpc control message %s validation limit %s configuration value must be greater than 0:%d", e.controlMsg, e.limitStr, e.limit) -} - -// NewInvalidLimitConfigErr returns a new ErrValidationLimit. -func NewInvalidLimitConfigErr(controlMsg p2p.ControlMessageType, limitStr string, limit uint64) ErrInvalidLimitConfig { - return ErrInvalidLimitConfig{controlMsg: controlMsg, limit: limit, limitStr: limitStr} -} - -// IsErrInvalidLimitConfig returns whether an error is ErrInvalidLimitConfig -func IsErrInvalidLimitConfig(err error) bool { - var e ErrInvalidLimitConfig +// IsErrHardThreshold returns true if an error is ErrHardThreshold +func IsErrHardThreshold(err error) bool { + var e ErrHardThreshold return errors.As(err, &e) } // ErrRateLimitedControlMsg indicates the specified RPC control message is rate limited for the specified peer. type ErrRateLimitedControlMsg struct { - controlMsg p2p.ControlMessageType + controlMsg p2pmsg.ControlMessageType } func (e ErrRateLimitedControlMsg) Error() string { @@ -68,53 +43,72 @@ func (e ErrRateLimitedControlMsg) Error() string { } // NewRateLimitedControlMsgErr returns a new ErrValidationLimit. -func NewRateLimitedControlMsgErr(controlMsg p2p.ControlMessageType) ErrRateLimitedControlMsg { +func NewRateLimitedControlMsgErr(controlMsg p2pmsg.ControlMessageType) ErrRateLimitedControlMsg { return ErrRateLimitedControlMsg{controlMsg: controlMsg} } -// IsErrRateLimitedControlMsg returns whether an error is ErrRateLimitedControlMsg +// IsErrRateLimitedControlMsg returns whether an error is ErrRateLimitedControlMsg. func IsErrRateLimitedControlMsg(err error) bool { var e ErrRateLimitedControlMsg return errors.As(err, &e) } -// ErrInvalidTopic error wrapper that indicates an error when checking if a Topic is a valid Flow Topic. -type ErrInvalidTopic struct { +// ErrDuplicateTopic error that indicates a duplicate topic in control message has been detected. +type ErrDuplicateTopic struct { topic channels.Topic - err error } -func (e ErrInvalidTopic) Error() string { - return fmt.Errorf("invalid topic %s: %w", e.topic, e.err).Error() +func (e ErrDuplicateTopic) Error() string { + return fmt.Errorf("duplicate topic %s", e.topic).Error() } -// NewInvalidTopicErr returns a new ErrMalformedTopic -func NewInvalidTopicErr(topic channels.Topic, err error) ErrInvalidTopic { - return ErrInvalidTopic{topic: topic, err: err} +// NewDuplicateTopicErr returns a new ErrDuplicateTopic. +func NewDuplicateTopicErr(topic channels.Topic) ErrDuplicateTopic { + return ErrDuplicateTopic{topic: topic} } -// IsErrInvalidTopic returns true if an error is ErrInvalidTopic -func IsErrInvalidTopic(err error) bool { - var e ErrInvalidTopic +// IsErrDuplicateTopic returns true if an error is ErrDuplicateTopic. +func IsErrDuplicateTopic(err error) bool { + var e ErrDuplicateTopic return errors.As(err, &e) } -// ErrDuplicateTopic error that indicates a duplicate topic in control message has been detected. -type ErrDuplicateTopic struct { +// ErrActiveClusterIdsNotSet error that indicates a cluster prefixed control message has been received but the cluster IDs have not been set yet. +type ErrActiveClusterIdsNotSet struct { topic channels.Topic } -func (e ErrDuplicateTopic) Error() string { - return fmt.Errorf("duplicate topic %s", e.topic).Error() +func (e ErrActiveClusterIdsNotSet) Error() string { + return fmt.Errorf("failed to validate cluster prefixed topic %s no active cluster IDs set", e.topic).Error() } -// NewIDuplicateTopicErr returns a new ErrDuplicateTopic -func NewIDuplicateTopicErr(topic channels.Topic) ErrDuplicateTopic { - return ErrDuplicateTopic{topic: topic} +// NewActiveClusterIdsNotSetErr returns a new ErrActiveClusterIdsNotSet. +func NewActiveClusterIdsNotSetErr(topic channels.Topic) ErrActiveClusterIdsNotSet { + return ErrActiveClusterIdsNotSet{topic: topic} } -// IsErrDuplicateTopic returns true if an error is ErrDuplicateTopic -func IsErrDuplicateTopic(err error) bool { - var e ErrDuplicateTopic +// IsErrActiveClusterIDsNotSet returns true if an error is ErrActiveClusterIdsNotSet. +func IsErrActiveClusterIDsNotSet(err error) bool { + var e ErrActiveClusterIdsNotSet + return errors.As(err, &e) +} + +// ErrUnstakedPeer error that indicates a cluster prefixed control message has been from an unstaked peer. +type ErrUnstakedPeer struct { + err error +} + +func (e ErrUnstakedPeer) Error() string { + return e.err.Error() +} + +// NewUnstakedPeerErr returns a new ErrUnstakedPeer. +func NewUnstakedPeerErr(err error) ErrUnstakedPeer { + return ErrUnstakedPeer{err: err} +} + +// IsErrUnstakedPeer returns true if an error is ErrUnstakedPeer. +func IsErrUnstakedPeer(err error) bool { + var e ErrUnstakedPeer return errors.As(err, &e) } diff --git a/network/p2p/inspector/validation/errors_test.go b/network/p2p/inspector/validation/errors_test.go new file mode 100644 index 00000000000..9bef259fd41 --- /dev/null +++ b/network/p2p/inspector/validation/errors_test.go @@ -0,0 +1,81 @@ +package validation + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/onflow/flow-go/network/channels" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" +) + +// TestErrActiveClusterIDsNotSetRoundTrip ensures correct error formatting for ErrActiveClusterIdsNotSet. +func TestErrActiveClusterIDsNotSetRoundTrip(t *testing.T) { + topic := channels.Topic("test-topic") + err := NewActiveClusterIdsNotSetErr(topic) + + // tests the error message formatting. + expectedErrMsg := fmt.Errorf("failed to validate cluster prefixed topic %s no active cluster IDs set", topic).Error() + assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") + + // tests the IsErrActiveClusterIDsNotSet function. + assert.True(t, IsErrActiveClusterIDsNotSet(err), "IsErrActiveClusterIDsNotSet should return true for ErrActiveClusterIdsNotSet error") + + // test IsErrActiveClusterIDsNotSet with a different error type. + dummyErr := fmt.Errorf("dummy error") + assert.False(t, IsErrActiveClusterIDsNotSet(dummyErr), "IsErrActiveClusterIDsNotSet should return false for non-ErrActiveClusterIdsNotSet error") +} + +// TestErrHardThresholdRoundTrip ensures correct error formatting for ErrHardThreshold. +func TestErrHardThresholdRoundTrip(t *testing.T) { + controlMsg := p2pmsg.CtrlMsgGraft + amount := uint64(100) + hardThreshold := uint64(500) + err := NewHardThresholdErr(controlMsg, amount, hardThreshold) + + // tests the error message formatting. + expectedErrMsg := fmt.Sprintf("number of %s messges received exceeds the configured hard threshold: received %d hard threshold %d", controlMsg, amount, hardThreshold) + assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") + + // tests the IsErrHardThreshold function. + assert.True(t, IsErrHardThreshold(err), "IsErrHardThreshold should return true for ErrHardThreshold error") + + // test IsErrHardThreshold with a different error type. + dummyErr := fmt.Errorf("dummy error") + assert.False(t, IsErrHardThreshold(dummyErr), "IsErrHardThreshold should return false for non-ErrHardThreshold error") +} + +// TestErrRateLimitedControlMsgRoundTrip ensures correct error formatting for ErrRateLimitedControlMsg. +func TestErrRateLimitedControlMsgRoundTrip(t *testing.T) { + controlMsg := p2pmsg.CtrlMsgGraft + err := NewRateLimitedControlMsgErr(controlMsg) + + // tests the error message formatting. + expectedErrMsg := fmt.Sprintf("control message %s is rate limited for peer", controlMsg) + assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") + + // tests the IsErrRateLimitedControlMsg function. + assert.True(t, IsErrRateLimitedControlMsg(err), "IsErrRateLimitedControlMsg should return true for ErrRateLimitedControlMsg error") + + // test IsErrRateLimitedControlMsg with a different error type. + dummyErr := fmt.Errorf("dummy error") + assert.False(t, IsErrRateLimitedControlMsg(dummyErr), "IsErrRateLimitedControlMsg should return false for non-ErrRateLimitedControlMsg error") +} + +// TestErrDuplicateTopicRoundTrip ensures correct error formatting for ErrDuplicateTopic. +func TestErrDuplicateTopicRoundTrip(t *testing.T) { + topic := channels.Topic("topic") + err := NewDuplicateTopicErr(topic) + + // tests the error message formatting. + expectedErrMsg := fmt.Errorf("duplicate topic %s", topic).Error() + assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") + + // tests the IsErrDuplicateTopic function. + assert.True(t, IsErrDuplicateTopic(err), "IsErrDuplicateTopic should return true for ErrDuplicateTopic error") + + // test IsErrDuplicateTopic with a different error type. + dummyErr := fmt.Errorf("dummy error") + assert.False(t, IsErrDuplicateTopic(dummyErr), "IsErrDuplicateTopic should return false for non-ErrDuplicateTopic error") +} diff --git a/network/p2p/inspector/validation/inspect_message_request.go b/network/p2p/inspector/validation/inspect_message_request.go new file mode 100644 index 00000000000..bbd68878428 --- /dev/null +++ b/network/p2p/inspector/validation/inspect_message_request.go @@ -0,0 +1,31 @@ +package validation + +import ( + "fmt" + + pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/network/p2p/inspector/internal" + "github.com/onflow/flow-go/network/p2p/p2pconf" +) + +// InspectMsgRequest represents a short digest of an RPC control message. It is used for further message inspection by component workers. +type InspectMsgRequest struct { + // Nonce adds random value so that when msg req is stored on hero store a unique ID can be created from the struct fields. + Nonce []byte + // Peer sender of the message. + Peer peer.ID + // CtrlMsg the control message that will be inspected. + ctrlMsg *pubsub_pb.ControlMessage + validationConfig *p2pconf.CtrlMsgValidationConfig +} + +// NewInspectMsgRequest returns a new *InspectMsgRequest. +func NewInspectMsgRequest(from peer.ID, validationConfig *p2pconf.CtrlMsgValidationConfig, ctrlMsg *pubsub_pb.ControlMessage) (*InspectMsgRequest, error) { + nonce, err := internal.Nonce() + if err != nil { + return nil, fmt.Errorf("failed to get inspect message request nonce: %w", err) + } + return &InspectMsgRequest{Nonce: nonce, Peer: from, validationConfig: validationConfig, ctrlMsg: ctrlMsg}, nil +} diff --git a/network/p2p/inspector/validation/utils.go b/network/p2p/inspector/validation/utils.go new file mode 100644 index 00000000000..b15d74548be --- /dev/null +++ b/network/p2p/inspector/validation/utils.go @@ -0,0 +1,14 @@ +package validation + +import "github.com/onflow/flow-go/network/channels" + +type duplicateTopicTracker map[channels.Topic]struct{} + +func (d duplicateTopicTracker) set(topic channels.Topic) { + d[topic] = struct{}{} +} + +func (d duplicateTopicTracker) isDuplicate(topic channels.Topic) bool { + _, ok := d[topic] + return ok +} diff --git a/network/p2p/inspector/validation/validation_inspector_config.go b/network/p2p/inspector/validation/validation_inspector_config.go new file mode 100644 index 00000000000..ccad4018c04 --- /dev/null +++ b/network/p2p/inspector/validation/validation_inspector_config.go @@ -0,0 +1,14 @@ +package validation + +const ( + // DefaultNumberOfWorkers default number of workers for the inspector component. + DefaultNumberOfWorkers = 5 + // DefaultControlMsgValidationInspectorQueueCacheSize is the default size of the inspect message queue. + DefaultControlMsgValidationInspectorQueueCacheSize = 100 + // DefaultClusterPrefixedControlMsgsReceivedCacheSize is the default size of the cluster prefixed topics received record cache. + DefaultClusterPrefixedControlMsgsReceivedCacheSize = 150 + // DefaultClusterPrefixedControlMsgsReceivedCacheDecay the default cache decay value for cluster prefixed topics received cached counters. + DefaultClusterPrefixedControlMsgsReceivedCacheDecay = 0.99 + // rpcInspectorComponentName the rpc inspector component name. + rpcInspectorComponentName = "gossipsub_rpc_validation_inspector" +) diff --git a/network/p2p/libp2pNode.go b/network/p2p/libp2pNode.go index 1a7a87bd03d..061e45a43ff 100644 --- a/network/p2p/libp2pNode.go +++ b/network/p2p/libp2pNode.go @@ -10,9 +10,11 @@ import ( "github.com/libp2p/go-libp2p/core/protocol" "github.com/libp2p/go-libp2p/core/routing" + "github.com/onflow/flow-go/engine/collection" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p/unicast/protocols" ) @@ -31,6 +33,17 @@ type LibP2PNode interface { PeerConnections // PeerScore exposes the peer score API. PeerScore + // DisallowListNotificationConsumer exposes the disallow list notification consumer API for the node so that + // it will be notified when a new disallow list update is distributed. + DisallowListNotificationConsumer + // CollectionClusterChangesConsumer is the interface for consuming the events of changes in the collection cluster. + // This is used to notify the node of changes in the collection cluster. + // LibP2PNode implements this interface and consumes the events to be notified of changes in the clustering channels. + // The clustering channels are used by the collection nodes of a cluster to communicate with each other. + // As the cluster (and hence their cluster channels) of collection nodes changes over time (per epoch) the node needs to be notified of these changes. + CollectionClusterChangesConsumer + // DisallowListOracle exposes the disallow list oracle API for external consumers to query about the disallow list. + DisallowListOracle // Start the libp2p node. Start(ctx irrecoverable.SignalerContext) // Stop terminates the libp2p node. @@ -87,16 +100,21 @@ type Subscriptions interface { SetUnicastManager(uniMgr UnicastManager) } +// CollectionClusterChangesConsumer is the interface for consuming the events of changes in the collection cluster. +// This is used to notify the node of changes in the collection cluster. +// LibP2PNode implements this interface and consumes the events to be notified of changes in the clustering channels. +// The clustering channels are used by the collection nodes of a cluster to communicate with each other. +// As the cluster (and hence their cluster channels) of collection nodes changes over time (per epoch) the node needs to be notified of these changes. +type CollectionClusterChangesConsumer interface { + collection.ClusterEvents +} + // PeerScore is the interface for the peer score module. It is used to expose the peer score to other // components of the node. It is also used to set the peer score exposer implementation. type PeerScore interface { - // SetPeerScoreExposer sets the node's peer score exposer implementation. - // SetPeerScoreExposer may be called at most once. It is an irrecoverable error to call this - // method if the node's peer score exposer has already been set. - SetPeerScoreExposer(e PeerScoreExposer) // PeerScoreExposer returns the node's peer score exposer implementation. // If the node's peer score exposer has not been set, the second return value will be false. - PeerScoreExposer() (PeerScoreExposer, bool) + PeerScoreExposer() PeerScoreExposer } // PeerConnections subset of funcs related to underlying libp2p host connections. @@ -109,3 +127,38 @@ type PeerConnections interface { // to the peer is not empty. This indicates a bug within libp2p. IsConnected(peerID peer.ID) (bool, error) } + +// DisallowListNotificationConsumer is an interface for consuming disallow/allow list update notifications. +type DisallowListNotificationConsumer interface { + // OnDisallowListNotification is called when a new disallow list update notification is distributed. + // Any error on consuming event must handle internally. + // The implementation must be concurrency safe. + // Args: + // id: peer ID of the peer being disallow-listed. + // cause: cause of the peer being disallow-listed (only this cause is added to the peer's disallow-listed causes). + // Returns: + // none + OnDisallowListNotification(id peer.ID, cause network.DisallowListedCause) + + // OnAllowListNotification is called when a new allow list update notification is distributed. + // Any error on consuming event must handle internally. + // The implementation must be concurrency safe. + // Args: + // id: peer ID of the peer being allow-listed. + // cause: cause of the peer being allow-listed (only this cause is removed from the peer's disallow-listed causes). + // Returns: + // none + OnAllowListNotification(id peer.ID, cause network.DisallowListedCause) +} + +// DisallowListOracle is an interface for querying disallow-listed peers. +type DisallowListOracle interface { + // IsDisallowListed determines whether the given peer is disallow-listed for any reason. + // Args: + // - peerID: the peer to check. + // Returns: + // - []network.DisallowListedCause: the list of causes for which the given peer is disallow-listed. If the peer is not disallow-listed for any reason, + // a nil slice is returned. + // - bool: true if the peer is disallow-listed for any reason, false otherwise. + IsDisallowListed(peerId peer.ID) ([]network.DisallowListedCause, bool) +} diff --git a/network/p2p/message/types.go b/network/p2p/message/types.go new file mode 100644 index 00000000000..087cbd21455 --- /dev/null +++ b/network/p2p/message/types.go @@ -0,0 +1,20 @@ +package p2pmsg + +// ControlMessageType is the type of control message, as defined in the libp2p pubsub spec. +type ControlMessageType string + +func (c ControlMessageType) String() string { + return string(c) +} + +const ( + CtrlMsgIHave ControlMessageType = "IHAVE" + CtrlMsgIWant ControlMessageType = "IWANT" + CtrlMsgGraft ControlMessageType = "GRAFT" + CtrlMsgPrune ControlMessageType = "PRUNE" +) + +// ControlMessageTypes returns list of all libp2p control message types. +func ControlMessageTypes() []ControlMessageType { + return []ControlMessageType{CtrlMsgIHave, CtrlMsgIWant, CtrlMsgGraft, CtrlMsgPrune} +} diff --git a/network/p2p/middleware/middleware.go b/network/p2p/middleware/middleware.go index 58e15638943..c908e8d7f18 100644 --- a/network/p2p/middleware/middleware.go +++ b/network/p2p/middleware/middleware.go @@ -35,7 +35,6 @@ import ( "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/network/p2p/utils" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/validator" flowpubsub "github.com/onflow/flow-go/network/validator/pubsub" _ "github.com/onflow/flow-go/utils/binstat" @@ -65,8 +64,7 @@ const ( ) var ( - _ network.Middleware = (*Middleware)(nil) - _ p2p.DisallowListNotificationConsumer = (*Middleware)(nil) + _ network.Middleware = (*Middleware)(nil) // ErrUnicastMsgWithoutSub error is provided to the slashing violations consumer in the case where // the middleware receives a message via unicast but does not have a corresponding subscription for @@ -78,9 +76,11 @@ var ( // our neighbours on the peer-to-peer network. type Middleware struct { sync.Mutex + component.Component ctx context.Context log zerolog.Logger ov network.Overlay + // TODO: using a waitgroup here doesn't actually guarantee that we'll wait for all // goroutines to exit, because new goroutines could be started after we've already // returned from wg.Wait(). We need to solve this the right way using ComponentManager @@ -88,7 +88,6 @@ type Middleware struct { wg sync.WaitGroup libP2PNode p2p.LibP2PNode preferredUnicasts []protocols.ProtocolName - me flow.Identifier bitswapMetrics module.BitswapMetrics rootBlockID flow.Identifier validators []network.MessageValidator @@ -97,21 +96,20 @@ type Middleware struct { idTranslator p2p.IDTranslator previousProtocolStatePeers []peer.AddrInfo codec network.Codec - slashingViolationsConsumer slashing.ViolationsConsumer + slashingViolationsConsumer network.ViolationsConsumer unicastRateLimiters *ratelimit.RateLimiters authorizedSenderValidator *validator.AuthorizedSenderValidator - component.Component } -type MiddlewareOption func(*Middleware) +type OptionFn func(*Middleware) -func WithMessageValidators(validators ...network.MessageValidator) MiddlewareOption { +func WithMessageValidators(validators ...network.MessageValidator) OptionFn { return func(mw *Middleware) { mw.validators = validators } } -func WithPreferredUnicastProtocols(unicasts []protocols.ProtocolName) MiddlewareOption { +func WithPreferredUnicastProtocols(unicasts []protocols.ProtocolName) OptionFn { return func(mw *Middleware) { mw.preferredUnicasts = unicasts } @@ -119,19 +117,38 @@ func WithPreferredUnicastProtocols(unicasts []protocols.ProtocolName) Middleware // WithPeerManagerFilters sets a list of p2p.PeerFilter funcs that are used to // filter out peers provided by the peer manager PeersProvider. -func WithPeerManagerFilters(peerManagerFilters []p2p.PeerFilter) MiddlewareOption { +func WithPeerManagerFilters(peerManagerFilters []p2p.PeerFilter) OptionFn { return func(mw *Middleware) { mw.peerManagerFilters = peerManagerFilters } } // WithUnicastRateLimiters sets the unicast rate limiters. -func WithUnicastRateLimiters(rateLimiters *ratelimit.RateLimiters) MiddlewareOption { +func WithUnicastRateLimiters(rateLimiters *ratelimit.RateLimiters) OptionFn { return func(mw *Middleware) { mw.unicastRateLimiters = rateLimiters } } +// Config is the configuration for the middleware. +type Config struct { + Logger zerolog.Logger + Libp2pNode p2p.LibP2PNode + FlowId flow.Identifier // This node's Flow ID + BitSwapMetrics module.BitswapMetrics + RootBlockID flow.Identifier + UnicastMessageTimeout time.Duration + IdTranslator p2p.IDTranslator + Codec network.Codec +} + +// Validate validates the configuration, and sets default values for any missing fields. +func (cfg *Config) Validate() { + if cfg.UnicastMessageTimeout <= 0 { + cfg.UnicastMessageTimeout = DefaultUnicastTimeout + } +} + // NewMiddleware creates a new middleware instance // libP2PNodeFactory is the factory used to create a LibP2PNode // flowID is this node's Flow ID @@ -141,35 +158,20 @@ func WithUnicastRateLimiters(rateLimiters *ratelimit.RateLimiters) MiddlewareOpt // validators are the set of the different message validators that each inbound messages is passed through // During normal operations any error returned by Middleware.start is considered to be catastrophic // and will be thrown by the irrecoverable.SignalerContext causing the node to crash. -func NewMiddleware( - log zerolog.Logger, - libP2PNode p2p.LibP2PNode, - flowID flow.Identifier, - bitswapMet module.BitswapMetrics, - rootBlockID flow.Identifier, - unicastMessageTimeout time.Duration, - idTranslator p2p.IDTranslator, - codec network.Codec, - slashingViolationsConsumer slashing.ViolationsConsumer, - opts ...MiddlewareOption) *Middleware { - - if unicastMessageTimeout <= 0 { - unicastMessageTimeout = DefaultUnicastTimeout - } +func NewMiddleware(cfg *Config, opts ...OptionFn) *Middleware { + cfg.Validate() // create the node entity and inject dependencies & config mw := &Middleware{ - log: log, - me: flowID, - libP2PNode: libP2PNode, - bitswapMetrics: bitswapMet, - rootBlockID: rootBlockID, - validators: DefaultValidators(log, flowID), - unicastMessageTimeout: unicastMessageTimeout, - idTranslator: idTranslator, - codec: codec, - slashingViolationsConsumer: slashingViolationsConsumer, - unicastRateLimiters: ratelimit.NoopRateLimiters(), + log: cfg.Logger, + libP2PNode: cfg.Libp2pNode, + bitswapMetrics: cfg.BitSwapMetrics, + rootBlockID: cfg.RootBlockID, + validators: DefaultValidators(cfg.Logger, cfg.FlowId), + unicastMessageTimeout: cfg.UnicastMessageTimeout, + idTranslator: cfg.IdTranslator, + codec: cfg.Codec, + unicastRateLimiters: ratelimit.NoopRateLimiters(), } for _, opt := range opts { @@ -188,13 +190,24 @@ func NewMiddleware( }) } builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - // TODO: refactor to avoid storing ctx altogether mw.ctx = ctx + if mw.ov == nil { + ctx.Throw(fmt.Errorf("overlay has not been set")) + } - if err := mw.start(ctx); err != nil { - ctx.Throw(err) + mw.authorizedSenderValidator = validator.NewAuthorizedSenderValidator( + mw.log, + mw.slashingViolationsConsumer, + mw.ov.Identity) + + err := mw.libP2PNode.WithDefaultUnicastProtocol(mw.handleIncomingStream, mw.preferredUnicasts) + if err != nil { + ctx.Throw(fmt.Errorf("could not register preferred unicast protocols on libp2p node: %w", err)) } + mw.UpdateNodeAddresses() + mw.libP2PNode.WithPeersProvider(mw.authorizedPeers) + ready() <-ctx.Done() @@ -202,7 +215,6 @@ func NewMiddleware( // wait for the readConnection and readSubscription routines to stop mw.wg.Wait() - mw.log.Info().Str("component", "middleware").Msg("stopped subroutines") }) @@ -253,22 +265,6 @@ func (m *Middleware) peerIDs(flowIDs flow.IdentifierList) peer.IDSlice { return result } -// Me returns the flow identifier of this middleware -func (m *Middleware) Me() flow.Identifier { - return m.me -} - -// GetIPPort returns the ip address and port number associated with the middleware -// All errors returned from this function can be considered benign. -func (m *Middleware) GetIPPort() (string, string, error) { - ipOrHostname, port, err := m.libP2PNode.GetIPPort() - if err != nil { - return "", "", fmt.Errorf("failed to get ip and port from libP2P node: %w", err) - } - - return ipOrHostname, port, nil -} - func (m *Middleware) UpdateNodeAddresses() { m.log.Info().Msg("Updating protocol state node addresses") @@ -298,33 +294,23 @@ func (m *Middleware) SetOverlay(ov network.Overlay) { m.ov = ov } -// start will start the middleware. -// No errors are expected during normal operation. -func (m *Middleware) start(ctx context.Context) error { - if m.ov == nil { - return fmt.Errorf("could not start middleware: overlay must be configured by calling SetOverlay before middleware can be started") - } - - m.authorizedSenderValidator = validator.NewAuthorizedSenderValidator(m.log, m.slashingViolationsConsumer, m.ov.Identity) - - err := m.libP2PNode.WithDefaultUnicastProtocol(m.handleIncomingStream, m.preferredUnicasts) - if err != nil { - return fmt.Errorf("could not register preferred unicast protocols on libp2p node: %w", err) - } - - m.UpdateNodeAddresses() - - m.libP2PNode.WithPeersProvider(m.topologyPeers) - - return nil +// SetSlashingViolationsConsumer sets the slashing violations consumer. +func (m *Middleware) SetSlashingViolationsConsumer(consumer network.ViolationsConsumer) { + m.slashingViolationsConsumer = consumer } -// topologyPeers callback used by the peer manager to get the list of peer ID's -// which this node should be directly connected to as peers. The peer ID list -// returned will be filtered through any configured m.peerManagerFilters. If the -// underlying libp2p node has a peer manager configured this func will be used as the -// peers provider. -func (m *Middleware) topologyPeers() peer.IDSlice { +// authorizedPeers is a peer manager callback used by the underlying libp2p node that updates who can connect to this node (as +// well as who this node can connect to). +// and who is not allowed to connect to this node. This function is called by the peer manager and connection gater components +// of libp2p. +// +// Args: +// none +// Returns: +// - peer.IDSlice: a list of peer IDs that are allowed to connect to this node (and that this node can connect to). Any peer +// not in this list is assumed to be disconnected from this node (if connected) and not allowed to connect to this node. +// This is the guarantee that the underlying libp2p node implementation makes. +func (m *Middleware) authorizedPeers() peer.IDSlice { peerIDs := make([]peer.ID, 0) for _, id := range m.peerIDs(m.ov.Topology().NodeIDs()) { peerAllowed := true @@ -348,14 +334,15 @@ func (m *Middleware) topologyPeers() peer.IDSlice { return peerIDs } -// OnDisallowListNotification is called when a new disallow list update notification is distributed. -// It disconnects from all peers in the disallow list. -func (m *Middleware) OnDisallowListNotification(notification *p2p.DisallowListUpdateNotification) { - for _, pid := range m.peerIDs(notification.DisallowList) { - err := m.libP2PNode.RemovePeer(pid) - if err != nil { - m.log.Error().Err(err).Str("peer_id", pid.String()).Msg("failed to disconnect from blocklisted peer") - } +func (m *Middleware) OnDisallowListNotification(notification *network.DisallowListingUpdate) { + for _, pid := range m.peerIDs(notification.FlowIds) { + m.libP2PNode.OnDisallowListNotification(pid, notification.Cause) + } +} + +func (m *Middleware) OnAllowListNotification(notification *network.AllowListingUpdate) { + for _, pid := range m.peerIDs(notification.FlowIds) { + m.libP2PNode.OnAllowListNotification(pid, notification.Cause) } } @@ -526,7 +513,7 @@ func (m *Middleware) handleIncomingStream(s libp2pnetwork.Stream) { // ignore messages if node does not have subscription to topic if !m.libP2PNode.HasSubscription(topic) { - violation := &slashing.Violation{ + violation := &network.Violation{ Identity: nil, PeerID: remotePeer.String(), Channel: channel, Protocol: message.ProtocolTypeUnicast, } @@ -657,9 +644,9 @@ func (m *Middleware) processUnicastStreamMessage(remotePeer peer.ID, msg *messag // TODO: once we've implemented per topic message size limits per the TODO above, // we can remove this check - maxSize, err := unicastMaxMsgSizeByCode(msg.Payload) + maxSize, err := UnicastMaxMsgSizeByCode(msg.Payload) if err != nil { - m.slashingViolationsConsumer.OnUnknownMsgTypeError(&slashing.Violation{ + m.slashingViolationsConsumer.OnUnknownMsgTypeError(&network.Violation{ Identity: nil, PeerID: remotePeer.String(), MsgType: "", Channel: channel, Protocol: message.ProtocolTypeUnicast, Err: err, }) return @@ -713,14 +700,14 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe switch { case codec.IsErrUnknownMsgCode(err): // slash peer if message contains unknown message code byte - violation := &slashing.Violation{ + violation := &network.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) return case codec.IsErrMsgUnmarshal(err) || codec.IsErrInvalidEncoding(err): // slash if peer sent a message that could not be marshalled into the message type denoted by the message code byte - violation := &slashing.Violation{ + violation := &network.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } m.slashingViolationsConsumer.OnInvalidMsgError(violation) @@ -730,7 +717,7 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe // don't crash as a result of external inputs since that creates a DoS vector // collect slashing data because this could potentially lead to slashing err = fmt.Errorf("unexpected error during message validation: %w", err) - violation := &slashing.Violation{ + violation := &network.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } m.slashingViolationsConsumer.OnUnexpectedError(violation) @@ -752,7 +739,6 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe // processMessage processes a message and eventually passes it to the overlay func (m *Middleware) processMessage(scope *network.IncomingMessageScope) { - logger := m.log.With(). Str("channel", scope.Channel().String()). Str("type", scope.Protocol().String()). @@ -832,15 +818,15 @@ func (m *Middleware) IsConnected(nodeID flow.Identifier) (bool, error) { // unicastMaxMsgSize returns the max permissible size for a unicast message func unicastMaxMsgSize(messageType string) int { switch messageType { - case "messages.ChunkDataResponse": + case "*messages.ChunkDataResponse": return LargeMsgMaxUnicastMsgSize default: return DefaultMaxUnicastMsgSize } } -// unicastMaxMsgSizeByCode returns the max permissible size for a unicast message code -func unicastMaxMsgSizeByCode(payload []byte) (int, error) { +// UnicastMaxMsgSizeByCode returns the max permissible size for a unicast message code +func UnicastMaxMsgSizeByCode(payload []byte) (int, error) { msgCode, err := codec.MessageCodeFromPayload(payload) if err != nil { return 0, err diff --git a/network/p2p/middleware/middleware_test.go b/network/p2p/middleware/middleware_test.go new file mode 100644 index 00000000000..9b9cc1dbc0e --- /dev/null +++ b/network/p2p/middleware/middleware_test.go @@ -0,0 +1,36 @@ +package middleware_test + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/message" + "github.com/onflow/flow-go/network/p2p/middleware" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestChunkDataPackMaxMessageSize tests that the max message size for a chunk data pack response is set to the large message size. +func TestChunkDataPackMaxMessageSize(t *testing.T) { + // creates an outgoing chunk data pack response message (imitating an EN is sending a chunk data pack response to VN). + msg, err := network.NewOutgoingScope( + flow.IdentifierList{unittest.IdentifierFixture()}, + channels.ProvideChunks, + &messages.ChunkDataResponse{ + ChunkDataPack: *unittest.ChunkDataPackFixture(unittest.IdentifierFixture()), + Nonce: rand.Uint64(), + }, + unittest.NetworkCodec().Encode, + message.ProtocolTypeUnicast) + require.NoError(t, err) + + // get the max message size for the message + size, err := middleware.UnicastMaxMsgSizeByCode(msg.Proto().Payload) + require.NoError(t, err) + require.Equal(t, middleware.LargeMsgMaxUnicastMsgSize, size) +} diff --git a/network/p2p/mock/adjust_function.go b/network/p2p/mock/adjust_function.go new file mode 100644 index 00000000000..675dddb2efd --- /dev/null +++ b/network/p2p/mock/adjust_function.go @@ -0,0 +1,42 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + p2p "github.com/onflow/flow-go/network/p2p" + mock "github.com/stretchr/testify/mock" +) + +// AdjustFunction is an autogenerated mock type for the AdjustFunction type +type AdjustFunction struct { + mock.Mock +} + +// Execute provides a mock function with given fields: record +func (_m *AdjustFunction) Execute(record p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord { + ret := _m.Called(record) + + var r0 p2p.GossipSubSpamRecord + if rf, ok := ret.Get(0).(func(p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord); ok { + r0 = rf(record) + } else { + r0 = ret.Get(0).(p2p.GossipSubSpamRecord) + } + + return r0 +} + +type mockConstructorTestingTNewAdjustFunction interface { + mock.TestingT + Cleanup(func()) +} + +// NewAdjustFunction creates a new instance of AdjustFunction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewAdjustFunction(t mockConstructorTestingTNewAdjustFunction) *AdjustFunction { + mock := &AdjustFunction{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/collection_cluster_changes_consumer.go b/network/p2p/mock/collection_cluster_changes_consumer.go new file mode 100644 index 00000000000..cb76577f06f --- /dev/null +++ b/network/p2p/mock/collection_cluster_changes_consumer.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// CollectionClusterChangesConsumer is an autogenerated mock type for the CollectionClusterChangesConsumer type +type CollectionClusterChangesConsumer struct { + mock.Mock +} + +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *CollectionClusterChangesConsumer) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewCollectionClusterChangesConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewCollectionClusterChangesConsumer creates a new instance of CollectionClusterChangesConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCollectionClusterChangesConsumer(t mockConstructorTestingTNewCollectionClusterChangesConsumer) *CollectionClusterChangesConsumer { + mock := &CollectionClusterChangesConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/connection_gater.go b/network/p2p/mock/connection_gater.go index d5943e8efa9..c316e3f31d0 100644 --- a/network/p2p/mock/connection_gater.go +++ b/network/p2p/mock/connection_gater.go @@ -10,6 +10,8 @@ import ( network "github.com/libp2p/go-libp2p/core/network" + p2p "github.com/onflow/flow-go/network/p2p" + peer "github.com/libp2p/go-libp2p/core/peer" ) @@ -98,6 +100,11 @@ func (_m *ConnectionGater) InterceptUpgraded(_a0 network.Conn) (bool, control.Di return r0, r1 } +// SetDisallowListOracle provides a mock function with given fields: oracle +func (_m *ConnectionGater) SetDisallowListOracle(oracle p2p.DisallowListOracle) { + _m.Called(oracle) +} + type mockConstructorTestingTNewConnectionGater interface { mock.TestingT Cleanup(func()) diff --git a/network/p2p/mock/connector.go b/network/p2p/mock/connector.go index d1e6733cbab..2c8a49c9070 100644 --- a/network/p2p/mock/connector.go +++ b/network/p2p/mock/connector.go @@ -15,9 +15,9 @@ type Connector struct { mock.Mock } -// UpdatePeers provides a mock function with given fields: ctx, peerIDs -func (_m *Connector) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) { - _m.Called(ctx, peerIDs) +// Connect provides a mock function with given fields: ctx, peerChan +func (_m *Connector) Connect(ctx context.Context, peerChan <-chan peer.AddrInfo) { + _m.Called(ctx, peerChan) } type mockConstructorTestingTNewConnector interface { diff --git a/network/p2p/mock/connector_factory.go b/network/p2p/mock/connector_factory.go new file mode 100644 index 00000000000..a22788969f8 --- /dev/null +++ b/network/p2p/mock/connector_factory.go @@ -0,0 +1,56 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + host "github.com/libp2p/go-libp2p/core/host" + mock "github.com/stretchr/testify/mock" + + p2p "github.com/onflow/flow-go/network/p2p" +) + +// ConnectorFactory is an autogenerated mock type for the ConnectorFactory type +type ConnectorFactory struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0 +func (_m *ConnectorFactory) Execute(_a0 host.Host) (p2p.Connector, error) { + ret := _m.Called(_a0) + + var r0 p2p.Connector + var r1 error + if rf, ok := ret.Get(0).(func(host.Host) (p2p.Connector, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(host.Host) p2p.Connector); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(p2p.Connector) + } + } + + if rf, ok := ret.Get(1).(func(host.Host) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewConnectorFactory interface { + mock.TestingT + Cleanup(func()) +} + +// NewConnectorFactory creates a new instance of ConnectorFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnectorFactory(t mockConstructorTestingTNewConnectorFactory) *ConnectorFactory { + mock := &ConnectorFactory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/connector_host.go b/network/p2p/mock/connector_host.go new file mode 100644 index 00000000000..5cb468884e8 --- /dev/null +++ b/network/p2p/mock/connector_host.go @@ -0,0 +1,116 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + network "github.com/libp2p/go-libp2p/core/network" + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// ConnectorHost is an autogenerated mock type for the ConnectorHost type +type ConnectorHost struct { + mock.Mock +} + +// ClosePeer provides a mock function with given fields: peerId +func (_m *ConnectorHost) ClosePeer(peerId peer.ID) error { + ret := _m.Called(peerId) + + var r0 error + if rf, ok := ret.Get(0).(func(peer.ID) error); ok { + r0 = rf(peerId) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Connections provides a mock function with given fields: +func (_m *ConnectorHost) Connections() []network.Conn { + ret := _m.Called() + + var r0 []network.Conn + if rf, ok := ret.Get(0).(func() []network.Conn); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]network.Conn) + } + } + + return r0 +} + +// ID provides a mock function with given fields: +func (_m *ConnectorHost) ID() peer.ID { + ret := _m.Called() + + var r0 peer.ID + if rf, ok := ret.Get(0).(func() peer.ID); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(peer.ID) + } + + return r0 +} + +// IsConnectedTo provides a mock function with given fields: peerId +func (_m *ConnectorHost) IsConnectedTo(peerId peer.ID) bool { + ret := _m.Called(peerId) + + var r0 bool + if rf, ok := ret.Get(0).(func(peer.ID) bool); ok { + r0 = rf(peerId) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// IsProtected provides a mock function with given fields: peerId +func (_m *ConnectorHost) IsProtected(peerId peer.ID) bool { + ret := _m.Called(peerId) + + var r0 bool + if rf, ok := ret.Get(0).(func(peer.ID) bool); ok { + r0 = rf(peerId) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// PeerInfo provides a mock function with given fields: peerId +func (_m *ConnectorHost) PeerInfo(peerId peer.ID) peer.AddrInfo { + ret := _m.Called(peerId) + + var r0 peer.AddrInfo + if rf, ok := ret.Get(0).(func(peer.ID) peer.AddrInfo); ok { + r0 = rf(peerId) + } else { + r0 = ret.Get(0).(peer.AddrInfo) + } + + return r0 +} + +type mockConstructorTestingTNewConnectorHost interface { + mock.TestingT + Cleanup(func()) +} + +// NewConnectorHost creates a new instance of ConnectorHost. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnectorHost(t mockConstructorTestingTNewConnectorHost) *ConnectorHost { + mock := &ConnectorHost{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/create_node_func.go b/network/p2p/mock/create_node_func.go index 3169c71cb1e..1a57772cbeb 100644 --- a/network/p2p/mock/create_node_func.go +++ b/network/p2p/mock/create_node_func.go @@ -16,13 +16,13 @@ type CreateNodeFunc struct { mock.Mock } -// Execute provides a mock function with given fields: _a0, _a1, _a2, _a3 -func (_m *CreateNodeFunc) Execute(_a0 zerolog.Logger, _a1 host.Host, _a2 p2p.ProtocolPeerCache, _a3 p2p.PeerManager) p2p.LibP2PNode { - ret := _m.Called(_a0, _a1, _a2, _a3) +// Execute provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 +func (_m *CreateNodeFunc) Execute(_a0 zerolog.Logger, _a1 host.Host, _a2 p2p.ProtocolPeerCache, _a3 p2p.PeerManager, _a4 *p2p.DisallowListCacheConfig) p2p.LibP2PNode { + ret := _m.Called(_a0, _a1, _a2, _a3, _a4) var r0 p2p.LibP2PNode - if rf, ok := ret.Get(0).(func(zerolog.Logger, host.Host, p2p.ProtocolPeerCache, p2p.PeerManager) p2p.LibP2PNode); ok { - r0 = rf(_a0, _a1, _a2, _a3) + if rf, ok := ret.Get(0).(func(zerolog.Logger, host.Host, p2p.ProtocolPeerCache, p2p.PeerManager, *p2p.DisallowListCacheConfig) p2p.LibP2PNode); ok { + r0 = rf(_a0, _a1, _a2, _a3, _a4) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(p2p.LibP2PNode) diff --git a/network/p2p/mock/disallow_list_cache.go b/network/p2p/mock/disallow_list_cache.go new file mode 100644 index 00000000000..54d7fcf0d3c --- /dev/null +++ b/network/p2p/mock/disallow_list_cache.go @@ -0,0 +1,98 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + network "github.com/onflow/flow-go/network" + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// DisallowListCache is an autogenerated mock type for the DisallowListCache type +type DisallowListCache struct { + mock.Mock +} + +// AllowFor provides a mock function with given fields: peerID, cause +func (_m *DisallowListCache) AllowFor(peerID peer.ID, cause network.DisallowListedCause) []network.DisallowListedCause { + ret := _m.Called(peerID, cause) + + var r0 []network.DisallowListedCause + if rf, ok := ret.Get(0).(func(peer.ID, network.DisallowListedCause) []network.DisallowListedCause); ok { + r0 = rf(peerID, cause) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]network.DisallowListedCause) + } + } + + return r0 +} + +// DisallowFor provides a mock function with given fields: peerID, cause +func (_m *DisallowListCache) DisallowFor(peerID peer.ID, cause network.DisallowListedCause) ([]network.DisallowListedCause, error) { + ret := _m.Called(peerID, cause) + + var r0 []network.DisallowListedCause + var r1 error + if rf, ok := ret.Get(0).(func(peer.ID, network.DisallowListedCause) ([]network.DisallowListedCause, error)); ok { + return rf(peerID, cause) + } + if rf, ok := ret.Get(0).(func(peer.ID, network.DisallowListedCause) []network.DisallowListedCause); ok { + r0 = rf(peerID, cause) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]network.DisallowListedCause) + } + } + + if rf, ok := ret.Get(1).(func(peer.ID, network.DisallowListedCause) error); ok { + r1 = rf(peerID, cause) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IsDisallowListed provides a mock function with given fields: peerID +func (_m *DisallowListCache) IsDisallowListed(peerID peer.ID) ([]network.DisallowListedCause, bool) { + ret := _m.Called(peerID) + + var r0 []network.DisallowListedCause + var r1 bool + if rf, ok := ret.Get(0).(func(peer.ID) ([]network.DisallowListedCause, bool)); ok { + return rf(peerID) + } + if rf, ok := ret.Get(0).(func(peer.ID) []network.DisallowListedCause); ok { + r0 = rf(peerID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]network.DisallowListedCause) + } + } + + if rf, ok := ret.Get(1).(func(peer.ID) bool); ok { + r1 = rf(peerID) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +type mockConstructorTestingTNewDisallowListCache interface { + mock.TestingT + Cleanup(func()) +} + +// NewDisallowListCache creates a new instance of DisallowListCache. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDisallowListCache(t mockConstructorTestingTNewDisallowListCache) *DisallowListCache { + mock := &DisallowListCache{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/disallow_list_consumer.go b/network/p2p/mock/disallow_list_consumer.go deleted file mode 100644 index 2800a5aa909..00000000000 --- a/network/p2p/mock/disallow_list_consumer.go +++ /dev/null @@ -1,33 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mockp2p - -import ( - flow "github.com/onflow/flow-go/model/flow" - mock "github.com/stretchr/testify/mock" -) - -// DisallowListConsumer is an autogenerated mock type for the DisallowListConsumer type -type DisallowListConsumer struct { - mock.Mock -} - -// OnNodeDisallowListUpdate provides a mock function with given fields: list -func (_m *DisallowListConsumer) OnNodeDisallowListUpdate(list flow.IdentifierList) { - _m.Called(list) -} - -type mockConstructorTestingTNewDisallowListConsumer interface { - mock.TestingT - Cleanup(func()) -} - -// NewDisallowListConsumer creates a new instance of DisallowListConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewDisallowListConsumer(t mockConstructorTestingTNewDisallowListConsumer) *DisallowListConsumer { - mock := &DisallowListConsumer{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/network/p2p/mock/disallow_list_notification_consumer.go b/network/p2p/mock/disallow_list_notification_consumer.go index 7df8437ddcf..0d30cddea03 100644 --- a/network/p2p/mock/disallow_list_notification_consumer.go +++ b/network/p2p/mock/disallow_list_notification_consumer.go @@ -3,8 +3,10 @@ package mockp2p import ( - p2p "github.com/onflow/flow-go/network/p2p" + network "github.com/onflow/flow-go/network" mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" ) // DisallowListNotificationConsumer is an autogenerated mock type for the DisallowListNotificationConsumer type @@ -12,9 +14,14 @@ type DisallowListNotificationConsumer struct { mock.Mock } -// OnDisallowListNotification provides a mock function with given fields: _a0 -func (_m *DisallowListNotificationConsumer) OnDisallowListNotification(_a0 *p2p.DisallowListUpdateNotification) { - _m.Called(_a0) +// OnAllowListNotification provides a mock function with given fields: id, cause +func (_m *DisallowListNotificationConsumer) OnAllowListNotification(id peer.ID, cause network.DisallowListedCause) { + _m.Called(id, cause) +} + +// OnDisallowListNotification provides a mock function with given fields: id, cause +func (_m *DisallowListNotificationConsumer) OnDisallowListNotification(id peer.ID, cause network.DisallowListedCause) { + _m.Called(id, cause) } type mockConstructorTestingTNewDisallowListNotificationConsumer interface { diff --git a/network/p2p/mock/disallow_list_notification_distributor.go b/network/p2p/mock/disallow_list_notification_distributor.go deleted file mode 100644 index 82419cb87e1..00000000000 --- a/network/p2p/mock/disallow_list_notification_distributor.go +++ /dev/null @@ -1,88 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mockp2p - -import ( - flow "github.com/onflow/flow-go/model/flow" - irrecoverable "github.com/onflow/flow-go/module/irrecoverable" - - mock "github.com/stretchr/testify/mock" - - p2p "github.com/onflow/flow-go/network/p2p" -) - -// DisallowListNotificationDistributor is an autogenerated mock type for the DisallowListNotificationDistributor type -type DisallowListNotificationDistributor struct { - mock.Mock -} - -// AddConsumer provides a mock function with given fields: _a0 -func (_m *DisallowListNotificationDistributor) AddConsumer(_a0 p2p.DisallowListNotificationConsumer) { - _m.Called(_a0) -} - -// DistributeBlockListNotification provides a mock function with given fields: list -func (_m *DisallowListNotificationDistributor) DistributeBlockListNotification(list flow.IdentifierList) error { - ret := _m.Called(list) - - var r0 error - if rf, ok := ret.Get(0).(func(flow.IdentifierList) error); ok { - r0 = rf(list) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Done provides a mock function with given fields: -func (_m *DisallowListNotificationDistributor) Done() <-chan struct{} { - ret := _m.Called() - - var r0 <-chan struct{} - if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan struct{}) - } - } - - return r0 -} - -// Ready provides a mock function with given fields: -func (_m *DisallowListNotificationDistributor) Ready() <-chan struct{} { - ret := _m.Called() - - var r0 <-chan struct{} - if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan struct{}) - } - } - - return r0 -} - -// Start provides a mock function with given fields: _a0 -func (_m *DisallowListNotificationDistributor) Start(_a0 irrecoverable.SignalerContext) { - _m.Called(_a0) -} - -type mockConstructorTestingTNewDisallowListNotificationDistributor interface { - mock.TestingT - Cleanup(func()) -} - -// NewDisallowListNotificationDistributor creates a new instance of DisallowListNotificationDistributor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewDisallowListNotificationDistributor(t mockConstructorTestingTNewDisallowListNotificationDistributor) *DisallowListNotificationDistributor { - mock := &DisallowListNotificationDistributor{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/network/p2p/mock/disallow_list_oracle.go b/network/p2p/mock/disallow_list_oracle.go new file mode 100644 index 00000000000..8779bce7186 --- /dev/null +++ b/network/p2p/mock/disallow_list_oracle.go @@ -0,0 +1,56 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + network "github.com/onflow/flow-go/network" + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// DisallowListOracle is an autogenerated mock type for the DisallowListOracle type +type DisallowListOracle struct { + mock.Mock +} + +// IsDisallowListed provides a mock function with given fields: peerId +func (_m *DisallowListOracle) IsDisallowListed(peerId peer.ID) ([]network.DisallowListedCause, bool) { + ret := _m.Called(peerId) + + var r0 []network.DisallowListedCause + var r1 bool + if rf, ok := ret.Get(0).(func(peer.ID) ([]network.DisallowListedCause, bool)); ok { + return rf(peerId) + } + if rf, ok := ret.Get(0).(func(peer.ID) []network.DisallowListedCause); ok { + r0 = rf(peerId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]network.DisallowListedCause) + } + } + + if rf, ok := ret.Get(1).(func(peer.ID) bool); ok { + r1 = rf(peerId) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +type mockConstructorTestingTNewDisallowListOracle interface { + mock.TestingT + Cleanup(func()) +} + +// NewDisallowListOracle creates a new instance of DisallowListOracle. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDisallowListOracle(t mockConstructorTestingTNewDisallowListOracle) *DisallowListOracle { + mock := &DisallowListOracle{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/gossip_sub_builder.go b/network/p2p/mock/gossip_sub_builder.go index 33a910b4a70..08d82bd03c6 100644 --- a/network/p2p/mock/gossip_sub_builder.go +++ b/network/p2p/mock/gossip_sub_builder.go @@ -4,18 +4,12 @@ package mockp2p import ( host "github.com/libp2p/go-libp2p/core/host" - channels "github.com/onflow/flow-go/network/channels" - irrecoverable "github.com/onflow/flow-go/module/irrecoverable" mock "github.com/stretchr/testify/mock" - module "github.com/onflow/flow-go/module" - p2p "github.com/onflow/flow-go/network/p2p" - peer "github.com/libp2p/go-libp2p/core/peer" - pubsub "github.com/libp2p/go-libp2p-pubsub" routing "github.com/libp2p/go-libp2p/core/routing" @@ -29,13 +23,12 @@ type GossipSubBuilder struct { } // Build provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) Build(_a0 irrecoverable.SignalerContext) (p2p.PubSubAdapter, p2p.PeerScoreTracer, error) { +func (_m *GossipSubBuilder) Build(_a0 irrecoverable.SignalerContext) (p2p.PubSubAdapter, error) { ret := _m.Called(_a0) var r0 p2p.PubSubAdapter - var r1 p2p.PeerScoreTracer - var r2 error - if rf, ok := ret.Get(0).(func(irrecoverable.SignalerContext) (p2p.PubSubAdapter, p2p.PeerScoreTracer, error)); ok { + var r1 error + if rf, ok := ret.Get(0).(func(irrecoverable.SignalerContext) (p2p.PubSubAdapter, error)); ok { return rf(_a0) } if rf, ok := ret.Get(0).(func(irrecoverable.SignalerContext) p2p.PubSubAdapter); ok { @@ -46,25 +39,22 @@ func (_m *GossipSubBuilder) Build(_a0 irrecoverable.SignalerContext) (p2p.PubSub } } - if rf, ok := ret.Get(1).(func(irrecoverable.SignalerContext) p2p.PeerScoreTracer); ok { + if rf, ok := ret.Get(1).(func(irrecoverable.SignalerContext) error); ok { r1 = rf(_a0) } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(p2p.PeerScoreTracer) - } + r1 = ret.Error(1) } - if rf, ok := ret.Get(2).(func(irrecoverable.SignalerContext) error); ok { - r2 = rf(_a0) - } else { - r2 = ret.Error(2) - } + return r0, r1 +} - return r0, r1, r2 +// EnableGossipSubScoringWithOverride provides a mock function with given fields: _a0 +func (_m *GossipSubBuilder) EnableGossipSubScoringWithOverride(_a0 *p2p.PeerScoringConfigOverride) { + _m.Called(_a0) } -// SetAppSpecificScoreParams provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) SetAppSpecificScoreParams(_a0 func(peer.ID) float64) { +// OverrideDefaultRpcInspectorSuiteFactory provides a mock function with given fields: _a0 +func (_m *GossipSubBuilder) OverrideDefaultRpcInspectorSuiteFactory(_a0 p2p.GossipSubRpcInspectorSuiteFactoryFunc) { _m.Called(_a0) } @@ -78,22 +68,6 @@ func (_m *GossipSubBuilder) SetGossipSubFactory(_a0 p2p.GossipSubFactoryFunc) { _m.Called(_a0) } -// SetGossipSubPeerScoring provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) SetGossipSubPeerScoring(_a0 bool) { - _m.Called(_a0) -} - -// SetGossipSubRPCInspectors provides a mock function with given fields: inspectors -func (_m *GossipSubBuilder) SetGossipSubRPCInspectors(inspectors ...p2p.GossipSubRPCInspector) { - _va := make([]interface{}, len(inspectors)) - for _i := range inspectors { - _va[_i] = inspectors[_i] - } - var _ca []interface{} - _ca = append(_ca, _va...) - _m.Called(_ca...) -} - // SetGossipSubScoreTracerInterval provides a mock function with given fields: _a0 func (_m *GossipSubBuilder) SetGossipSubScoreTracerInterval(_a0 time.Duration) { _m.Called(_a0) @@ -109,11 +83,6 @@ func (_m *GossipSubBuilder) SetHost(_a0 host.Host) { _m.Called(_a0) } -// SetIDProvider provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) SetIDProvider(_a0 module.IdentityProvider) { - _m.Called(_a0) -} - // SetRoutingSystem provides a mock function with given fields: _a0 func (_m *GossipSubBuilder) SetRoutingSystem(_a0 routing.Routing) { _m.Called(_a0) @@ -124,11 +93,6 @@ func (_m *GossipSubBuilder) SetSubscriptionFilter(_a0 pubsub.SubscriptionFilter) _m.Called(_a0) } -// SetTopicScoreParams provides a mock function with given fields: topic, topicScoreParams -func (_m *GossipSubBuilder) SetTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) { - _m.Called(topic, topicScoreParams) -} - type mockConstructorTestingTNewGossipSubBuilder interface { mock.TestingT Cleanup(func()) diff --git a/network/p2p/mock/gossip_sub_factory_func.go b/network/p2p/mock/gossip_sub_factory_func.go index 06cd0346c8c..14aa9a7cec4 100644 --- a/network/p2p/mock/gossip_sub_factory_func.go +++ b/network/p2p/mock/gossip_sub_factory_func.go @@ -18,25 +18,25 @@ type GossipSubFactoryFunc struct { mock.Mock } -// Execute provides a mock function with given fields: _a0, _a1, _a2, _a3 -func (_m *GossipSubFactoryFunc) Execute(_a0 context.Context, _a1 zerolog.Logger, _a2 host.Host, _a3 p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error) { - ret := _m.Called(_a0, _a1, _a2, _a3) +// Execute provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 +func (_m *GossipSubFactoryFunc) Execute(_a0 context.Context, _a1 zerolog.Logger, _a2 host.Host, _a3 p2p.PubSubAdapterConfig, _a4 p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error) { + ret := _m.Called(_a0, _a1, _a2, _a3, _a4) var r0 p2p.PubSubAdapter var r1 error - if rf, ok := ret.Get(0).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error)); ok { - return rf(_a0, _a1, _a2, _a3) + if rf, ok := ret.Get(0).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig, p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error)); ok { + return rf(_a0, _a1, _a2, _a3, _a4) } - if rf, ok := ret.Get(0).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig) p2p.PubSubAdapter); ok { - r0 = rf(_a0, _a1, _a2, _a3) + if rf, ok := ret.Get(0).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig, p2p.CollectionClusterChangesConsumer) p2p.PubSubAdapter); ok { + r0 = rf(_a0, _a1, _a2, _a3, _a4) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(p2p.PubSubAdapter) } } - if rf, ok := ret.Get(1).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig) error); ok { - r1 = rf(_a0, _a1, _a2, _a3) + if rf, ok := ret.Get(1).(func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig, p2p.CollectionClusterChangesConsumer) error); ok { + r1 = rf(_a0, _a1, _a2, _a3, _a4) } else { r1 = ret.Error(1) } diff --git a/network/p2p/mock/gossip_sub_inspector_notif_distributor.go b/network/p2p/mock/gossip_sub_inspector_notif_distributor.go new file mode 100644 index 00000000000..b378c9fac2b --- /dev/null +++ b/network/p2p/mock/gossip_sub_inspector_notif_distributor.go @@ -0,0 +1,86 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + mock "github.com/stretchr/testify/mock" + + p2p "github.com/onflow/flow-go/network/p2p" +) + +// GossipSubInspectorNotifDistributor is an autogenerated mock type for the GossipSubInspectorNotifDistributor type +type GossipSubInspectorNotifDistributor struct { + mock.Mock +} + +// AddConsumer provides a mock function with given fields: _a0 +func (_m *GossipSubInspectorNotifDistributor) AddConsumer(_a0 p2p.GossipSubInvCtrlMsgNotifConsumer) { + _m.Called(_a0) +} + +// Distribute provides a mock function with given fields: notification +func (_m *GossipSubInspectorNotifDistributor) Distribute(notification *p2p.InvCtrlMsgNotif) error { + ret := _m.Called(notification) + + var r0 error + if rf, ok := ret.Get(0).(func(*p2p.InvCtrlMsgNotif) error); ok { + r0 = rf(notification) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Done provides a mock function with given fields: +func (_m *GossipSubInspectorNotifDistributor) Done() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// Ready provides a mock function with given fields: +func (_m *GossipSubInspectorNotifDistributor) Ready() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// Start provides a mock function with given fields: _a0 +func (_m *GossipSubInspectorNotifDistributor) Start(_a0 irrecoverable.SignalerContext) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewGossipSubInspectorNotifDistributor interface { + mock.TestingT + Cleanup(func()) +} + +// NewGossipSubInspectorNotifDistributor creates a new instance of GossipSubInspectorNotifDistributor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGossipSubInspectorNotifDistributor(t mockConstructorTestingTNewGossipSubInspectorNotifDistributor) *GossipSubInspectorNotifDistributor { + mock := &GossipSubInspectorNotifDistributor{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/gossip_sub_inspector_notification_distributor.go b/network/p2p/mock/gossip_sub_inspector_notification_distributor.go index 57e779e2597..757cd8fa363 100644 --- a/network/p2p/mock/gossip_sub_inspector_notification_distributor.go +++ b/network/p2p/mock/gossip_sub_inspector_notification_distributor.go @@ -15,16 +15,16 @@ type GossipSubInspectorNotificationDistributor struct { } // AddConsumer provides a mock function with given fields: _a0 -func (_m *GossipSubInspectorNotificationDistributor) AddConsumer(_a0 p2p.GossipSubInvalidControlMessageNotificationConsumer) { +func (_m *GossipSubInspectorNotificationDistributor) AddConsumer(_a0 p2p.GossipSubInvCtrlMsgNotifConsumer) { _m.Called(_a0) } // DistributeInvalidControlMessageNotification provides a mock function with given fields: notification -func (_m *GossipSubInspectorNotificationDistributor) DistributeInvalidControlMessageNotification(notification *p2p.InvalidControlMessageNotification) error { +func (_m *GossipSubInspectorNotificationDistributor) Distribute(notification *p2p.InvCtrlMsgNotif) error { ret := _m.Called(notification) var r0 error - if rf, ok := ret.Get(0).(func(*p2p.InvalidControlMessageNotification) error); ok { + if rf, ok := ret.Get(0).(func(*p2p.InvCtrlMsgNotif) error); ok { r0 = rf(notification) } else { r0 = ret.Error(0) diff --git a/network/p2p/mock/gossip_sub_inspector_suite.go b/network/p2p/mock/gossip_sub_inspector_suite.go new file mode 100644 index 00000000000..90c7e5b15d7 --- /dev/null +++ b/network/p2p/mock/gossip_sub_inspector_suite.go @@ -0,0 +1,99 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + flow "github.com/onflow/flow-go/model/flow" + irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + + mock "github.com/stretchr/testify/mock" + + p2p "github.com/onflow/flow-go/network/p2p" + + peer "github.com/libp2p/go-libp2p/core/peer" + + pubsub "github.com/libp2p/go-libp2p-pubsub" +) + +// GossipSubInspectorSuite is an autogenerated mock type for the GossipSubInspectorSuite type +type GossipSubInspectorSuite struct { + mock.Mock +} + +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *GossipSubInspectorSuite) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + +// AddInvalidControlMessageConsumer provides a mock function with given fields: _a0 +func (_m *GossipSubInspectorSuite) AddInvalidControlMessageConsumer(_a0 p2p.GossipSubInvCtrlMsgNotifConsumer) { + _m.Called(_a0) +} + +// Done provides a mock function with given fields: +func (_m *GossipSubInspectorSuite) Done() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// InspectFunc provides a mock function with given fields: +func (_m *GossipSubInspectorSuite) InspectFunc() func(peer.ID, *pubsub.RPC) error { + ret := _m.Called() + + var r0 func(peer.ID, *pubsub.RPC) error + if rf, ok := ret.Get(0).(func() func(peer.ID, *pubsub.RPC) error); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(func(peer.ID, *pubsub.RPC) error) + } + } + + return r0 +} + +// Ready provides a mock function with given fields: +func (_m *GossipSubInspectorSuite) Ready() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// Start provides a mock function with given fields: _a0 +func (_m *GossipSubInspectorSuite) Start(_a0 irrecoverable.SignalerContext) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewGossipSubInspectorSuite interface { + mock.TestingT + Cleanup(func()) +} + +// NewGossipSubInspectorSuite creates a new instance of GossipSubInspectorSuite. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGossipSubInspectorSuite(t mockConstructorTestingTNewGossipSubInspectorSuite) *GossipSubInspectorSuite { + mock := &GossipSubInspectorSuite{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/gossip_sub_inv_ctrl_msg_notif_consumer.go b/network/p2p/mock/gossip_sub_inv_ctrl_msg_notif_consumer.go new file mode 100644 index 00000000000..56de1ef6093 --- /dev/null +++ b/network/p2p/mock/gossip_sub_inv_ctrl_msg_notif_consumer.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + p2p "github.com/onflow/flow-go/network/p2p" + mock "github.com/stretchr/testify/mock" +) + +// GossipSubInvCtrlMsgNotifConsumer is an autogenerated mock type for the GossipSubInvCtrlMsgNotifConsumer type +type GossipSubInvCtrlMsgNotifConsumer struct { + mock.Mock +} + +// OnInvalidControlMessageNotification provides a mock function with given fields: _a0 +func (_m *GossipSubInvCtrlMsgNotifConsumer) OnInvalidControlMessageNotification(_a0 *p2p.InvCtrlMsgNotif) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewGossipSubInvCtrlMsgNotifConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewGossipSubInvCtrlMsgNotifConsumer creates a new instance of GossipSubInvCtrlMsgNotifConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGossipSubInvCtrlMsgNotifConsumer(t mockConstructorTestingTNewGossipSubInvCtrlMsgNotifConsumer) *GossipSubInvCtrlMsgNotifConsumer { + mock := &GossipSubInvCtrlMsgNotifConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/gossip_sub_invalid_control_message_notification_consumer.go b/network/p2p/mock/gossip_sub_invalid_control_message_notification_consumer.go index c0dfd93bedb..8df3aae5870 100644 --- a/network/p2p/mock/gossip_sub_invalid_control_message_notification_consumer.go +++ b/network/p2p/mock/gossip_sub_invalid_control_message_notification_consumer.go @@ -13,7 +13,7 @@ type GossipSubInvalidControlMessageNotificationConsumer struct { } // OnInvalidControlMessageNotification provides a mock function with given fields: _a0 -func (_m *GossipSubInvalidControlMessageNotificationConsumer) OnInvalidControlMessageNotification(_a0 *p2p.InvalidControlMessageNotification) { +func (_m *GossipSubInvalidControlMessageNotificationConsumer) OnInvalidControlMessageNotification(_a0 *p2p.InvCtrlMsgNotif) { _m.Called(_a0) } diff --git a/network/p2p/mock/gossip_sub_msg_validation_rpc_inspector.go b/network/p2p/mock/gossip_sub_msg_validation_rpc_inspector.go new file mode 100644 index 00000000000..41d3a409533 --- /dev/null +++ b/network/p2p/mock/gossip_sub_msg_validation_rpc_inspector.go @@ -0,0 +1,104 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + flow "github.com/onflow/flow-go/model/flow" + irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" + + pubsub "github.com/libp2p/go-libp2p-pubsub" +) + +// GossipSubMsgValidationRpcInspector is an autogenerated mock type for the GossipSubMsgValidationRpcInspector type +type GossipSubMsgValidationRpcInspector struct { + mock.Mock +} + +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *GossipSubMsgValidationRpcInspector) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + +// Done provides a mock function with given fields: +func (_m *GossipSubMsgValidationRpcInspector) Done() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// Inspect provides a mock function with given fields: _a0, _a1 +func (_m *GossipSubMsgValidationRpcInspector) Inspect(_a0 peer.ID, _a1 *pubsub.RPC) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(peer.ID, *pubsub.RPC) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Name provides a mock function with given fields: +func (_m *GossipSubMsgValidationRpcInspector) Name() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// Ready provides a mock function with given fields: +func (_m *GossipSubMsgValidationRpcInspector) Ready() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// Start provides a mock function with given fields: _a0 +func (_m *GossipSubMsgValidationRpcInspector) Start(_a0 irrecoverable.SignalerContext) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewGossipSubMsgValidationRpcInspector interface { + mock.TestingT + Cleanup(func()) +} + +// NewGossipSubMsgValidationRpcInspector creates a new instance of GossipSubMsgValidationRpcInspector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGossipSubMsgValidationRpcInspector(t mockConstructorTestingTNewGossipSubMsgValidationRpcInspector) *GossipSubMsgValidationRpcInspector { + mock := &GossipSubMsgValidationRpcInspector{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/gossip_sub_rpc_inspector_suite_factory_func.go b/network/p2p/mock/gossip_sub_rpc_inspector_suite_factory_func.go new file mode 100644 index 00000000000..03ea2329d85 --- /dev/null +++ b/network/p2p/mock/gossip_sub_rpc_inspector_suite_factory_func.go @@ -0,0 +1,66 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + flow "github.com/onflow/flow-go/model/flow" + metrics "github.com/onflow/flow-go/module/metrics" + + mock "github.com/stretchr/testify/mock" + + module "github.com/onflow/flow-go/module" + + network "github.com/onflow/flow-go/network" + + p2p "github.com/onflow/flow-go/network/p2p" + + p2pconf "github.com/onflow/flow-go/network/p2p/p2pconf" + + zerolog "github.com/rs/zerolog" +) + +// GossipSubRpcInspectorSuiteFactoryFunc is an autogenerated mock type for the GossipSubRpcInspectorSuiteFactoryFunc type +type GossipSubRpcInspectorSuiteFactoryFunc struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4, _a5, _a6 +func (_m *GossipSubRpcInspectorSuiteFactoryFunc) Execute(_a0 zerolog.Logger, _a1 flow.Identifier, _a2 *p2pconf.GossipSubRPCInspectorsConfig, _a3 module.GossipSubMetrics, _a4 metrics.HeroCacheMetricsFactory, _a5 network.NetworkingType, _a6 module.IdentityProvider) (p2p.GossipSubInspectorSuite, error) { + ret := _m.Called(_a0, _a1, _a2, _a3, _a4, _a5, _a6) + + var r0 p2p.GossipSubInspectorSuite + var r1 error + if rf, ok := ret.Get(0).(func(zerolog.Logger, flow.Identifier, *p2pconf.GossipSubRPCInspectorsConfig, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider) (p2p.GossipSubInspectorSuite, error)); ok { + return rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6) + } + if rf, ok := ret.Get(0).(func(zerolog.Logger, flow.Identifier, *p2pconf.GossipSubRPCInspectorsConfig, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider) p2p.GossipSubInspectorSuite); ok { + r0 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(p2p.GossipSubInspectorSuite) + } + } + + if rf, ok := ret.Get(1).(func(zerolog.Logger, flow.Identifier, *p2pconf.GossipSubRPCInspectorsConfig, module.GossipSubMetrics, metrics.HeroCacheMetricsFactory, network.NetworkingType, module.IdentityProvider) error); ok { + r1 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewGossipSubRpcInspectorSuiteFactoryFunc interface { + mock.TestingT + Cleanup(func()) +} + +// NewGossipSubRpcInspectorSuiteFactoryFunc creates a new instance of GossipSubRpcInspectorSuiteFactoryFunc. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGossipSubRpcInspectorSuiteFactoryFunc(t mockConstructorTestingTNewGossipSubRpcInspectorSuiteFactoryFunc) *GossipSubRpcInspectorSuiteFactoryFunc { + mock := &GossipSubRpcInspectorSuiteFactoryFunc{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/gossip_sub_spam_record_cache.go b/network/p2p/mock/gossip_sub_spam_record_cache.go new file mode 100644 index 00000000000..35e674fdffb --- /dev/null +++ b/network/p2p/mock/gossip_sub_spam_record_cache.go @@ -0,0 +1,117 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + p2p "github.com/onflow/flow-go/network/p2p" + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// GossipSubSpamRecordCache is an autogenerated mock type for the GossipSubSpamRecordCache type +type GossipSubSpamRecordCache struct { + mock.Mock +} + +// Add provides a mock function with given fields: peerId, record +func (_m *GossipSubSpamRecordCache) Add(peerId peer.ID, record p2p.GossipSubSpamRecord) bool { + ret := _m.Called(peerId, record) + + var r0 bool + if rf, ok := ret.Get(0).(func(peer.ID, p2p.GossipSubSpamRecord) bool); ok { + r0 = rf(peerId, record) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// Get provides a mock function with given fields: peerID +func (_m *GossipSubSpamRecordCache) Get(peerID peer.ID) (*p2p.GossipSubSpamRecord, error, bool) { + ret := _m.Called(peerID) + + var r0 *p2p.GossipSubSpamRecord + var r1 error + var r2 bool + if rf, ok := ret.Get(0).(func(peer.ID) (*p2p.GossipSubSpamRecord, error, bool)); ok { + return rf(peerID) + } + if rf, ok := ret.Get(0).(func(peer.ID) *p2p.GossipSubSpamRecord); ok { + r0 = rf(peerID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*p2p.GossipSubSpamRecord) + } + } + + if rf, ok := ret.Get(1).(func(peer.ID) error); ok { + r1 = rf(peerID) + } else { + r1 = ret.Error(1) + } + + if rf, ok := ret.Get(2).(func(peer.ID) bool); ok { + r2 = rf(peerID) + } else { + r2 = ret.Get(2).(bool) + } + + return r0, r1, r2 +} + +// Has provides a mock function with given fields: peerID +func (_m *GossipSubSpamRecordCache) Has(peerID peer.ID) bool { + ret := _m.Called(peerID) + + var r0 bool + if rf, ok := ret.Get(0).(func(peer.ID) bool); ok { + r0 = rf(peerID) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// Update provides a mock function with given fields: peerID, updateFunc +func (_m *GossipSubSpamRecordCache) Update(peerID peer.ID, updateFunc p2p.UpdateFunction) (*p2p.GossipSubSpamRecord, error) { + ret := _m.Called(peerID, updateFunc) + + var r0 *p2p.GossipSubSpamRecord + var r1 error + if rf, ok := ret.Get(0).(func(peer.ID, p2p.UpdateFunction) (*p2p.GossipSubSpamRecord, error)); ok { + return rf(peerID, updateFunc) + } + if rf, ok := ret.Get(0).(func(peer.ID, p2p.UpdateFunction) *p2p.GossipSubSpamRecord); ok { + r0 = rf(peerID, updateFunc) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*p2p.GossipSubSpamRecord) + } + } + + if rf, ok := ret.Get(1).(func(peer.ID, p2p.UpdateFunction) error); ok { + r1 = rf(peerID, updateFunc) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewGossipSubSpamRecordCache interface { + mock.TestingT + Cleanup(func()) +} + +// NewGossipSubSpamRecordCache creates a new instance of GossipSubSpamRecordCache. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGossipSubSpamRecordCache(t mockConstructorTestingTNewGossipSubSpamRecordCache) *GossipSubSpamRecordCache { + mock := &GossipSubSpamRecordCache{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/lib_p2_p_factory_func.go b/network/p2p/mock/lib_p2_p_factory_func.go deleted file mode 100644 index cde65cd1e35..00000000000 --- a/network/p2p/mock/lib_p2_p_factory_func.go +++ /dev/null @@ -1,54 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mockp2p - -import ( - p2p "github.com/onflow/flow-go/network/p2p" - mock "github.com/stretchr/testify/mock" -) - -// LibP2PFactoryFunc is an autogenerated mock type for the LibP2PFactoryFunc type -type LibP2PFactoryFunc struct { - mock.Mock -} - -// Execute provides a mock function with given fields: -func (_m *LibP2PFactoryFunc) Execute() (p2p.LibP2PNode, error) { - ret := _m.Called() - - var r0 p2p.LibP2PNode - var r1 error - if rf, ok := ret.Get(0).(func() (p2p.LibP2PNode, error)); ok { - return rf() - } - if rf, ok := ret.Get(0).(func() p2p.LibP2PNode); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(p2p.LibP2PNode) - } - } - - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type mockConstructorTestingTNewLibP2PFactoryFunc interface { - mock.TestingT - Cleanup(func()) -} - -// NewLibP2PFactoryFunc creates a new instance of LibP2PFactoryFunc. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewLibP2PFactoryFunc(t mockConstructorTestingTNewLibP2PFactoryFunc) *LibP2PFactoryFunc { - mock := &LibP2PFactoryFunc{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/network/p2p/mock/lib_p2_p_node.go b/network/p2p/mock/lib_p2_p_node.go index 326b2280eca..5813983110e 100644 --- a/network/p2p/mock/lib_p2_p_node.go +++ b/network/p2p/mock/lib_p2_p_node.go @@ -8,6 +8,10 @@ import ( context "context" + flow "github.com/onflow/flow-go/model/flow" + + flow_gonetwork "github.com/onflow/flow-go/network" + host "github.com/libp2p/go-libp2p/core/host" irrecoverable "github.com/onflow/flow-go/module/irrecoverable" @@ -34,6 +38,11 @@ type LibP2PNode struct { mock.Mock } +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *LibP2PNode) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + // AddPeer provides a mock function with given fields: ctx, peerInfo func (_m *LibP2PNode) AddPeer(ctx context.Context, peerInfo peer.AddrInfo) error { ret := _m.Called(ctx, peerInfo) @@ -191,6 +200,32 @@ func (_m *LibP2PNode) IsConnected(peerID peer.ID) (bool, error) { return r0, r1 } +// IsDisallowListed provides a mock function with given fields: peerId +func (_m *LibP2PNode) IsDisallowListed(peerId peer.ID) ([]flow_gonetwork.DisallowListedCause, bool) { + ret := _m.Called(peerId) + + var r0 []flow_gonetwork.DisallowListedCause + var r1 bool + if rf, ok := ret.Get(0).(func(peer.ID) ([]flow_gonetwork.DisallowListedCause, bool)); ok { + return rf(peerId) + } + if rf, ok := ret.Get(0).(func(peer.ID) []flow_gonetwork.DisallowListedCause); ok { + r0 = rf(peerId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]flow_gonetwork.DisallowListedCause) + } + } + + if rf, ok := ret.Get(1).(func(peer.ID) bool); ok { + r1 = rf(peerId) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + // ListPeers provides a mock function with given fields: topic func (_m *LibP2PNode) ListPeers(topic string) []peer.ID { ret := _m.Called(topic) @@ -207,6 +242,16 @@ func (_m *LibP2PNode) ListPeers(topic string) []peer.ID { return r0 } +// OnAllowListNotification provides a mock function with given fields: id, cause +func (_m *LibP2PNode) OnAllowListNotification(id peer.ID, cause flow_gonetwork.DisallowListedCause) { + _m.Called(id, cause) +} + +// OnDisallowListNotification provides a mock function with given fields: id, cause +func (_m *LibP2PNode) OnDisallowListNotification(id peer.ID, cause flow_gonetwork.DisallowListedCause) { + _m.Called(id, cause) +} + // PeerManagerComponent provides a mock function with given fields: func (_m *LibP2PNode) PeerManagerComponent() component.Component { ret := _m.Called() @@ -224,14 +269,10 @@ func (_m *LibP2PNode) PeerManagerComponent() component.Component { } // PeerScoreExposer provides a mock function with given fields: -func (_m *LibP2PNode) PeerScoreExposer() (p2p.PeerScoreExposer, bool) { +func (_m *LibP2PNode) PeerScoreExposer() p2p.PeerScoreExposer { ret := _m.Called() var r0 p2p.PeerScoreExposer - var r1 bool - if rf, ok := ret.Get(0).(func() (p2p.PeerScoreExposer, bool)); ok { - return rf() - } if rf, ok := ret.Get(0).(func() p2p.PeerScoreExposer); ok { r0 = rf() } else { @@ -240,13 +281,7 @@ func (_m *LibP2PNode) PeerScoreExposer() (p2p.PeerScoreExposer, bool) { } } - if rf, ok := ret.Get(1).(func() bool); ok { - r1 = rf() - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 + return r0 } // Publish provides a mock function with given fields: ctx, topic, data @@ -335,11 +370,6 @@ func (_m *LibP2PNode) SetComponentManager(cm *component.ComponentManager) { _m.Called(cm) } -// SetPeerScoreExposer provides a mock function with given fields: e -func (_m *LibP2PNode) SetPeerScoreExposer(e p2p.PeerScoreExposer) { - _m.Called(e) -} - // SetPubSub provides a mock function with given fields: ps func (_m *LibP2PNode) SetPubSub(ps p2p.PubSubAdapter) { _m.Called(ps) diff --git a/network/p2p/mock/network_config_option.go b/network/p2p/mock/network_config_option.go new file mode 100644 index 00000000000..89fc2bc5b78 --- /dev/null +++ b/network/p2p/mock/network_config_option.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + p2p "github.com/onflow/flow-go/network/p2p" + mock "github.com/stretchr/testify/mock" +) + +// NetworkConfigOption is an autogenerated mock type for the NetworkConfigOption type +type NetworkConfigOption struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0 +func (_m *NetworkConfigOption) Execute(_a0 *p2p.NetworkConfig) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewNetworkConfigOption interface { + mock.TestingT + Cleanup(func()) +} + +// NewNetworkConfigOption creates a new instance of NetworkConfigOption. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewNetworkConfigOption(t mockConstructorTestingTNewNetworkConfigOption) *NetworkConfigOption { + mock := &NetworkConfigOption{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/network_option.go b/network/p2p/mock/network_option.go new file mode 100644 index 00000000000..470eb615c23 --- /dev/null +++ b/network/p2p/mock/network_option.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + p2p "github.com/onflow/flow-go/network/p2p" + mock "github.com/stretchr/testify/mock" +) + +// NetworkOption is an autogenerated mock type for the NetworkOption type +type NetworkOption struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0 +func (_m *NetworkOption) Execute(_a0 *p2p.Network) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewNetworkOption interface { + mock.TestingT + Cleanup(func()) +} + +// NewNetworkOption creates a new instance of NetworkOption. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewNetworkOption(t mockConstructorTestingTNewNetworkOption) *NetworkOption { + mock := &NetworkOption{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/network_param_option.go b/network/p2p/mock/network_param_option.go new file mode 100644 index 00000000000..aa84df68497 --- /dev/null +++ b/network/p2p/mock/network_param_option.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + p2p "github.com/onflow/flow-go/network/p2p" + mock "github.com/stretchr/testify/mock" +) + +// NetworkParamOption is an autogenerated mock type for the NetworkParamOption type +type NetworkParamOption struct { + mock.Mock +} + +// Execute provides a mock function with given fields: _a0 +func (_m *NetworkParamOption) Execute(_a0 *p2p.NetworkConfig) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewNetworkParamOption interface { + mock.TestingT + Cleanup(func()) +} + +// NewNetworkParamOption creates a new instance of NetworkParamOption. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewNetworkParamOption(t mockConstructorTestingTNewNetworkParamOption) *NetworkParamOption { + mock := &NetworkParamOption{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/node_builder.go b/network/p2p/mock/node_builder.go index 4b7dff23f67..15bb6c10306 100644 --- a/network/p2p/mock/node_builder.go +++ b/network/p2p/mock/node_builder.go @@ -13,8 +13,6 @@ import ( mock "github.com/stretchr/testify/mock" - module "github.com/onflow/flow-go/module" - network "github.com/libp2p/go-libp2p/core/network" p2p "github.com/onflow/flow-go/network/p2p" @@ -57,13 +55,29 @@ func (_m *NodeBuilder) Build() (p2p.LibP2PNode, error) { return r0, r1 } -// EnableGossipSubPeerScoring provides a mock function with given fields: _a0, _a1 -func (_m *NodeBuilder) EnableGossipSubPeerScoring(_a0 module.IdentityProvider, _a1 *p2p.PeerScoringConfig) p2p.NodeBuilder { - ret := _m.Called(_a0, _a1) +// EnableGossipSubScoringWithOverride provides a mock function with given fields: _a0 +func (_m *NodeBuilder) EnableGossipSubScoringWithOverride(_a0 *p2p.PeerScoringConfigOverride) p2p.NodeBuilder { + ret := _m.Called(_a0) var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(module.IdentityProvider, *p2p.PeerScoringConfig) p2p.NodeBuilder); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(*p2p.PeerScoringConfigOverride) p2p.NodeBuilder); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(p2p.NodeBuilder) + } + } + + return r0 +} + +// OverrideDefaultRpcInspectorSuiteFactory provides a mock function with given fields: _a0 +func (_m *NodeBuilder) OverrideDefaultRpcInspectorSuiteFactory(_a0 p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder { + ret := _m.Called(_a0) + + var r0 p2p.NodeBuilder + if rf, ok := ret.Get(0).(func(p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder); ok { + r0 = rf(_a0) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(p2p.NodeBuilder) @@ -90,11 +104,11 @@ func (_m *NodeBuilder) SetBasicResolver(_a0 madns.BasicResolver) p2p.NodeBuilder } // SetConnectionGater provides a mock function with given fields: _a0 -func (_m *NodeBuilder) SetConnectionGater(_a0 connmgr.ConnectionGater) p2p.NodeBuilder { +func (_m *NodeBuilder) SetConnectionGater(_a0 p2p.ConnectionGater) p2p.NodeBuilder { ret := _m.Called(_a0) var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(connmgr.ConnectionGater) p2p.NodeBuilder); ok { + if rf, ok := ret.Get(0).(func(p2p.ConnectionGater) p2p.NodeBuilder); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { @@ -153,28 +167,6 @@ func (_m *NodeBuilder) SetGossipSubFactory(_a0 p2p.GossipSubFactoryFunc, _a1 p2p return r0 } -// SetGossipSubRPCInspectors provides a mock function with given fields: inspectors -func (_m *NodeBuilder) SetGossipSubRPCInspectors(inspectors ...p2p.GossipSubRPCInspector) p2p.NodeBuilder { - _va := make([]interface{}, len(inspectors)) - for _i := range inspectors { - _va[_i] = inspectors[_i] - } - var _ca []interface{} - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(...p2p.GossipSubRPCInspector) p2p.NodeBuilder); ok { - r0 = rf(inspectors...) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(p2p.NodeBuilder) - } - } - - return r0 -} - // SetGossipSubScoreTracerInterval provides a mock function with given fields: _a0 func (_m *NodeBuilder) SetGossipSubScoreTracerInterval(_a0 time.Duration) p2p.NodeBuilder { ret := _m.Called(_a0) @@ -207,22 +199,6 @@ func (_m *NodeBuilder) SetGossipSubTracer(_a0 p2p.PubSubTracer) p2p.NodeBuilder return r0 } -// SetPeerManagerOptions provides a mock function with given fields: _a0, _a1 -func (_m *NodeBuilder) SetPeerManagerOptions(_a0 bool, _a1 time.Duration) p2p.NodeBuilder { - ret := _m.Called(_a0, _a1) - - var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(bool, time.Duration) p2p.NodeBuilder); ok { - r0 = rf(_a0, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(p2p.NodeBuilder) - } - } - - return r0 -} - // SetRateLimiterDistributor provides a mock function with given fields: _a0 func (_m *NodeBuilder) SetRateLimiterDistributor(_a0 p2p.UnicastRateLimiterDistributor) p2p.NodeBuilder { ret := _m.Called(_a0) diff --git a/network/p2p/mock/peer_score.go b/network/p2p/mock/peer_score.go index 374d03d6749..81ea79ec570 100644 --- a/network/p2p/mock/peer_score.go +++ b/network/p2p/mock/peer_score.go @@ -13,14 +13,10 @@ type PeerScore struct { } // PeerScoreExposer provides a mock function with given fields: -func (_m *PeerScore) PeerScoreExposer() (p2p.PeerScoreExposer, bool) { +func (_m *PeerScore) PeerScoreExposer() p2p.PeerScoreExposer { ret := _m.Called() var r0 p2p.PeerScoreExposer - var r1 bool - if rf, ok := ret.Get(0).(func() (p2p.PeerScoreExposer, bool)); ok { - return rf() - } if rf, ok := ret.Get(0).(func() p2p.PeerScoreExposer); ok { r0 = rf() } else { @@ -29,18 +25,7 @@ func (_m *PeerScore) PeerScoreExposer() (p2p.PeerScoreExposer, bool) { } } - if rf, ok := ret.Get(1).(func() bool); ok { - r1 = rf() - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 -} - -// SetPeerScoreExposer provides a mock function with given fields: e -func (_m *PeerScore) SetPeerScoreExposer(e p2p.PeerScoreExposer) { - _m.Called(e) + return r0 } type mockConstructorTestingTNewPeerScore interface { diff --git a/network/p2p/mock/peer_updater.go b/network/p2p/mock/peer_updater.go new file mode 100644 index 00000000000..7a708e6c3df --- /dev/null +++ b/network/p2p/mock/peer_updater.go @@ -0,0 +1,36 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// PeerUpdater is an autogenerated mock type for the PeerUpdater type +type PeerUpdater struct { + mock.Mock +} + +// UpdatePeers provides a mock function with given fields: ctx, peerIDs +func (_m *PeerUpdater) UpdatePeers(ctx context.Context, peerIDs peer.IDSlice) { + _m.Called(ctx, peerIDs) +} + +type mockConstructorTestingTNewPeerUpdater interface { + mock.TestingT + Cleanup(func()) +} + +// NewPeerUpdater creates a new instance of PeerUpdater. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewPeerUpdater(t mockConstructorTestingTNewPeerUpdater) *PeerUpdater { + mock := &PeerUpdater{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/pub_sub_adapter.go b/network/p2p/mock/pub_sub_adapter.go index d8f2cf533a2..ec05980dea1 100644 --- a/network/p2p/mock/pub_sub_adapter.go +++ b/network/p2p/mock/pub_sub_adapter.go @@ -3,7 +3,9 @@ package mockp2p import ( + flow "github.com/onflow/flow-go/model/flow" irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + mock "github.com/stretchr/testify/mock" p2p "github.com/onflow/flow-go/network/p2p" @@ -16,6 +18,11 @@ type PubSubAdapter struct { mock.Mock } +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *PubSubAdapter) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + // Done provides a mock function with given fields: func (_m *PubSubAdapter) Done() <-chan struct{} { ret := _m.Called() @@ -90,6 +97,22 @@ func (_m *PubSubAdapter) ListPeers(topic string) []peer.ID { return r0 } +// PeerScoreExposer provides a mock function with given fields: +func (_m *PubSubAdapter) PeerScoreExposer() p2p.PeerScoreExposer { + ret := _m.Called() + + var r0 p2p.PeerScoreExposer + if rf, ok := ret.Get(0).(func() p2p.PeerScoreExposer); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(p2p.PeerScoreExposer) + } + } + + return r0 +} + // Ready provides a mock function with given fields: func (_m *PubSubAdapter) Ready() <-chan struct{} { ret := _m.Called() diff --git a/network/p2p/mock/pub_sub_adapter_config.go b/network/p2p/mock/pub_sub_adapter_config.go index 575ddbe9b70..113ef45a163 100644 --- a/network/p2p/mock/pub_sub_adapter_config.go +++ b/network/p2p/mock/pub_sub_adapter_config.go @@ -14,15 +14,9 @@ type PubSubAdapterConfig struct { mock.Mock } -// WithAppSpecificRpcInspectors provides a mock function with given fields: _a0 -func (_m *PubSubAdapterConfig) WithAppSpecificRpcInspectors(_a0 ...p2p.GossipSubRPCInspector) { - _va := make([]interface{}, len(_a0)) - for _i := range _a0 { - _va[_i] = _a0[_i] - } - var _ca []interface{} - _ca = append(_ca, _va...) - _m.Called(_ca...) +// WithInspectorSuite provides a mock function with given fields: _a0 +func (_m *PubSubAdapterConfig) WithInspectorSuite(_a0 p2p.GossipSubInspectorSuite) { + _m.Called(_a0) } // WithMessageIdFunction provides a mock function with given fields: f diff --git a/network/p2p/mock/score_option_builder.go b/network/p2p/mock/score_option_builder.go index eabe096b50a..d0f437bfc12 100644 --- a/network/p2p/mock/score_option_builder.go +++ b/network/p2p/mock/score_option_builder.go @@ -14,15 +14,43 @@ type ScoreOptionBuilder struct { } // BuildFlowPubSubScoreOption provides a mock function with given fields: -func (_m *ScoreOptionBuilder) BuildFlowPubSubScoreOption() pubsub.Option { +func (_m *ScoreOptionBuilder) BuildFlowPubSubScoreOption() (*pubsub.PeerScoreParams, *pubsub.PeerScoreThresholds) { ret := _m.Called() - var r0 pubsub.Option - if rf, ok := ret.Get(0).(func() pubsub.Option); ok { + var r0 *pubsub.PeerScoreParams + var r1 *pubsub.PeerScoreThresholds + if rf, ok := ret.Get(0).(func() (*pubsub.PeerScoreParams, *pubsub.PeerScoreThresholds)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *pubsub.PeerScoreParams); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(pubsub.Option) + r0 = ret.Get(0).(*pubsub.PeerScoreParams) + } + } + + if rf, ok := ret.Get(1).(func() *pubsub.PeerScoreThresholds); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*pubsub.PeerScoreThresholds) + } + } + + return r0, r1 +} + +// TopicScoreParams provides a mock function with given fields: _a0 +func (_m *ScoreOptionBuilder) TopicScoreParams(_a0 *pubsub.Topic) *pubsub.TopicScoreParams { + ret := _m.Called(_a0) + + var r0 *pubsub.TopicScoreParams + if rf, ok := ret.Get(0).(func(*pubsub.Topic) *pubsub.TopicScoreParams); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*pubsub.TopicScoreParams) } } diff --git a/network/p2p/mock/subscription_validator.go b/network/p2p/mock/subscription_validator.go new file mode 100644 index 00000000000..b7f71843639 --- /dev/null +++ b/network/p2p/mock/subscription_validator.go @@ -0,0 +1,60 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" + + p2p "github.com/onflow/flow-go/network/p2p" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// SubscriptionValidator is an autogenerated mock type for the SubscriptionValidator type +type SubscriptionValidator struct { + mock.Mock +} + +// CheckSubscribedToAllowedTopics provides a mock function with given fields: pid, role +func (_m *SubscriptionValidator) CheckSubscribedToAllowedTopics(pid peer.ID, role flow.Role) error { + ret := _m.Called(pid, role) + + var r0 error + if rf, ok := ret.Get(0).(func(peer.ID, flow.Role) error); ok { + r0 = rf(pid, role) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// RegisterSubscriptionProvider provides a mock function with given fields: provider +func (_m *SubscriptionValidator) RegisterSubscriptionProvider(provider p2p.SubscriptionProvider) error { + ret := _m.Called(provider) + + var r0 error + if rf, ok := ret.Get(0).(func(p2p.SubscriptionProvider) error); ok { + r0 = rf(provider) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewSubscriptionValidator interface { + mock.TestingT + Cleanup(func()) +} + +// NewSubscriptionValidator creates a new instance of SubscriptionValidator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewSubscriptionValidator(t mockConstructorTestingTNewSubscriptionValidator) *SubscriptionValidator { + mock := &SubscriptionValidator{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/update_function.go b/network/p2p/mock/update_function.go new file mode 100644 index 00000000000..1b1b98ed66b --- /dev/null +++ b/network/p2p/mock/update_function.go @@ -0,0 +1,42 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mockp2p + +import ( + p2p "github.com/onflow/flow-go/network/p2p" + mock "github.com/stretchr/testify/mock" +) + +// UpdateFunction is an autogenerated mock type for the UpdateFunction type +type UpdateFunction struct { + mock.Mock +} + +// Execute provides a mock function with given fields: record +func (_m *UpdateFunction) Execute(record p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord { + ret := _m.Called(record) + + var r0 p2p.GossipSubSpamRecord + if rf, ok := ret.Get(0).(func(p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord); ok { + r0 = rf(record) + } else { + r0 = ret.Get(0).(p2p.GossipSubSpamRecord) + } + + return r0 +} + +type mockConstructorTestingTNewUpdateFunction interface { + mock.TestingT + Cleanup(func()) +} + +// NewUpdateFunction creates a new instance of UpdateFunction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewUpdateFunction(t mockConstructorTestingTNewUpdateFunction) *UpdateFunction { + mock := &UpdateFunction{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/network.go b/network/p2p/network.go index b5bf83c8c11..a288b92c7d7 100644 --- a/network/p2p/network.go +++ b/network/p2p/network.go @@ -17,21 +17,16 @@ import ( "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network" + alspmgr "github.com/onflow/flow-go/network/alsp/manager" netcache "github.com/onflow/flow-go/network/cache" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/message" - "github.com/onflow/flow-go/network/p2p/conduit" "github.com/onflow/flow-go/network/queue" + "github.com/onflow/flow-go/network/slashing" _ "github.com/onflow/flow-go/utils/binstat" "github.com/onflow/flow-go/utils/logging" ) -const ( - // DefaultReceiveCacheSize represents size of receive cache that keeps hash of incoming messages - // for sake of deduplication. - DefaultReceiveCacheSize = 10e4 -) - // NotEjectedFilter is an identity filter that, when applied to the identity // table at a given snapshot, returns all nodes that we should communicate with // over the networking layer. @@ -40,14 +35,6 @@ const ( // be included in network communication. We omit any nodes that have been ejected. var NotEjectedFilter = filter.Not(filter.Ejected) -type NetworkOptFunction func(*Network) - -func WithConduitFactory(f network.ConduitFactory) NetworkOptFunction { - return func(n *Network) { - n.conduitFactory = f - } -} - // Network represents the overlay network of our peer-to-peer network, including // the protocols for handshakes, authentication, gossiping and heartbeats. type Network struct { @@ -66,6 +53,8 @@ type Network struct { topology network.Topology registerEngineRequests chan *registerEngineRequest registerBlobServiceRequests chan *registerBlobServiceRequest + misbehaviorReportManager network.MisbehaviorReportManager + slashingViolationsConsumer network.ViolationsConsumer } var _ network.Network = &Network{} @@ -96,7 +85,9 @@ type registerBlobServiceResp struct { var ErrNetworkShutdown = errors.New("network has already shutdown") -type NetworkParameters struct { +// NetworkConfig is a configuration struct for the network. It contains all the +// necessary components to create a new network. +type NetworkConfig struct { Logger zerolog.Logger Codec network.Codec Me module.Local @@ -106,22 +97,64 @@ type NetworkParameters struct { Metrics module.NetworkCoreMetrics IdentityProvider module.IdentityProvider ReceiveCache *netcache.ReceiveCache - Options []NetworkOptFunction + ConduitFactory network.ConduitFactory + AlspCfg *alspmgr.MisbehaviorReportManagerConfig +} + +// NetworkConfigOption is a function that can be used to override network config parmeters. +type NetworkConfigOption func(*NetworkConfig) + +// WithAlspConfig overrides the default misbehavior report manager config. It is mostly used for testing purposes. +// Note: do not override the default misbehavior report manager config in production unless you know what you are doing. +// Args: +// cfg: misbehavior report manager config +// Returns: +// NetworkConfigOption: network param option +func WithAlspConfig(cfg *alspmgr.MisbehaviorReportManagerConfig) NetworkConfigOption { + return func(params *NetworkConfig) { + params.AlspCfg = cfg + } +} + +// NetworkOption is a function that can be used to override network attributes. +// It is mostly used for testing purposes. +// Note: do not override network attributes in production unless you know what you are doing. +type NetworkOption func(*Network) + +// WithAlspManager sets the misbehavior report manager for the network. It overrides the default +// misbehavior report manager that is created from the config. +// Note that this option is mostly used for testing purposes, do not use it in production unless you +// know what you are doing. +// +// Args: +// +// mgr: misbehavior report manager +// +// Returns: +// +// NetworkOption: network option +func WithAlspManager(mgr network.MisbehaviorReportManager) NetworkOption { + return func(n *Network) { + n.misbehaviorReportManager = mgr + } } // NewNetwork creates a new naive overlay network, using the given middleware to // communicate to direct peers, using the given codec for serialization, and // using the given state & cache interfaces to track volatile information. // csize determines the size of the cache dedicated to keep track of received messages -func NewNetwork(param *NetworkParameters) (*Network, error) { - +func NewNetwork(param *NetworkConfig, opts ...NetworkOption) (*Network, error) { mw, err := param.MiddlewareFactory() if err != nil { return nil, fmt.Errorf("could not create middleware: %w", err) } + misbehaviorMngr, err := alspmgr.NewMisbehaviorReportManager(param.AlspCfg, mw) + if err != nil { + return nil, fmt.Errorf("could not create misbehavior report manager: %w", err) + } n := &Network{ - logger: param.Logger, + logger: param.Logger.With().Str("component", "network").Logger(), codec: param.Codec, me: param.Me, mw: mw, @@ -130,15 +163,18 @@ func NewNetwork(param *NetworkParameters) (*Network, error) { metrics: param.Metrics, subscriptionManager: param.SubscriptionManager, identityProvider: param.IdentityProvider, - conduitFactory: conduit.NewDefaultConduitFactory(), + conduitFactory: param.ConduitFactory, registerEngineRequests: make(chan *registerEngineRequest), registerBlobServiceRequests: make(chan *registerBlobServiceRequest), + misbehaviorReportManager: misbehaviorMngr, } - for _, opt := range param.Options { + for _, opt := range opts { opt(n) } + n.slashingViolationsConsumer = slashing.NewSlashingViolationsConsumer(param.Logger, param.Metrics, n) + n.mw.SetSlashingViolationsConsumer(n.slashingViolationsConsumer) n.mw.SetOverlay(n) if err := n.conduitFactory.RegisterAdapter(n); err != nil { @@ -146,6 +182,23 @@ func NewNetwork(param *NetworkParameters) (*Network, error) { } n.ComponentManager = component.NewComponentManagerBuilder(). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + n.logger.Debug().Msg("starting misbehavior manager") + n.misbehaviorReportManager.Start(ctx) + + select { + case <-n.misbehaviorReportManager.Ready(): + n.logger.Debug().Msg("misbehavior manager is ready") + ready() + case <-ctx.Done(): + // jumps to the end of the select statement to let a graceful shutdown. + } + + <-ctx.Done() + n.logger.Debug().Msg("stopping misbehavior manager") + <-n.misbehaviorReportManager.Done() + n.logger.Debug().Msg("misbehavior manager stopped") + }). AddWorker(n.runMiddleware). AddWorker(n.processRegisterEngineRequests). AddWorker(n.processRegisterBlobServiceRequests).Build() @@ -418,13 +471,16 @@ func (n *Network) PublishOnChannel(channel channels.Channel, message interface{} // MulticastOnChannel unreliably sends the specified event over the channel to randomly selected 'num' number of recipients // selected from the specified targetIDs. func (n *Network) MulticastOnChannel(channel channels.Channel, message interface{}, num uint, targetIDs ...flow.Identifier) error { - selectedIDs := flow.IdentifierList(targetIDs).Filter(n.removeSelfFilter()).Sample(num) + selectedIDs, err := flow.IdentifierList(targetIDs).Filter(n.removeSelfFilter()).Sample(num) + if err != nil { + return fmt.Errorf("sampling failed: %w", err) + } if len(selectedIDs) == 0 { return network.EmptyTargetList } - err := n.sendOnChannel(channel, message, selectedIDs) + err = n.sendOnChannel(channel, message, selectedIDs) // publishes the message to the selected targets if err != nil { @@ -505,3 +561,15 @@ func (n *Network) queueSubmitFunc(message interface{}) { func (n *Network) Topology() flow.IdentityList { return n.topology.Fanout(n.Identities()) } + +// ReportMisbehaviorOnChannel reports the misbehavior of a node on sending a message to the current node that appears +// valid based on the networking layer but is considered invalid by the current node based on the Flow protocol. +// The misbehavior report is sent to the current node's networking layer on the given channel to be processed. +// Args: +// - channel: The channel on which the misbehavior report is sent. +// - report: The misbehavior report to be sent. +// Returns: +// none +func (n *Network) ReportMisbehaviorOnChannel(channel channels.Channel, report network.MisbehaviorReport) { + n.misbehaviorReportManager.HandleMisbehaviorReport(channel, report) +} diff --git a/network/p2p/p2pbuilder/config.go b/network/p2p/p2pbuilder/config/config.go similarity index 79% rename from network/p2p/p2pbuilder/config.go rename to network/p2p/p2pbuilder/config/config.go index 953298b44d4..f784edee0cb 100644 --- a/network/p2p/p2pbuilder/config.go +++ b/network/p2p/p2pbuilder/config/config.go @@ -1,4 +1,4 @@ -package p2pbuilder +package p2pconfig import ( "time" @@ -29,4 +29,15 @@ type PeerManagerConfig struct { ConnectionPruning bool // UpdateInterval interval used by the libp2p node peer manager component to periodically request peer updates. UpdateInterval time.Duration + // ConnectorFactory is a factory function to create a new connector. + ConnectorFactory p2p.ConnectorFactory +} + +// PeerManagerDisableConfig returns a configuration that disables the peer manager. +func PeerManagerDisableConfig() *PeerManagerConfig { + return &PeerManagerConfig{ + ConnectionPruning: false, + UpdateInterval: 0, + ConnectorFactory: nil, + } } diff --git a/network/p2p/p2pbuilder/config/metrics.go b/network/p2p/p2pbuilder/config/metrics.go new file mode 100644 index 00000000000..1283035e5a6 --- /dev/null +++ b/network/p2p/p2pbuilder/config/metrics.go @@ -0,0 +1,20 @@ +package p2pconfig + +import ( + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" +) + +// MetricsConfig is a wrapper around the metrics configuration for the libp2p node. +// It is used to pass the metrics configuration to the libp2p node builder. +type MetricsConfig struct { + // HeroCacheFactory is the factory for the HeroCache metrics. It is used to + // create a HeroCache metrics instance for each cache when needed. By passing + // the factory to the libp2p node builder, the libp2p node can create the + // HeroCache metrics instance for each cache internally, which reduces the + // number of arguments needed to be passed to the libp2p node builder. + HeroCacheFactory metrics.HeroCacheMetricsFactory + + // LibP2PMetrics is the metrics instance for the libp2p node. + Metrics module.LibP2PMetrics +} diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index d6e6155a840..3e69590dc17 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -7,24 +7,35 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/rs/zerolog" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/module/mempool/queue" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/distributor" + "github.com/onflow/flow-go/network/p2p/inspector" + "github.com/onflow/flow-go/network/p2p/inspector/validation" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" + inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/p2pnode" "github.com/onflow/flow-go/network/p2p/scoring" "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/utils" + "github.com/onflow/flow-go/utils/logging" ) // The Builder struct is used to configure and create a new GossipSub pubsub system. type Builder struct { + networkType network.NetworkingType + sporkId flow.Identifier logger zerolog.Logger - metrics module.GossipSubMetrics + metricsCfg *p2pconfig.MetricsConfig h host.Host subscriptionFilter pubsub.SubscriptionFilter gossipSubFactory p2p.GossipSubFactoryFunc @@ -33,11 +44,12 @@ type Builder struct { gossipSubScoreTracerInterval time.Duration // the interval at which the gossipsub score tracer logs the peer scores. // gossipSubTracer is a callback interface that is called by the gossipsub implementation upon // certain events. Currently, we use it to log and observe the local mesh of the node. - gossipSubTracer p2p.PubSubTracer - peerScoringParameterOptions []scoring.PeerScoreParamsOption - idProvider module.IdentityProvider - routingSystem routing.Routing - rpcInspectors []p2p.GossipSubRPCInspector + gossipSubTracer p2p.PubSubTracer + scoreOptionConfig *scoring.ScoreOptionConfig + idProvider module.IdentityProvider + routingSystem routing.Routing + rpcInspectorConfig *p2pconf.GossipSubRPCInspectorsConfig + rpcInspectorSuiteFactory p2p.GossipSubRpcInspectorSuiteFactoryFunc } var _ p2p.GossipSubBuilder = (*Builder)(nil) @@ -79,14 +91,42 @@ func (g *Builder) SetGossipSubConfigFunc(gossipSubConfigFunc p2p.GossipSubAdapte g.gossipSubConfigFunc = gossipSubConfigFunc } -// SetGossipSubPeerScoring sets the gossipsub peer scoring of the builder. -// If the gossipsub peer scoring flag has already been set, a fatal error is logged. -func (g *Builder) SetGossipSubPeerScoring(gossipSubPeerScoring bool) { - if g.gossipSubPeerScoring { - g.logger.Fatal().Msg("gossipsub peer scoring has already been set") +// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. +// Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. +// Anything that is left to nil or zero value in the override will be ignored and the default value will be used. +// Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. +// Production Tip: use PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. +// Args: +// - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use PeerScoringConfigNoOverride for production. +// Returns: +// none +func (g *Builder) EnableGossipSubScoringWithOverride(override *p2p.PeerScoringConfigOverride) { + g.gossipSubPeerScoring = true // TODO: we should enable peer scoring by default. + if override == nil { return } - g.gossipSubPeerScoring = gossipSubPeerScoring + if override.AppSpecificScoreParams != nil { + g.logger.Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Msg("overriding app specific score params for gossipsub") + g.scoreOptionConfig.OverrideAppSpecificScoreFunction(override.AppSpecificScoreParams) + } + if override.TopicScoreParams != nil { + for topic, params := range override.TopicScoreParams { + topicLogger := utils.TopicScoreParamsLogger(g.logger, topic.String(), params) + topicLogger.Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Msg("overriding topic score params for gossipsub") + g.scoreOptionConfig.OverrideTopicScoreParams(topic, params) + } + } + if override.DecayInterval > 0 { + g.logger.Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Dur("decay_interval", override.DecayInterval). + Msg("overriding decay interval for gossipsub") + g.scoreOptionConfig.OverrideDecayInterval(override.DecayInterval) + } } // SetGossipSubScoreTracerInterval sets the gossipsub score tracer interval of the builder. @@ -109,17 +149,6 @@ func (g *Builder) SetGossipSubTracer(gossipSubTracer p2p.PubSubTracer) { g.gossipSubTracer = gossipSubTracer } -// SetIDProvider sets the identity provider of the builder. -// If the identity provider has already been set, a fatal error is logged. -func (g *Builder) SetIDProvider(idProvider module.IdentityProvider) { - if g.idProvider != nil { - g.logger.Fatal().Msg("id provider has already been set") - return - } - - g.idProvider = idProvider -} - // SetRoutingSystem sets the routing system of the builder. // If the routing system has already been set, a fatal error is logged. func (g *Builder) SetRoutingSystem(routingSystem routing.Routing) { @@ -130,42 +159,120 @@ func (g *Builder) SetRoutingSystem(routingSystem routing.Routing) { g.routingSystem = routingSystem } -func (g *Builder) SetTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) { - g.peerScoringParameterOptions = append(g.peerScoringParameterOptions, scoring.WithTopicScoreParams(topic, topicScoreParams)) -} - -func (g *Builder) SetAppSpecificScoreParams(f func(peer.ID) float64) { - g.peerScoringParameterOptions = append(g.peerScoringParameterOptions, scoring.WithAppSpecificScoreFunction(f)) -} - -// SetGossipSubRPCInspectors sets the gossipsub rpc inspectors. -func (g *Builder) SetGossipSubRPCInspectors(inspectors ...p2p.GossipSubRPCInspector) { - g.rpcInspectors = inspectors +// OverrideDefaultRpcInspectorSuiteFactory overrides the default rpc inspector suite factory. +// Note: this function should only be used for testing purposes. Never override the default rpc inspector suite factory unless you know what you are doing. +func (g *Builder) OverrideDefaultRpcInspectorSuiteFactory(factory p2p.GossipSubRpcInspectorSuiteFactoryFunc) { + g.logger.Warn().Msg("overriding default rpc inspector suite factory") + g.rpcInspectorSuiteFactory = factory } -func NewGossipSubBuilder(logger zerolog.Logger, metrics module.GossipSubMetrics) *Builder { - return &Builder{ - logger: logger.With().Str("component", "gossipsub").Logger(), - metrics: metrics, - gossipSubFactory: defaultGossipSubFactory(), - gossipSubConfigFunc: defaultGossipSubAdapterConfig(), - peerScoringParameterOptions: make([]scoring.PeerScoreParamsOption, 0), - rpcInspectors: make([]p2p.GossipSubRPCInspector, 0), +// NewGossipSubBuilder returns a new gossipsub builder. +// Args: +// - logger: the logger of the node. +// - metricsCfg: the metrics config of the node. +// - networkType: the network type of the node. +// - sporkId: the spork id of the node. +// - idProvider: the identity provider of the node. +// - rpcInspectorConfig: the rpc inspector config of the node. +// Returns: +// - a new gossipsub builder. +// Note: the builder is not thread-safe. It should only be used in the main thread. +func NewGossipSubBuilder( + logger zerolog.Logger, + metricsCfg *p2pconfig.MetricsConfig, + networkType network.NetworkingType, + sporkId flow.Identifier, + idProvider module.IdentityProvider, + rpcInspectorConfig *p2pconf.GossipSubRPCInspectorsConfig, +) *Builder { + lg := logger.With(). + Str("component", "gossipsub"). + Str("network-type", networkType.String()). + Logger() + + b := &Builder{ + logger: lg, + metricsCfg: metricsCfg, + sporkId: sporkId, + networkType: networkType, + idProvider: idProvider, + gossipSubFactory: defaultGossipSubFactory(), + gossipSubConfigFunc: defaultGossipSubAdapterConfig(), + scoreOptionConfig: scoring.NewScoreOptionConfig(lg, idProvider), + rpcInspectorConfig: rpcInspectorConfig, + rpcInspectorSuiteFactory: defaultInspectorSuite(), } + + return b } +// defaultGossipSubFactory returns the default gossipsub factory function. It is used to create the default gossipsub factory. +// Note: always use the default gossipsub factory function to create the gossipsub factory (unless you know what you are doing). func defaultGossipSubFactory() p2p.GossipSubFactoryFunc { - return func(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error) { - return p2pnode.NewGossipSubAdapter(ctx, logger, h, cfg) + return func( + ctx context.Context, + logger zerolog.Logger, + h host.Host, + cfg p2p.PubSubAdapterConfig, + clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error) { + return p2pnode.NewGossipSubAdapter(ctx, logger, h, cfg, clusterChangeConsumer) } } +// defaultGossipSubAdapterConfig returns the default gossipsub config function. It is used to create the default gossipsub config. +// Note: always use the default gossipsub config function to create the gossipsub config (unless you know what you are doing). func defaultGossipSubAdapterConfig() p2p.GossipSubAdapterConfigFunc { return func(cfg *p2p.BasePubSubAdapterConfig) p2p.PubSubAdapterConfig { return p2pnode.NewGossipSubAdapterConfig(cfg) } } +// defaultInspectorSuite returns the default inspector suite factory function. It is used to create the default inspector suite. +// Inspector suite is utilized to inspect the incoming gossipsub rpc messages from different perspectives. +// Note: always use the default inspector suite factory function to create the inspector suite (unless you know what you are doing). +func defaultInspectorSuite() p2p.GossipSubRpcInspectorSuiteFactoryFunc { + return func( + logger zerolog.Logger, + sporkId flow.Identifier, + inspectorCfg *p2pconf.GossipSubRPCInspectorsConfig, + gossipSubMetrics module.GossipSubMetrics, + heroCacheMetricsFactory metrics.HeroCacheMetricsFactory, + networkType network.NetworkingType, + idProvider module.IdentityProvider) (p2p.GossipSubInspectorSuite, error) { + metricsInspector := inspector.NewControlMsgMetricsInspector( + logger, + p2pnode.NewGossipSubControlMessageMetrics(gossipSubMetrics, logger), + inspectorCfg.GossipSubRPCMetricsInspectorConfigs.NumberOfWorkers, + []queue.HeroStoreConfigOption{ + queue.WithHeroStoreSizeLimit(inspectorCfg.GossipSubRPCMetricsInspectorConfigs.CacheSize), + queue.WithHeroStoreCollector(metrics.GossipSubRPCMetricsObserverInspectorQueueMetricFactory(heroCacheMetricsFactory, networkType)), + }...) + notificationDistributor := distributor.DefaultGossipSubInspectorNotificationDistributor( + logger, + []queue.HeroStoreConfigOption{ + queue.WithHeroStoreSizeLimit(inspectorCfg.GossipSubRPCInspectorNotificationCacheSize), + queue.WithHeroStoreCollector(metrics.RpcInspectorNotificationQueueMetricFactory(heroCacheMetricsFactory, networkType))}...) + + inspectMsgQueueCacheCollector := metrics.GossipSubRPCInspectorQueueMetricFactory(heroCacheMetricsFactory, networkType) + clusterPrefixedCacheCollector := metrics.GossipSubRPCInspectorClusterPrefixedCacheMetricFactory(heroCacheMetricsFactory, networkType) + rpcValidationInspector, err := validation.NewControlMsgValidationInspector( + logger, + sporkId, + &inspectorCfg.GossipSubRPCValidationInspectorConfigs, + notificationDistributor, + inspectMsgQueueCacheCollector, + clusterPrefixedCacheCollector, + idProvider, + gossipSubMetrics, + ) + if err != nil { + return nil, fmt.Errorf("failed to create new control message valiadation inspector: %w", err) + } + + return inspectorbuilder.NewGossipSubInspectorSuite([]p2p.GossipSubRPCInspector{metricsInspector, rpcValidationInspector}, notificationDistributor), nil + } +} + // Build creates a new GossipSub pubsub system. // It returns the newly created GossipSub pubsub system and any errors encountered during its creation. // Arguments: @@ -176,14 +283,14 @@ func defaultGossipSubAdapterConfig() p2p.GossipSubAdapterConfigFunc { // - p2p.PeerScoreTracer: a peer score tracer for the GossipSub pubsub system (if enabled, otherwise nil). // - error: if an error occurs during the creation of the GossipSub pubsub system, it is returned. Otherwise, nil is returned. // Note that on happy path, the returned error is nil. Any error returned is unexpected and should be handled as irrecoverable. -func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, p2p.PeerScoreTracer, error) { +func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, error) { gossipSubConfigs := g.gossipSubConfigFunc(&p2p.BasePubSubAdapterConfig{ MaxMessageSize: p2pnode.DefaultMaxPubSubMsgSize, }) gossipSubConfigs.WithMessageIdFunction(utils.MessageID) if g.routingSystem == nil { - return nil, nil, fmt.Errorf("could not create gossipsub: routing system is nil") + return nil, fmt.Errorf("could not create gossipsub: routing system is nil") } gossipSubConfigs.WithRoutingDiscovery(g.routingSystem) @@ -191,40 +298,60 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, p gossipSubConfigs.WithSubscriptionFilter(g.subscriptionFilter) } + inspectorSuite, err := g.rpcInspectorSuiteFactory( + g.logger, + g.sporkId, + g.rpcInspectorConfig, + g.metricsCfg.Metrics, + g.metricsCfg.HeroCacheFactory, + g.networkType, + g.idProvider) + if err != nil { + return nil, fmt.Errorf("could not create gossipsub inspector suite: %w", err) + } + gossipSubConfigs.WithInspectorSuite(inspectorSuite) + var scoreOpt *scoring.ScoreOption var scoreTracer p2p.PeerScoreTracer if g.gossipSubPeerScoring { - scoreOpt = scoring.NewScoreOption(g.logger, g.idProvider, g.peerScoringParameterOptions...) + g.scoreOptionConfig.SetRegisterNotificationConsumerFunc(inspectorSuite.AddInvalidControlMessageConsumer) + scoreOpt = scoring.NewScoreOption(g.scoreOptionConfig) gossipSubConfigs.WithScoreOption(scoreOpt) if g.gossipSubScoreTracerInterval > 0 { scoreTracer = tracer.NewGossipSubScoreTracer( g.logger, g.idProvider, - g.metrics, + g.metricsCfg.Metrics, g.gossipSubScoreTracerInterval) gossipSubConfigs.WithScoreTracer(scoreTracer) } - } - gossipSubConfigs.WithAppSpecificRpcInspectors(g.rpcInspectors...) + } else { + g.logger.Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Msg("gossipsub peer scoring is disabled") + } if g.gossipSubTracer != nil { gossipSubConfigs.WithTracer(g.gossipSubTracer) } if g.h == nil { - return nil, nil, fmt.Errorf("could not create gossipsub: host is nil") + return nil, fmt.Errorf("could not create gossipsub: host is nil") } - gossipSub, err := g.gossipSubFactory(ctx, g.logger, g.h, gossipSubConfigs) + gossipSub, err := g.gossipSubFactory(ctx, g.logger, g.h, gossipSubConfigs, inspectorSuite) if err != nil { - return nil, nil, fmt.Errorf("could not create gossipsub: %w", err) + return nil, fmt.Errorf("could not create gossipsub: %w", err) } if scoreOpt != nil { - scoreOpt.SetSubscriptionProvider(scoring.NewSubscriptionProvider(g.logger, gossipSub)) + err := scoreOpt.SetSubscriptionProvider(scoring.NewSubscriptionProvider(g.logger, gossipSub)) + if err != nil { + return nil, fmt.Errorf("could not set subscription provider: %w", err) + } } - return gossipSub, scoreTracer, nil + return gossipSub, nil } diff --git a/network/p2p/inspector/aggregate.go b/network/p2p/p2pbuilder/inspector/aggregate.go similarity index 91% rename from network/p2p/inspector/aggregate.go rename to network/p2p/p2pbuilder/inspector/aggregate.go index 64fca023511..604a888fb45 100644 --- a/network/p2p/inspector/aggregate.go +++ b/network/p2p/p2pbuilder/inspector/aggregate.go @@ -32,3 +32,7 @@ func (a *AggregateRPCInspector) Inspect(peerID peer.ID, rpc *pubsub.RPC) error { } return errs.ErrorOrNil() } + +func (a *AggregateRPCInspector) Inspectors() []p2p.GossipSubRPCInspector { + return a.inspectors +} diff --git a/network/p2p/p2pbuilder/inspector/rpc_inspector_builder.go b/network/p2p/p2pbuilder/inspector/rpc_inspector_builder.go deleted file mode 100644 index 75d484a7632..00000000000 --- a/network/p2p/p2pbuilder/inspector/rpc_inspector_builder.go +++ /dev/null @@ -1,219 +0,0 @@ -package inspector - -import ( - "fmt" - - "github.com/rs/zerolog" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/mempool/queue" - "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/distributor" - "github.com/onflow/flow-go/network/p2p/inspector" - "github.com/onflow/flow-go/network/p2p/inspector/validation" - "github.com/onflow/flow-go/network/p2p/p2pnode" -) - -type metricsCollectorFactory func() *metrics.HeroCacheCollector - -// GossipSubRPCValidationInspectorConfigs validation limits used for gossipsub RPC control message inspection. -type GossipSubRPCValidationInspectorConfigs struct { - // NumberOfWorkers number of worker pool workers. - NumberOfWorkers int - // CacheSize size of the queue used by worker pool for the control message validation inspector. - CacheSize uint32 - // GraftLimits GRAFT control message validation limits. - GraftLimits map[string]int - // PruneLimits PRUNE control message validation limits. - PruneLimits map[string]int -} - -// GossipSubRPCMetricsInspectorConfigs rpc metrics observer inspector configuration. -type GossipSubRPCMetricsInspectorConfigs struct { - // NumberOfWorkers number of worker pool workers. - NumberOfWorkers int - // CacheSize size of the queue used by worker pool for the control message metrics inspector. - CacheSize uint32 -} - -// GossipSubRPCInspectorsConfig encompasses configuration related to gossipsub RPC message inspectors. -type GossipSubRPCInspectorsConfig struct { - // GossipSubRPCInspectorNotificationCacheSize size of the queue for notifications about invalid RPC messages. - GossipSubRPCInspectorNotificationCacheSize uint32 - // ValidationInspectorConfigs control message validation inspector validation configuration and limits. - ValidationInspectorConfigs *GossipSubRPCValidationInspectorConfigs - // MetricsInspectorConfigs control message metrics inspector configuration. - MetricsInspectorConfigs *GossipSubRPCMetricsInspectorConfigs -} - -func DefaultGossipSubRPCInspectorsConfig() *GossipSubRPCInspectorsConfig { - return &GossipSubRPCInspectorsConfig{ - GossipSubRPCInspectorNotificationCacheSize: distributor.DefaultGossipSubInspectorNotificationQueueCacheSize, - ValidationInspectorConfigs: &GossipSubRPCValidationInspectorConfigs{ - NumberOfWorkers: validation.DefaultNumberOfWorkers, - CacheSize: validation.DefaultControlMsgValidationInspectorQueueCacheSize, - GraftLimits: map[string]int{ - validation.DiscardThresholdMapKey: validation.DefaultGraftDiscardThreshold, - validation.SafetyThresholdMapKey: validation.DefaultGraftSafetyThreshold, - validation.RateLimitMapKey: validation.DefaultGraftRateLimit, - }, - PruneLimits: map[string]int{ - validation.DiscardThresholdMapKey: validation.DefaultPruneDiscardThreshold, - validation.SafetyThresholdMapKey: validation.DefaultPruneSafetyThreshold, - validation.RateLimitMapKey: validation.DefaultPruneRateLimit, - }, - }, - MetricsInspectorConfigs: &GossipSubRPCMetricsInspectorConfigs{ - NumberOfWorkers: inspector.DefaultControlMsgMetricsInspectorNumberOfWorkers, - CacheSize: inspector.DefaultControlMsgMetricsInspectorQueueCacheSize, - }, - } -} - -// GossipSubInspectorBuilder builder that constructs all rpc inspectors used by gossip sub. The following -// rpc inspectors are created with this builder. -// - validation inspector: performs validation on all control messages. -// - metrics inspector: observes metrics for each rpc message received. -type GossipSubInspectorBuilder struct { - logger zerolog.Logger - sporkID flow.Identifier - inspectorsConfig *GossipSubRPCInspectorsConfig - distributor p2p.GossipSubInspectorNotificationDistributor - netMetrics module.NetworkMetrics - metricsRegistry prometheus.Registerer - metricsEnabled bool - publicNetwork bool -} - -// NewGossipSubInspectorBuilder returns new *GossipSubInspectorBuilder. -func NewGossipSubInspectorBuilder(logger zerolog.Logger, sporkID flow.Identifier, inspectorsConfig *GossipSubRPCInspectorsConfig, distributor p2p.GossipSubInspectorNotificationDistributor) *GossipSubInspectorBuilder { - return &GossipSubInspectorBuilder{ - logger: logger, - sporkID: sporkID, - inspectorsConfig: inspectorsConfig, - distributor: distributor, - netMetrics: metrics.NewNoopCollector(), - metricsEnabled: p2p.MetricsDisabled, - publicNetwork: p2p.PublicNetworkEnabled, - } -} - -// SetMetricsEnabled disable and enable metrics collection for the inspectors underlying hero store cache. -func (b *GossipSubInspectorBuilder) SetMetricsEnabled(metricsEnabled bool) *GossipSubInspectorBuilder { - b.metricsEnabled = metricsEnabled - return b -} - -// SetMetrics sets the network metrics and registry. -func (b *GossipSubInspectorBuilder) SetMetrics(netMetrics module.NetworkMetrics, metricsRegistry prometheus.Registerer) *GossipSubInspectorBuilder { - b.netMetrics = netMetrics - b.metricsRegistry = metricsRegistry - return b -} - -// SetPublicNetwork used to differentiate between libp2p nodes used for public vs private networks. -// Currently, there are different metrics collectors for public vs private networks. -func (b *GossipSubInspectorBuilder) SetPublicNetwork(public bool) *GossipSubInspectorBuilder { - b.publicNetwork = public - return b -} - -// heroStoreOpts builds the gossipsub rpc validation inspector hero store opts. -// These options are used in the underlying worker pool hero store. -func (b *GossipSubInspectorBuilder) heroStoreOpts(size uint32, collectorFactory metricsCollectorFactory) []queue.HeroStoreConfigOption { - heroStoreOpts := []queue.HeroStoreConfigOption{queue.WithHeroStoreSizeLimit(size)} - if b.metricsEnabled { - heroStoreOpts = append(heroStoreOpts, queue.WithHeroStoreCollector(collectorFactory())) - } - return heroStoreOpts -} - -func (b *GossipSubInspectorBuilder) validationInspectorMetricsCollectorFactory() metricsCollectorFactory { - return func() *metrics.HeroCacheCollector { - return metrics.GossipSubRPCValidationInspectorQueueMetricFactory(b.publicNetwork, b.metricsRegistry) - } -} - -func (b *GossipSubInspectorBuilder) metricsInspectorMetricsCollectorFactory() metricsCollectorFactory { - return func() *metrics.HeroCacheCollector { - return metrics.GossipSubRPCMetricsObserverInspectorQueueMetricFactory(b.publicNetwork, b.metricsRegistry) - } -} - -// buildGossipSubMetricsInspector builds the gossipsub rpc metrics inspector. -func (b *GossipSubInspectorBuilder) buildGossipSubMetricsInspector() p2p.GossipSubRPCInspector { - gossipSubMetrics := p2pnode.NewGossipSubControlMessageMetrics(b.netMetrics, b.logger) - metricsInspectorHeroStoreOpts := b.heroStoreOpts(b.inspectorsConfig.MetricsInspectorConfigs.CacheSize, b.metricsInspectorMetricsCollectorFactory()) - metricsInspector := inspector.NewControlMsgMetricsInspector(b.logger, gossipSubMetrics, b.inspectorsConfig.MetricsInspectorConfigs.NumberOfWorkers, metricsInspectorHeroStoreOpts...) - return metricsInspector -} - -// validationInspectorConfig returns a new inspector.ControlMsgValidationInspectorConfig using configuration provided by the node builder. -func (b *GossipSubInspectorBuilder) validationInspectorConfig(validationConfigs *GossipSubRPCValidationInspectorConfigs, opts ...queue.HeroStoreConfigOption) (*validation.ControlMsgValidationInspectorConfig, error) { - // setup rpc validation configuration for each control message type - graftValidationCfg, err := validation.NewCtrlMsgValidationConfig(p2p.CtrlMsgGraft, validationConfigs.GraftLimits) - if err != nil { - return nil, fmt.Errorf("failed to create gossupsub RPC validation configuration: %w", err) - } - pruneValidationCfg, err := validation.NewCtrlMsgValidationConfig(p2p.CtrlMsgPrune, validationConfigs.PruneLimits) - if err != nil { - return nil, fmt.Errorf("failed to create gossupsub RPC validation configuration: %w", err) - } - - // setup gossip sub RPC control message inspector config - controlMsgRPCInspectorCfg := &validation.ControlMsgValidationInspectorConfig{ - NumberOfWorkers: validationConfigs.NumberOfWorkers, - InspectMsgStoreOpts: opts, - GraftValidationCfg: graftValidationCfg, - PruneValidationCfg: pruneValidationCfg, - } - return controlMsgRPCInspectorCfg, nil -} - -// buildGossipSubValidationInspector builds the gossipsub rpc validation inspector. -func (b *GossipSubInspectorBuilder) buildGossipSubValidationInspector() (p2p.GossipSubRPCInspector, error) { - rpcValidationInspectorHeroStoreOpts := b.heroStoreOpts(b.inspectorsConfig.ValidationInspectorConfigs.CacheSize, b.validationInspectorMetricsCollectorFactory()) - controlMsgRPCInspectorCfg, err := b.validationInspectorConfig(b.inspectorsConfig.ValidationInspectorConfigs, rpcValidationInspectorHeroStoreOpts...) - if err != nil { - return nil, fmt.Errorf("failed to create gossipsub rpc inspector config: %w", err) - } - rpcValidationInspector := validation.NewControlMsgValidationInspector(b.logger, b.sporkID, controlMsgRPCInspectorCfg, b.distributor) - return rpcValidationInspector, nil -} - -// Build builds the rpc inspectors used by gossipsub. -// Any returned error from this func indicates a problem setting up rpc inspectors. -// In libp2p node setup, the returned error should be treated as a fatal error. -func (b *GossipSubInspectorBuilder) Build() ([]p2p.GossipSubRPCInspector, error) { - metricsInspector := b.buildGossipSubMetricsInspector() - validationInspector, err := b.buildGossipSubValidationInspector() - if err != nil { - return nil, err - } - return []p2p.GossipSubRPCInspector{metricsInspector, validationInspector}, nil -} - -// DefaultRPCValidationConfig returns default RPC control message inspector config. -func DefaultRPCValidationConfig(opts ...queue.HeroStoreConfigOption) *validation.ControlMsgValidationInspectorConfig { - graftCfg, _ := validation.NewCtrlMsgValidationConfig(p2p.CtrlMsgGraft, validation.CtrlMsgValidationLimits{ - validation.DiscardThresholdMapKey: validation.DefaultGraftDiscardThreshold, - validation.SafetyThresholdMapKey: validation.DefaultGraftSafetyThreshold, - validation.RateLimitMapKey: validation.DefaultGraftRateLimit, - }) - pruneCfg, _ := validation.NewCtrlMsgValidationConfig(p2p.CtrlMsgPrune, validation.CtrlMsgValidationLimits{ - validation.DiscardThresholdMapKey: validation.DefaultPruneDiscardThreshold, - validation.SafetyThresholdMapKey: validation.DefaultPruneSafetyThreshold, - validation.RateLimitMapKey: validation.DefaultPruneRateLimit, - }) - - return &validation.ControlMsgValidationInspectorConfig{ - NumberOfWorkers: validation.DefaultNumberOfWorkers, - InspectMsgStoreOpts: opts, - GraftValidationCfg: graftCfg, - PruneValidationCfg: pruneCfg, - } -} diff --git a/network/p2p/p2pbuilder/inspector/suite.go b/network/p2p/p2pbuilder/inspector/suite.go new file mode 100644 index 00000000000..eda1d68089b --- /dev/null +++ b/network/p2p/p2pbuilder/inspector/suite.go @@ -0,0 +1,85 @@ +package inspector + +import ( + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/network/p2p" +) + +// GossipSubInspectorSuite encapsulates what is exposed to the libp2p node regarding the gossipsub RPC inspectors as +// well as their notification distributors. +type GossipSubInspectorSuite struct { + component.Component + aggregatedInspector *AggregateRPCInspector + ctrlMsgInspectDistributor p2p.GossipSubInspectorNotifDistributor +} + +var _ p2p.GossipSubInspectorSuite = (*GossipSubInspectorSuite)(nil) + +// NewGossipSubInspectorSuite creates a new GossipSubInspectorSuite. +// The suite is composed of the aggregated inspector, which is used to inspect the gossipsub rpc messages, and the +// control message notification distributor, which is used to notify consumers when a misbehaving peer regarding gossipsub +// control messages is detected. +// The suite is also a component, which is used to start and stop the rpc inspectors. +// Args: +// - inspectors: the rpc inspectors that are used to inspect the gossipsub rpc messages. +// - ctrlMsgInspectDistributor: the notification distributor that is used to notify consumers when a misbehaving peer +// +// regarding gossipsub control messages is detected. +// Returns: +// - the new GossipSubInspectorSuite. +func NewGossipSubInspectorSuite(inspectors []p2p.GossipSubRPCInspector, ctrlMsgInspectDistributor p2p.GossipSubInspectorNotifDistributor) *GossipSubInspectorSuite { + s := &GossipSubInspectorSuite{ + ctrlMsgInspectDistributor: ctrlMsgInspectDistributor, + aggregatedInspector: NewAggregateRPCInspector(inspectors...), + } + + builder := component.NewComponentManagerBuilder() + for _, inspector := range inspectors { + inspector := inspector // capture loop variable + builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + inspector.Start(ctx) + + select { + case <-ctx.Done(): + case <-inspector.Ready(): + ready() + } + + <-inspector.Done() + }) + } + + s.Component = builder.Build() + return s +} + +// InspectFunc returns the inspect function that is used to inspect the gossipsub rpc messages. +// This function follows a dependency injection pattern, where the inspect function is injected into the gossipsu, and +// is called whenever a gossipsub rpc message is received. +func (s *GossipSubInspectorSuite) InspectFunc() func(peer.ID, *pubsub.RPC) error { + return s.aggregatedInspector.Inspect +} + +// AddInvalidControlMessageConsumer adds a consumer to the invalid control message notification distributor. +// This consumer is notified when a misbehaving peer regarding gossipsub control messages is detected. This follows a pub/sub +// pattern where the consumer is notified when a new notification is published. +// A consumer is only notified once for each notification, and only receives notifications that were published after it was added. +func (s *GossipSubInspectorSuite) AddInvalidControlMessageConsumer(c p2p.GossipSubInvCtrlMsgNotifConsumer) { + s.ctrlMsgInspectDistributor.AddConsumer(c) +} + +// ActiveClustersChanged is called when the list of active collection nodes cluster is changed. +// GossipSubInspectorSuite consumes this event and forwards it to all the respective rpc inspectors, that are +// concerned with this cluster-based topics (i.e., channels), so that they can update their internal state. +func (s *GossipSubInspectorSuite) ActiveClustersChanged(list flow.ChainIDList) { + for _, rpcInspector := range s.aggregatedInspector.Inspectors() { + if r, ok := rpcInspector.(p2p.GossipSubMsgValidationRpcInspector); ok { + r.ActiveClustersChanged(list) + } + } +} diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index 156b990a9c5..ac3f30fd4d0 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -21,65 +21,29 @@ import ( madns "github.com/multiformats/go-multiaddr-dns" "github.com/rs/zerolog" + fcrypto "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + flownet "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/dht" + "github.com/onflow/flow-go/network/p2p/keyutils" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" + gossipsubbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/gossipsub" + "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/p2pnode" "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/tracer" + "github.com/onflow/flow-go/network/p2p/unicast" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/unicast/stream" "github.com/onflow/flow-go/network/p2p/utils" - - fcrypto "github.com/onflow/flow-go/crypto" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/component" - "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/network/p2p/keyutils" - gossipsubbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/gossipsub" - "github.com/onflow/flow-go/network/p2p/unicast" -) - -const ( - // defaultMemoryLimitRatio flow default - defaultMemoryLimitRatio = 0.2 - // defaultFileDescriptorsRatio libp2p default - defaultFileDescriptorsRatio = 0.5 - // defaultPeerBaseLimitConnsInbound default value for libp2p PeerBaseLimitConnsInbound. This limit - // restricts the amount of inbound connections from a peer to 1, forcing libp2p to reuse the connection. - // Without this limit peers can end up in a state where there exists n number of connections per peer which - // can lead to resource exhaustion of the libp2p node. - defaultPeerBaseLimitConnsInbound = 1 - - // defaultPeerScoringEnabled is the default value for enabling peer scoring. - defaultPeerScoringEnabled = true // enable peer scoring by default on node builder - - // defaultMeshTracerLoggingInterval is the default interval at which the mesh tracer logs the mesh - // topology. This is used for debugging and forensics purposes. - // Note that we purposefully choose this logging interval high enough to avoid spamming the logs. Moreover, the - // mesh updates will be logged individually and separately. The logging interval is only used to log the mesh - // topology as a whole specially when there are no updates to the mesh topology for a long time. - defaultMeshTracerLoggingInterval = 1 * time.Minute - - // defaultGossipSubScoreTracerInterval is the default interval at which the gossipsub score tracer logs the peer scores. - // This is used for debugging and forensics purposes. - // Note that we purposefully choose this logging interval high enough to avoid spamming the logs. - defaultGossipSubScoreTracerInterval = 1 * time.Minute ) -// DefaultGossipSubConfig returns the default configuration for the gossipsub protocol. -func DefaultGossipSubConfig() *GossipSubConfig { - return &GossipSubConfig{ - PeerScoring: defaultPeerScoringEnabled, - LocalMeshLogInterval: defaultMeshTracerLoggingInterval, - ScoreTracerInterval: defaultGossipSubScoreTracerInterval, - RPCInspectors: make([]p2p.GossipSubRPCInspector, 0), - } -} - -// LibP2PFactoryFunc is a factory function type for generating libp2p Node instances. -type LibP2PFactoryFunc func() (p2p.LibP2PNode, error) type GossipSubFactoryFunc func(context.Context, zerolog.Logger, host.Host, p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error) type CreateNodeFunc func(logger zerolog.Logger, host host.Host, @@ -87,114 +51,62 @@ type CreateNodeFunc func(logger zerolog.Logger, peerManager *connection.PeerManager) p2p.LibP2PNode type GossipSubAdapterConfigFunc func(*p2p.BasePubSubAdapterConfig) p2p.PubSubAdapterConfig -// DefaultLibP2PNodeFactory returns a LibP2PFactoryFunc which generates the libp2p host initialized with the -// default options for the host, the pubsub and the ping service. -func DefaultLibP2PNodeFactory(log zerolog.Logger, - address string, - flowKey fcrypto.PrivateKey, - sporkId flow.Identifier, - idProvider module.IdentityProvider, - metrics module.NetworkMetrics, - resolver madns.BasicResolver, - role string, - connGaterCfg *ConnectionGaterConfig, - peerManagerCfg *PeerManagerConfig, - gossipCfg *GossipSubConfig, - rCfg *ResourceManagerConfig, - uniCfg *UnicastConfig, -) p2p.LibP2PFactoryFunc { - return func() (p2p.LibP2PNode, error) { - builder, err := DefaultNodeBuilder(log, - address, - flowKey, - sporkId, - idProvider, - metrics, - resolver, - role, - connGaterCfg, - peerManagerCfg, - gossipCfg, - rCfg, - uniCfg) - - if err != nil { - return nil, fmt.Errorf("could not create node builder: %w", err) - } - - return builder.Build() - } -} - -// ResourceManagerConfig returns the resource manager configuration for the libp2p node. -// The resource manager is used to limit the number of open connections and streams (as well as any other resources -// used by libp2p) for each peer. -type ResourceManagerConfig struct { - MemoryLimitRatio float64 // maximum allowed fraction of memory to be allocated by the libp2p resources in (0,1] - FileDescriptorsRatio float64 // maximum allowed fraction of file descriptors to be allocated by the libp2p resources in (0,1] - PeerBaseLimitConnsInbound int // the maximum amount of allowed inbound connections per peer -} - -// GossipSubConfig is the configuration for the GossipSub pubsub implementation. -type GossipSubConfig struct { - // LocalMeshLogInterval is the interval at which the local mesh is logged. - LocalMeshLogInterval time.Duration - // ScoreTracerInterval is the interval at which the score tracer logs the peer scores. - ScoreTracerInterval time.Duration - // PeerScoring is whether to enable GossipSub peer scoring. - PeerScoring bool - // RPCInspectors gossipsub RPC control message inspectors - RPCInspectors []p2p.GossipSubRPCInspector -} - -func DefaultResourceManagerConfig() *ResourceManagerConfig { - return &ResourceManagerConfig{ - MemoryLimitRatio: defaultMemoryLimitRatio, - FileDescriptorsRatio: defaultFileDescriptorsRatio, - PeerBaseLimitConnsInbound: defaultPeerBaseLimitConnsInbound, - } -} - type LibP2PNodeBuilder struct { gossipSubBuilder p2p.GossipSubBuilder - sporkID flow.Identifier - addr string + sporkId flow.Identifier + address string networkKey fcrypto.PrivateKey logger zerolog.Logger metrics module.LibP2PMetrics basicResolver madns.BasicResolver resourceManager network.ResourceManager - resourceManagerCfg *ResourceManagerConfig + resourceManagerCfg *p2pconf.ResourceManagerConfig connManager connmgr.ConnManager - connGater connmgr.ConnectionGater + connGater p2p.ConnectionGater routingFactory func(context.Context, host.Host) (routing.Routing, error) - peerManagerEnablePruning bool - peerManagerUpdateInterval time.Duration + peerManagerConfig *p2pconfig.PeerManagerConfig createNode p2p.CreateNodeFunc createStreamRetryInterval time.Duration rateLimiterDistributor p2p.UnicastRateLimiterDistributor gossipSubTracer p2p.PubSubTracer + disallowListCacheCfg *p2p.DisallowListCacheConfig } -func NewNodeBuilder(logger zerolog.Logger, - metrics module.LibP2PMetrics, - addr string, +func NewNodeBuilder( + logger zerolog.Logger, + metricsConfig *p2pconfig.MetricsConfig, + networkingType flownet.NetworkingType, + address string, networkKey fcrypto.PrivateKey, - sporkID flow.Identifier, - rCfg *ResourceManagerConfig) *LibP2PNodeBuilder { + sporkId flow.Identifier, + idProvider module.IdentityProvider, + rCfg *p2pconf.ResourceManagerConfig, + rpcInspectorCfg *p2pconf.GossipSubRPCInspectorsConfig, + peerManagerConfig *p2pconfig.PeerManagerConfig, + disallowListCacheCfg *p2p.DisallowListCacheConfig) *LibP2PNodeBuilder { return &LibP2PNodeBuilder{ - logger: logger, - sporkID: sporkID, - addr: addr, - networkKey: networkKey, - createNode: DefaultCreateNodeFunc, - metrics: metrics, - resourceManagerCfg: rCfg, - gossipSubBuilder: gossipsubbuilder.NewGossipSubBuilder(logger, metrics), + logger: logger, + sporkId: sporkId, + address: address, + networkKey: networkKey, + createNode: DefaultCreateNodeFunc, + metrics: metricsConfig.Metrics, + resourceManagerCfg: rCfg, + disallowListCacheCfg: disallowListCacheCfg, + gossipSubBuilder: gossipsubbuilder.NewGossipSubBuilder( + logger, + metricsConfig, + networkingType, + sporkId, + idProvider, + rpcInspectorCfg), + peerManagerConfig: peerManagerConfig, } } +var _ p2p.NodeBuilder = &LibP2PNodeBuilder{} + // SetBasicResolver sets the DNS resolver for the node. func (builder *LibP2PNodeBuilder) SetBasicResolver(br madns.BasicResolver) p2p.NodeBuilder { builder.basicResolver = br @@ -220,7 +132,7 @@ func (builder *LibP2PNodeBuilder) SetConnectionManager(manager connmgr.ConnManag } // SetConnectionGater sets the connection gater for the node. -func (builder *LibP2PNodeBuilder) SetConnectionGater(gater connmgr.ConnectionGater) p2p.NodeBuilder { +func (builder *LibP2PNodeBuilder) SetConnectionGater(gater p2p.ConnectionGater) p2p.NodeBuilder { builder.connGater = gater return builder } @@ -237,31 +149,17 @@ func (builder *LibP2PNodeBuilder) SetGossipSubFactory(gf p2p.GossipSubFactoryFun return builder } -// EnableGossipSubPeerScoring enables peer scoring for the GossipSub pubsub system. -// Arguments: -// - module.IdentityProvider: the identity provider for the node (must be set before calling this method). -// - *PeerScoringConfig: the peer scoring configuration for the GossipSub pubsub system. If nil, the default configuration is used. -func (builder *LibP2PNodeBuilder) EnableGossipSubPeerScoring(provider module.IdentityProvider, config *p2p.PeerScoringConfig) p2p.NodeBuilder { - builder.gossipSubBuilder.SetGossipSubPeerScoring(true) - builder.gossipSubBuilder.SetIDProvider(provider) - if config != nil { - if config.AppSpecificScoreParams != nil { - builder.gossipSubBuilder.SetAppSpecificScoreParams(config.AppSpecificScoreParams) - } - if config.TopicScoreParams != nil { - for topic, params := range config.TopicScoreParams { - builder.gossipSubBuilder.SetTopicScoreParams(topic, params) - } - } - } - - return builder -} - -// SetPeerManagerOptions sets the peer manager options. -func (builder *LibP2PNodeBuilder) SetPeerManagerOptions(connectionPruning bool, updateInterval time.Duration) p2p.NodeBuilder { - builder.peerManagerEnablePruning = connectionPruning - builder.peerManagerUpdateInterval = updateInterval +// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. +// Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. +// Anything that is left to nil or zero value in the override will be ignored and the default value will be used. +// Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. +// Production Tip: use PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. +// Args: +// - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use PeerScoringConfigNoOverride for production. +// Returns: +// none +func (builder *LibP2PNodeBuilder) EnableGossipSubScoringWithOverride(config *p2p.PeerScoringConfigOverride) p2p.NodeBuilder { + builder.gossipSubBuilder.EnableGossipSubScoringWithOverride(config) return builder } @@ -291,8 +189,8 @@ func (builder *LibP2PNodeBuilder) SetGossipSubScoreTracerInterval(interval time. return builder } -func (builder *LibP2PNodeBuilder) SetGossipSubRPCInspectors(inspectors ...p2p.GossipSubRPCInspector) p2p.NodeBuilder { - builder.gossipSubBuilder.SetGossipSubRPCInspectors(inspectors...) +func (builder *LibP2PNodeBuilder) OverrideDefaultRpcInspectorSuiteFactory(factory p2p.GossipSubRpcInspectorSuiteFactoryFunc) p2p.NodeBuilder { + builder.gossipSubBuilder.OverrideDefaultRpcInspectorSuiteFactory(factory) return builder } @@ -341,7 +239,6 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { } else { // setting up default resource manager, by hooking in the resource manager metrics reporter. limits := rcmgr.DefaultLimits - libp2p.SetDefaultServiceLimits(&limits) mem, err := allowedMemory(builder.resourceManagerCfg.MemoryLimitRatio) @@ -377,7 +274,7 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { opts = append(opts, libp2p.ConnectionGater(builder.connGater)) } - h, err := DefaultLibP2PHost(builder.addr, builder.networkKey, opts...) + h, err := DefaultLibP2PHost(builder.address, builder.networkKey, opts...) if err != nil { return nil, err } @@ -389,24 +286,37 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { } var peerManager p2p.PeerManager - if builder.peerManagerUpdateInterval > 0 { - connector, err := connection.NewLibp2pConnector(builder.logger, h, builder.peerManagerEnablePruning) + if builder.peerManagerConfig.UpdateInterval > 0 { + connector, err := builder.peerManagerConfig.ConnectorFactory(h) + if err != nil { + return nil, fmt.Errorf("failed to create libp2p connector: %w", err) + } + peerUpdater, err := connection.NewPeerUpdater(&connection.PeerUpdaterConfig{ + PruneConnections: builder.peerManagerConfig.ConnectionPruning, + Logger: builder.logger, + Host: connection.NewConnectorHost(h), + Connector: connector, + }) if err != nil { return nil, fmt.Errorf("failed to create libp2p connector: %w", err) } - peerManager = connection.NewPeerManager(builder.logger, builder.peerManagerUpdateInterval, connector) + peerManager = connection.NewPeerManager(builder.logger, builder.peerManagerConfig.UpdateInterval, peerUpdater) if builder.rateLimiterDistributor != nil { builder.rateLimiterDistributor.AddConsumer(peerManager) } } - node := builder.createNode(builder.logger, h, pCache, peerManager) + node := builder.createNode(builder.logger, h, pCache, peerManager, builder.disallowListCacheCfg) + + if builder.connGater != nil { + builder.connGater.SetDisallowListOracle(node) + } unicastManager := unicast.NewUnicastManager(builder.logger, stream.NewLibP2PStreamFactory(h), - builder.sporkID, + builder.sporkId, builder.createStreamRetryInterval, node, builder.metrics) @@ -423,13 +333,10 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { builder.gossipSubBuilder.SetRoutingSystem(routingSystem) // gossipsub is created here, because it needs to be created during the node startup. - gossipSub, scoreTracer, err := builder.gossipSubBuilder.Build(ctx) + gossipSub, err := builder.gossipSubBuilder.Build(ctx) if err != nil { ctx.Throw(fmt.Errorf("could not create gossipsub: %w", err)) } - if scoreTracer != nil { - node.SetPeerScoreExposer(scoreTracer) - } node.SetPubSub(gossipSub) gossipSub.Start(ctx) ready() @@ -516,26 +423,32 @@ func defaultLibP2POptions(address string, key fcrypto.PrivateKey) ([]config.Opti func DefaultCreateNodeFunc(logger zerolog.Logger, host host.Host, pCache p2p.ProtocolPeerCache, - peerManager p2p.PeerManager) p2p.LibP2PNode { - return p2pnode.NewNode(logger, host, pCache, peerManager) + peerManager p2p.PeerManager, + disallowListCacheCfg *p2p.DisallowListCacheConfig) p2p.LibP2PNode { + return p2pnode.NewNode(logger, host, pCache, peerManager, disallowListCacheCfg) } // DefaultNodeBuilder returns a node builder. -func DefaultNodeBuilder(log zerolog.Logger, +func DefaultNodeBuilder( + logger zerolog.Logger, address string, + networkingType flownet.NetworkingType, flowKey fcrypto.PrivateKey, sporkId flow.Identifier, idProvider module.IdentityProvider, - metrics module.LibP2PMetrics, + metricsCfg *p2pconfig.MetricsConfig, resolver madns.BasicResolver, role string, - connGaterCfg *ConnectionGaterConfig, - peerManagerCfg *PeerManagerConfig, - gossipCfg *GossipSubConfig, - rCfg *ResourceManagerConfig, - uniCfg *UnicastConfig) (p2p.NodeBuilder, error) { - - connManager, err := connection.NewConnManager(log, metrics, connection.DefaultConnManagerConfig()) + connGaterCfg *p2pconfig.ConnectionGaterConfig, + peerManagerCfg *p2pconfig.PeerManagerConfig, + gossipCfg *p2pconf.GossipSubConfig, + rpcInspectorCfg *p2pconf.GossipSubRPCInspectorsConfig, + rCfg *p2pconf.ResourceManagerConfig, + uniCfg *p2pconfig.UnicastConfig, + connMgrConfig *netconf.ConnectionManagerConfig, + disallowListCacheCfg *p2p.DisallowListCacheConfig) (p2p.NodeBuilder, error) { + + connManager, err := connection.NewConnManager(logger, metricsCfg.Metrics, connMgrConfig) if err != nil { return nil, fmt.Errorf("could not create connection manager: %w", err) } @@ -544,30 +457,51 @@ func DefaultNodeBuilder(log zerolog.Logger, peerFilter := notEjectedPeerFilter(idProvider) peerFilters := []p2p.PeerFilter{peerFilter} - connGater := connection.NewConnGater(log, + connGater := connection.NewConnGater(logger, idProvider, connection.WithOnInterceptPeerDialFilters(append(peerFilters, connGaterCfg.InterceptPeerDialFilters...)), connection.WithOnInterceptSecuredFilters(append(peerFilters, connGaterCfg.InterceptSecuredFilters...))) - builder := NewNodeBuilder(log, metrics, address, flowKey, sporkId, rCfg). + builder := NewNodeBuilder( + logger, + metricsCfg, + networkingType, + address, + flowKey, + sporkId, + idProvider, + rCfg, + rpcInspectorCfg, + peerManagerCfg, + disallowListCacheCfg). SetBasicResolver(resolver). SetConnectionManager(connManager). SetConnectionGater(connGater). SetRoutingSystem(func(ctx context.Context, host host.Host) (routing.Routing, error) { - return dht.NewDHT(ctx, host, protocols.FlowDHTProtocolID(sporkId), log, metrics, dht.AsServer()) + return dht.NewDHT(ctx, host, protocols.FlowDHTProtocolID(sporkId), logger, metricsCfg.Metrics, dht.AsServer()) }). - SetPeerManagerOptions(peerManagerCfg.ConnectionPruning, peerManagerCfg.UpdateInterval). SetStreamCreationRetryInterval(uniCfg.StreamRetryInterval). SetCreateNode(DefaultCreateNodeFunc). - SetRateLimiterDistributor(uniCfg.RateLimiterDistributor). - SetGossipSubRPCInspectors(gossipCfg.RPCInspectors...) + SetRateLimiterDistributor(uniCfg.RateLimiterDistributor) if gossipCfg.PeerScoring { - // currently, we only enable peer scoring with default parameters. So, we set the score parameters to nil. - builder.EnableGossipSubPeerScoring(idProvider, nil) + // In production, we never override the default scoring config. + builder.EnableGossipSubScoringWithOverride(p2p.PeerScoringConfigNoOverride) + } + + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: logger, + Metrics: metricsCfg.Metrics, + IDProvider: idProvider, + LoggerInterval: gossipCfg.LocalMeshLogInterval, + RpcSentTrackerCacheSize: gossipCfg.RPCSentTrackerCacheSize, + RpcSentTrackerWorkerQueueCacheSize: gossipCfg.RPCSentTrackerQueueCacheSize, + RpcSentTrackerNumOfWorkers: gossipCfg.RpcSentTrackerNumOfWorkers, + HeroCacheMetricsFactory: metricsCfg.HeroCacheFactory, + NetworkingType: flownet.PrivateNetwork, } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) - meshTracer := tracer.NewGossipSubMeshTracer(log, metrics, idProvider, gossipCfg.LocalMeshLogInterval) builder.SetGossipSubTracer(meshTracer) builder.SetGossipSubScoreTracerInterval(gossipCfg.ScoreTracerInterval) diff --git a/network/p2p/p2pbuilder/utils.go b/network/p2p/p2pbuilder/utils.go index 29b4d143698..066ddfdbff5 100644 --- a/network/p2p/p2pbuilder/utils.go +++ b/network/p2p/p2pbuilder/utils.go @@ -37,28 +37,30 @@ func newLimitConfigLogger(logger zerolog.Logger) *limitConfigLogger { } // withBaseLimit appends the base limit to the logger with the given prefix. -func (l *limitConfigLogger) withBaseLimit(prefix string, baseLimit rcmgr.BaseLimit) zerolog.Logger { +func (l *limitConfigLogger) withBaseLimit(prefix string, baseLimit rcmgr.ResourceLimits) zerolog.Logger { return l.logger.With(). Str("key", keyResourceManagerLimit). - Int(fmt.Sprintf("%s_streams", prefix), baseLimit.Streams). - Int(fmt.Sprintf("%s_streams_inbound", prefix), baseLimit.StreamsInbound). - Int(fmt.Sprintf("%s_streams_outbound", prefix), baseLimit.StreamsOutbound). - Int(fmt.Sprintf("%s_conns", prefix), baseLimit.Conns). - Int(fmt.Sprintf("%s_conns_inbound", prefix), baseLimit.ConnsInbound). - Int(fmt.Sprintf("%s_conns_outbound", prefix), baseLimit.ConnsOutbound). - Int(fmt.Sprintf("%s_file_descriptors", prefix), baseLimit.FD). - Int64(fmt.Sprintf("%s_memory", prefix), baseLimit.Memory).Logger() + Str(fmt.Sprintf("%s_streams", prefix), fmt.Sprintf("%v", baseLimit.Streams)). + Str(fmt.Sprintf("%s_streams_inbound", prefix), fmt.Sprintf("%v", baseLimit.StreamsInbound)). + Str(fmt.Sprintf("%s_streams_outbound", prefix), fmt.Sprintf("%v,", baseLimit.StreamsOutbound)). + Str(fmt.Sprintf("%s_conns", prefix), fmt.Sprintf("%v", baseLimit.Conns)). + Str(fmt.Sprintf("%s_conns_inbound", prefix), fmt.Sprintf("%v", baseLimit.ConnsInbound)). + Str(fmt.Sprintf("%s_conns_outbound", prefix), fmt.Sprintf("%v", baseLimit.ConnsOutbound)). + Str(fmt.Sprintf("%s_file_descriptors", prefix), fmt.Sprintf("%v", baseLimit.FD)). + Str(fmt.Sprintf("%s_memory", prefix), fmt.Sprintf("%v", baseLimit.Memory)).Logger() } -func (l *limitConfigLogger) logResourceManagerLimits(config rcmgr.LimitConfig) { - l.logGlobalResourceLimits(config) - l.logServiceLimits(config.Service) - l.logProtocolLimits(config.Protocol) - l.logPeerLimits(config.Peer) - l.logPeerProtocolLimits(config.ProtocolPeer) +func (l *limitConfigLogger) logResourceManagerLimits(config rcmgr.ConcreteLimitConfig) { + // PartialLimit config is the same as ConcreteLimit config, but with the exported fields. + pCfg := config.ToPartialLimitConfig() + l.logGlobalResourceLimits(pCfg) + l.logServiceLimits(pCfg.Service) + l.logProtocolLimits(pCfg.Protocol) + l.logPeerLimits(pCfg.Peer) + l.logPeerProtocolLimits(pCfg.ProtocolPeer) } -func (l *limitConfigLogger) logGlobalResourceLimits(config rcmgr.LimitConfig) { +func (l *limitConfigLogger) logGlobalResourceLimits(config rcmgr.PartialLimitConfig) { lg := l.withBaseLimit("system", config.System) lg.Info().Msg("system limits set") @@ -93,28 +95,28 @@ func (l *limitConfigLogger) logGlobalResourceLimits(config rcmgr.LimitConfig) { lg.Info().Msg("stream limits set") } -func (l *limitConfigLogger) logServiceLimits(s map[string]rcmgr.BaseLimit) { +func (l *limitConfigLogger) logServiceLimits(s map[string]rcmgr.ResourceLimits) { for sName, sLimits := range s { lg := l.withBaseLimit(fmt.Sprintf("service_%s", sName), sLimits) lg.Info().Msg("service limits set") } } -func (l *limitConfigLogger) logProtocolLimits(p map[protocol.ID]rcmgr.BaseLimit) { +func (l *limitConfigLogger) logProtocolLimits(p map[protocol.ID]rcmgr.ResourceLimits) { for pName, pLimits := range p { lg := l.withBaseLimit(fmt.Sprintf("protocol_%s", pName), pLimits) lg.Info().Msg("protocol limits set") } } -func (l *limitConfigLogger) logPeerLimits(p map[peer.ID]rcmgr.BaseLimit) { +func (l *limitConfigLogger) logPeerLimits(p map[peer.ID]rcmgr.ResourceLimits) { for pId, pLimits := range p { lg := l.withBaseLimit(fmt.Sprintf("peer_%s", pId.String()), pLimits) lg.Info().Msg("peer limits set") } } -func (l *limitConfigLogger) logPeerProtocolLimits(p map[protocol.ID]rcmgr.BaseLimit) { +func (l *limitConfigLogger) logPeerProtocolLimits(p map[protocol.ID]rcmgr.ResourceLimits) { for pName, pLimits := range p { lg := l.withBaseLimit(fmt.Sprintf("protocol_peer_%s", pName), pLimits) lg.Info().Msg("protocol peer limits set") diff --git a/network/p2p/p2pconf/errors.go b/network/p2p/p2pconf/errors.go new file mode 100644 index 00000000000..88417ced914 --- /dev/null +++ b/network/p2p/p2pconf/errors.go @@ -0,0 +1,32 @@ +package p2pconf + +import ( + "errors" + "fmt" + + p2pmsg "github.com/onflow/flow-go/network/p2p/message" +) + +// InvalidLimitConfigError indicates the validation limit is < 0. +type InvalidLimitConfigError struct { + err error +} + +func (e InvalidLimitConfigError) Error() string { + return e.err.Error() +} + +func (e InvalidLimitConfigError) Unwrap() error { + return e.err +} + +// NewInvalidLimitConfigErr returns a new ErrValidationLimit. +func NewInvalidLimitConfigErr(controlMsg p2pmsg.ControlMessageType, err error) InvalidLimitConfigError { + return InvalidLimitConfigError{fmt.Errorf("invalid rpc control message %s validation limit configuration: %w", controlMsg, err)} +} + +// IsInvalidLimitConfigError returns whether an error is ErrInvalidLimitConfig. +func IsInvalidLimitConfigError(err error) bool { + var e InvalidLimitConfigError + return errors.As(err, &e) +} diff --git a/network/p2p/p2pconf/errors_test.go b/network/p2p/p2pconf/errors_test.go new file mode 100644 index 00000000000..681d839a5fa --- /dev/null +++ b/network/p2p/p2pconf/errors_test.go @@ -0,0 +1,30 @@ +package p2pconf + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + p2pmsg "github.com/onflow/flow-go/network/p2p/message" +) + +// TestErrInvalidLimitConfigRoundTrip ensures correct error formatting for ErrInvalidLimitConfig. +func TestErrInvalidLimitConfigRoundTrip(t *testing.T) { + controlMsg := p2pmsg.CtrlMsgGraft + limit := uint64(500) + + e := fmt.Errorf("invalid rate limit value %d must be greater than 0", limit) + err := NewInvalidLimitConfigErr(controlMsg, e) + + // tests the error message formatting. + expectedErrMsg := fmt.Errorf("invalid rpc control message %s validation limit configuration: %w", controlMsg, e).Error() + assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") + + // tests the IsInvalidLimitConfigError function. + assert.True(t, IsInvalidLimitConfigError(err), "IsInvalidLimitConfigError should return true for ErrInvalidLimitConfig error") + + // test IsInvalidLimitConfigError with a different error type. + dummyErr := fmt.Errorf("dummy error") + assert.False(t, IsInvalidLimitConfigError(dummyErr), "IsInvalidLimitConfigError should return false for non-ErrInvalidLimitConfig error") +} diff --git a/network/p2p/p2pconf/gossipsub.go b/network/p2p/p2pconf/gossipsub.go new file mode 100644 index 00000000000..683dff67fdc --- /dev/null +++ b/network/p2p/p2pconf/gossipsub.go @@ -0,0 +1,40 @@ +package p2pconf + +import ( + "time" +) + +// ResourceManagerConfig returns the resource manager configuration for the libp2p node. +// The resource manager is used to limit the number of open connections and streams (as well as any other resources +// used by libp2p) for each peer. +type ResourceManagerConfig struct { + MemoryLimitRatio float64 `mapstructure:"libp2p-memory-limit-ratio"` // maximum allowed fraction of memory to be allocated by the libp2p resources in (0,1] + FileDescriptorsRatio float64 `mapstructure:"libp2p-file-descriptors-ratio"` // maximum allowed fraction of file descriptors to be allocated by the libp2p resources in (0,1] + PeerBaseLimitConnsInbound int `mapstructure:"libp2p-peer-base-limits-conns-inbound"` // the maximum amount of allowed inbound connections per peer +} + +// GossipSubConfig is the configuration for the GossipSub pubsub implementation. +type GossipSubConfig struct { + // GossipSubRPCInspectorsConfig configuration for all gossipsub RPC control message inspectors. + GossipSubRPCInspectorsConfig `mapstructure:",squash"` + + // GossipSubTracerConfig is the configuration for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. + GossipSubTracerConfig `mapstructure:",squash"` + + // PeerScoring is whether to enable GossipSub peer scoring. + PeerScoring bool `mapstructure:"gossipsub-peer-scoring-enabled"` +} + +// GossipSubTracerConfig is the config for the gossipsub tracer. GossipSub tracer is used to trace the local mesh events and peer scores. +type GossipSubTracerConfig struct { + // LocalMeshLogInterval is the interval at which the local mesh is logged. + LocalMeshLogInterval time.Duration `validate:"gt=0s" mapstructure:"gossipsub-local-mesh-logging-interval"` + // ScoreTracerInterval is the interval at which the score tracer logs the peer scores. + ScoreTracerInterval time.Duration `validate:"gt=0s" mapstructure:"gossipsub-score-tracer-interval"` + // RPCSentTrackerCacheSize cache size of the rpc sent tracker used by the gossipsub mesh tracer. + RPCSentTrackerCacheSize uint32 `validate:"gt=0" mapstructure:"gossipsub-rpc-sent-tracker-cache-size"` + // RPCSentTrackerQueueCacheSize cache size of the rpc sent tracker queue used for async tracking. + RPCSentTrackerQueueCacheSize uint32 `validate:"gt=0" mapstructure:"gossipsub-rpc-sent-tracker-queue-cache-size"` + // RpcSentTrackerNumOfWorkers number of workers for rpc sent tracker worker pool. + RpcSentTrackerNumOfWorkers int `validate:"gt=0" mapstructure:"gossipsub-rpc-sent-tracker-workers"` +} diff --git a/network/p2p/p2pconf/gossipsub_rpc_inspectors.go b/network/p2p/p2pconf/gossipsub_rpc_inspectors.go new file mode 100644 index 00000000000..d4f2bd6ccc1 --- /dev/null +++ b/network/p2p/p2pconf/gossipsub_rpc_inspectors.go @@ -0,0 +1,138 @@ +package p2pconf + +import ( + p2pmsg "github.com/onflow/flow-go/network/p2p/message" +) + +// GossipSubRPCInspectorsConfig encompasses configuration related to gossipsub RPC message inspectors. +type GossipSubRPCInspectorsConfig struct { + // GossipSubRPCValidationInspectorConfigs control message validation inspector validation configuration and limits. + GossipSubRPCValidationInspectorConfigs `mapstructure:",squash"` + // GossipSubRPCMetricsInspectorConfigs control message metrics inspector configuration. + GossipSubRPCMetricsInspectorConfigs `mapstructure:",squash"` + // GossipSubRPCInspectorNotificationCacheSize size of the queue for notifications about invalid RPC messages. + GossipSubRPCInspectorNotificationCacheSize uint32 `mapstructure:"gossipsub-rpc-inspector-notification-cache-size"` +} + +// GossipSubRPCValidationInspectorConfigs validation limits used for gossipsub RPC control message inspection. +type GossipSubRPCValidationInspectorConfigs struct { + ClusterPrefixedMessageConfig `mapstructure:",squash"` + // NumberOfWorkers number of worker pool workers. + NumberOfWorkers int `validate:"gte=1" mapstructure:"gossipsub-rpc-validation-inspector-workers"` + // CacheSize size of the queue used by worker pool for the control message validation inspector. + CacheSize uint32 `validate:"gte=100" mapstructure:"gossipsub-rpc-validation-inspector-queue-cache-size"` + // GraftLimits GRAFT control message validation limits. + GraftLimits struct { + HardThreshold uint64 `validate:"gt=0" mapstructure:"gossipsub-rpc-graft-hard-threshold"` + SafetyThreshold uint64 `validate:"gt=0" mapstructure:"gossipsub-rpc-graft-safety-threshold"` + RateLimit int `validate:"gte=0" mapstructure:"gossipsub-rpc-graft-rate-limit"` + } `mapstructure:",squash"` + // PruneLimits PRUNE control message validation limits. + PruneLimits struct { + HardThreshold uint64 `validate:"gt=0" mapstructure:"gossipsub-rpc-prune-hard-threshold"` + SafetyThreshold uint64 `validate:"gt=0" mapstructure:"gossipsub-rpc-prune-safety-threshold"` + RateLimit int `validate:"gte=0" mapstructure:"gossipsub-rpc-prune-rate-limit"` + } `mapstructure:",squash"` + // IHaveLimits IHAVE control message validation limits. + IHaveLimits struct { + HardThreshold uint64 `validate:"gt=0" mapstructure:"gossipsub-rpc-ihave-hard-threshold"` + SafetyThreshold uint64 `validate:"gt=0" mapstructure:"gossipsub-rpc-ihave-safety-threshold"` + RateLimit int `validate:"gte=0" mapstructure:"gossipsub-rpc-ihave-rate-limit"` + } `mapstructure:",squash"` + // IHaveSyncInspectSampleSizePercentage the percentage of topics to sample for sync pre-processing in float64 form. + IHaveSyncInspectSampleSizePercentage float64 `validate:"gte=.25" mapstructure:"ihave-sync-inspection-sample-size-percentage"` + // IHaveAsyncInspectSampleSizePercentage the percentage of topics to sample for async pre-processing in float64 form. + IHaveAsyncInspectSampleSizePercentage float64 `validate:"gte=.10" mapstructure:"ihave-async-inspection-sample-size-percentage"` + // IHaveInspectionMaxSampleSize the max number of ihave messages in a sample to be inspected. + IHaveInspectionMaxSampleSize float64 `validate:"gte=100" mapstructure:"ihave-max-sample-size"` +} + +// GetCtrlMsgValidationConfig returns the CtrlMsgValidationConfig for the specified p2p.ControlMessageType. +func (conf *GossipSubRPCValidationInspectorConfigs) GetCtrlMsgValidationConfig(controlMsg p2pmsg.ControlMessageType) (*CtrlMsgValidationConfig, bool) { + switch controlMsg { + case p2pmsg.CtrlMsgGraft: + return &CtrlMsgValidationConfig{ + ControlMsg: p2pmsg.CtrlMsgGraft, + HardThreshold: conf.GraftLimits.HardThreshold, + SafetyThreshold: conf.GraftLimits.SafetyThreshold, + RateLimit: conf.GraftLimits.RateLimit, + }, true + case p2pmsg.CtrlMsgPrune: + return &CtrlMsgValidationConfig{ + ControlMsg: p2pmsg.CtrlMsgPrune, + HardThreshold: conf.PruneLimits.HardThreshold, + SafetyThreshold: conf.PruneLimits.SafetyThreshold, + RateLimit: conf.PruneLimits.RateLimit, + }, true + case p2pmsg.CtrlMsgIHave: + return &CtrlMsgValidationConfig{ + ControlMsg: p2pmsg.CtrlMsgIHave, + HardThreshold: conf.IHaveLimits.HardThreshold, + SafetyThreshold: conf.IHaveLimits.SafetyThreshold, + RateLimit: conf.IHaveLimits.RateLimit, + }, true + default: + return nil, false + } +} + +// AllCtrlMsgValidationConfig returns all control message validation configs in a list. +func (conf *GossipSubRPCValidationInspectorConfigs) AllCtrlMsgValidationConfig() CtrlMsgValidationConfigs { + return CtrlMsgValidationConfigs{&CtrlMsgValidationConfig{ + ControlMsg: p2pmsg.CtrlMsgGraft, + HardThreshold: conf.GraftLimits.HardThreshold, + SafetyThreshold: conf.GraftLimits.SafetyThreshold, + RateLimit: conf.GraftLimits.RateLimit, + }, &CtrlMsgValidationConfig{ + ControlMsg: p2pmsg.CtrlMsgPrune, + HardThreshold: conf.PruneLimits.HardThreshold, + SafetyThreshold: conf.PruneLimits.SafetyThreshold, + RateLimit: conf.PruneLimits.RateLimit, + }, &CtrlMsgValidationConfig{ + ControlMsg: p2pmsg.CtrlMsgIHave, + HardThreshold: conf.IHaveLimits.HardThreshold, + SafetyThreshold: conf.IHaveLimits.SafetyThreshold, + RateLimit: conf.IHaveLimits.RateLimit, + }} +} + +// CtrlMsgValidationConfigs list of *CtrlMsgValidationConfig +type CtrlMsgValidationConfigs []*CtrlMsgValidationConfig + +// CtrlMsgValidationConfig configuration values for upper, lower threshold and rate limit. +type CtrlMsgValidationConfig struct { + // ControlMsg the type of RPC control message. + ControlMsg p2pmsg.ControlMessageType + // HardThreshold specifies the hard limit for the size of an RPC control message. + // While it is generally expected that RPC messages with a size greater than HardThreshold should be dropped, + // there are exceptions. For instance, if the message is an 'iHave', blocking processing is performed + // on a sample of the control message rather than dropping it. + HardThreshold uint64 `mapstructure:"hard-threshold"` + // SafetyThreshold specifies the lower limit for the size of the RPC control message, it is safe to skip validation for any RPC messages + // with a size < SafetyThreshold. These messages will be processed as soon as possible. + SafetyThreshold uint64 `mapstructure:"safety-threshold"` + // RateLimit number of allowed messages per second, use 0 to disable rate limiting. + RateLimit int `mapstructure:"rate-limit"` +} + +// ClusterPrefixedMessageConfig configuration values for cluster prefixed control message validation. +type ClusterPrefixedMessageConfig struct { + // ClusterPrefixHardThreshold the upper bound on the amount of cluster prefixed control messages that will be processed + // before a node starts to get penalized. This allows LN nodes to process some cluster prefixed control messages during startup + // when the cluster ID's provider is set asynchronously. It also allows processing of some stale messages that may be sent by nodes + // that fall behind in the protocol. After the amount of cluster prefixed control messages processed exceeds this threshold the node + // will be pushed to the edge of the network mesh. + ClusterPrefixHardThreshold float64 `validate:"gt=0" mapstructure:"gossipsub-rpc-cluster-prefixed-hard-threshold"` + // ClusterPrefixedControlMsgsReceivedCacheSize size of the cache used to track the amount of cluster prefixed topics received by peers. + ClusterPrefixedControlMsgsReceivedCacheSize uint32 `validate:"gt=0" mapstructure:"gossipsub-cluster-prefix-tracker-cache-size"` + // ClusterPrefixedControlMsgsReceivedCacheDecay decay val used for the geometric decay of cache counters used to keep track of cluster prefixed topics received by peers. + ClusterPrefixedControlMsgsReceivedCacheDecay float64 `validate:"gt=0" mapstructure:"gossipsub-cluster-prefix-tracker-cache-decay"` +} + +// GossipSubRPCMetricsInspectorConfigs rpc metrics observer inspector configuration. +type GossipSubRPCMetricsInspectorConfigs struct { + // NumberOfWorkers number of worker pool workers. + NumberOfWorkers int `validate:"gte=1" mapstructure:"gossipsub-rpc-metrics-inspector-workers"` + // CacheSize size of the queue used by worker pool for the control message metrics inspector. + CacheSize uint32 `validate:"gt=0" mapstructure:"gossipsub-rpc-metrics-inspector-cache-size"` +} diff --git a/network/p2p/p2pnode/disallow_listing_test.go b/network/p2p/p2pnode/disallow_listing_test.go new file mode 100644 index 00000000000..566b2d6ec5e --- /dev/null +++ b/network/p2p/p2pnode/disallow_listing_test.go @@ -0,0 +1,100 @@ +package p2pnode_test + +import ( + "context" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + mockmodule "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/connection" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestDisconnectingFromDisallowListedNode ensures that: +// (1) the node disconnects from a disallow listed node while the node is connected to other (allow listed) nodes. +// (2) new inbound or outbound connections to and from disallow-listed nodes are rejected. +// (3) When a disallow-listed node is allow-listed again, the node reconnects to it. +func TestDisconnectingFromDisallowListedNode(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + sporkID := unittest.IdentifierFixture() + idProvider := mockmodule.NewIdentityProvider(t) + + peerIDSlice := peer.IDSlice{} + // node 1 is the node that will be disallow-listing another node (node 2). + node1, identity1 := p2ptest.NodeFixture(t, + sporkID, + t.Name(), + idProvider, + p2ptest.WithPeerManagerEnabled(&p2pconfig.PeerManagerConfig{ + ConnectionPruning: true, + UpdateInterval: connection.DefaultPeerUpdateInterval, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + }, + func() peer.IDSlice { + return peerIDSlice + }), + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(p peer.ID) error { + // allow all the connections, except for the ones that are disallow-listed, which are determined when + // this connection gater object queries the disallow listing oracle that will be provided to it by + // the libp2p node. So, here, we don't need to do anything except just enabling the connection gater. + return nil + }))) + idProvider.On("ByPeerID", node1.Host().ID()).Return(&identity1, true).Maybe() + peerIDSlice = append(peerIDSlice, node1.Host().ID()) + + // node 2 is the node that will be disallow-listed by node 1. + node2, identity2 := p2ptest.NodeFixture(t, sporkID, t.Name(), idProvider) + idProvider.On("ByPeerID", node2.Host().ID()).Return(&identity2, true).Maybe() + peerIDSlice = append(peerIDSlice, node2.Host().ID()) + + // node 3 is the node that will be connected to node 1 (to ensure that node 1 is still able to connect to other nodes + // after disallow-listing node 2). + node3, identity3 := p2ptest.NodeFixture(t, sporkID, t.Name(), idProvider) + idProvider.On("ByPeerID", node3.Host().ID()).Return(&identity3, true).Maybe() + peerIDSlice = append(peerIDSlice, node3.Host().ID()) + + nodes := []p2p.LibP2PNode{node1, node2, node3} + ids := flow.IdentityList{&identity1, &identity2, &identity3} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + + // initially all nodes should be connected to each other. + p2ptest.RequireConnectedEventually(t, nodes, 100*time.Millisecond, 2*time.Second) + + // phase-1: node 1 disallow-lists node 2. + node1.OnDisallowListNotification(node2.Host().ID(), network.DisallowListedCauseAlsp) + + // eventually node 1 should be disconnected from node 2 while other nodes should remain connected. + // we choose a timeout of 2 seconds because peer manager updates peers every 1 second. + p2ptest.RequireEventuallyNotConnected(t, []p2p.LibP2PNode{node1}, []p2p.LibP2PNode{node2}, 100*time.Millisecond, 2*time.Second) + + // but nodes 1 and 3 should remain connected as well as nodes 2 and 3. + // we choose a short timeout because we expect the nodes to remain connected. + p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{node1, node3}, 1*time.Millisecond, 100*time.Millisecond) + p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{node2, node3}, 1*time.Millisecond, 100*time.Millisecond) + + // while node 2 is disallow-listed, it cannot connect to node 1. Also, node 1 cannot directly dial and connect to node 2, unless + // it is allow-listed again. + p2ptest.EnsureNotConnectedBetweenGroups(t, ctx, []p2p.LibP2PNode{node1}, []p2p.LibP2PNode{node2}) + + // phase-2: now we allow-list node 1 back + node1.OnAllowListNotification(node2.Host().ID(), network.DisallowListedCauseAlsp) + + // eventually node 1 should be connected to node 2 again, hence all nodes should be connected to each other. + // we choose a timeout of 5 seconds because peer manager updates peers every 1 second and we need to wait for + // any potential random backoffs to expire (min 1 second). + p2ptest.RequireConnectedEventually(t, nodes, 100*time.Millisecond, 5*time.Second) +} diff --git a/network/p2p/p2pnode/gossipSubAdapter.go b/network/p2p/p2pnode/gossipSubAdapter.go index 9fd1b148ab8..59bd2f2d65a 100644 --- a/network/p2p/p2pnode/gossipSubAdapter.go +++ b/network/p2p/p2pnode/gossipSubAdapter.go @@ -9,9 +9,11 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/utils/logging" ) @@ -20,12 +22,24 @@ import ( type GossipSubAdapter struct { component.Component gossipSub *pubsub.PubSub - logger zerolog.Logger + // topicScoreParamFunc is a function that returns the topic score params for a given topic. + // If no function is provided the node will join the topic with no scoring params. As the + // node will not be able to score other peers in the topic, it may be vulnerable to routing + // attacks on the topic that may also affect the overall function of the node. + // It is not recommended to use this adapter without a topicScoreParamFunc. Also in mature + // implementations of the Flow network, the topicScoreParamFunc must be a required parameter. + topicScoreParamFunc func(topic *pubsub.Topic) *pubsub.TopicScoreParams + logger zerolog.Logger + peerScoreExposer p2p.PeerScoreExposer + // clusterChangeConsumer is a callback that is invoked when the set of active clusters of collection nodes changes. + // This callback is implemented by the rpc inspector suite of the GossipSubAdapter, and consumes the cluster changes + // to update the rpc inspector state of the recent topics (i.e., channels). + clusterChangeConsumer p2p.CollectionClusterChangesConsumer } var _ p2p.PubSubAdapter = (*GossipSubAdapter)(nil) -func NewGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig) (p2p.PubSubAdapter, error) { +func NewGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host, cfg p2p.PubSubAdapterConfig, clusterChangeConsumer p2p.CollectionClusterChangesConsumer) (p2p.PubSubAdapter, error) { gossipSubConfig, ok := cfg.(*GossipSubAdapterConfig) if !ok { return nil, fmt.Errorf("invalid gossipsub config type: %T", cfg) @@ -39,8 +53,16 @@ func NewGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host builder := component.NewComponentManagerBuilder() a := &GossipSubAdapter{ - gossipSub: gossipSub, - logger: logger, + gossipSub: gossipSub, + logger: logger.With().Str("component", "gossipsub-adapter").Logger(), + clusterChangeConsumer: clusterChangeConsumer, + } + + topicScoreParamFunc, ok := gossipSubConfig.TopicScoreParamFunc() + if ok { + a.topicScoreParamFunc = topicScoreParamFunc + } else { + a.logger.Warn().Msg("no topic score param func provided") } if scoreTracer := gossipSubConfig.ScoreTracer(); scoreTracer != nil { @@ -53,6 +75,7 @@ func NewGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host <-scoreTracer.Done() a.logger.Debug().Str("component", "gossipsub_score_tracer").Msg("score tracer stopped") }) + a.peerScoreExposer = scoreTracer } if tracer := gossipSubConfig.PubSubTracer(); tracer != nil { @@ -67,17 +90,22 @@ func NewGossipSubAdapter(ctx context.Context, logger zerolog.Logger, h host.Host }) } - for _, inspector := range gossipSubConfig.RPCInspectors() { - rpcInspector := inspector + if inspectorSuite := gossipSubConfig.InspectorSuiteComponent(); inspectorSuite != nil { builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - ready() - componentName := rpcInspector.Name() - a.logger.Debug().Str("component", componentName).Msg("starting rpc inspector") - rpcInspector.Start(ctx) - a.logger.Debug().Str("component", componentName).Msg("rpc inspector started") - - <-rpcInspector.Done() - a.logger.Debug().Str("component", componentName).Msg("rpc inspector stopped") + a.logger.Debug().Str("component", "gossipsub_inspector_suite").Msg("starting inspector suite") + inspectorSuite.Start(ctx) + a.logger.Debug().Str("component", "gossipsub_inspector_suite").Msg("inspector suite started") + + select { + case <-ctx.Done(): + a.logger.Debug().Str("component", "gossipsub_inspector_suite").Msg("inspector suite context done") + case <-inspectorSuite.Ready(): + ready() + a.logger.Debug().Str("component", "gossipsub_inspector_suite").Msg("inspector suite ready") + } + + <-inspectorSuite.Done() + a.logger.Debug().Str("component", "gossipsub_inspector_suite").Msg("inspector suite stopped") }) } @@ -119,6 +147,21 @@ func (g *GossipSubAdapter) Join(topic string) (p2p.Topic, error) { if err != nil { return nil, fmt.Errorf("could not join topic %s: %w", topic, err) } + + if g.topicScoreParamFunc != nil { + topicParams := g.topicScoreParamFunc(t) + err = t.SetScoreParams(topicParams) + if err != nil { + return nil, fmt.Errorf("could not set score params for topic %s: %w", topic, err) + } + topicParamsLogger := utils.TopicScoreParamsLogger(g.logger, topic, topicParams) + topicParamsLogger.Info().Msg("joined topic with score params set") + } else { + g.logger.Warn(). + Bool(logging.KeyNetworkingSecurity, true). + Str("topic", topic). + Msg("joining topic without score params, this is not recommended from a security perspective") + } return NewGossipSubTopic(t), nil } @@ -129,3 +172,29 @@ func (g *GossipSubAdapter) GetTopics() []string { func (g *GossipSubAdapter) ListPeers(topic string) []peer.ID { return g.gossipSub.ListPeers(topic) } + +// PeerScoreExposer returns the peer score exposer for the gossipsub adapter. The exposer is a read-only interface +// for querying peer scores and returns the local scoring table of the underlying gossipsub node. +// The exposer is only available if the gossipsub adapter was configured with a score tracer. +// If the gossipsub adapter was not configured with a score tracer, the exposer will be nil. +// Args: +// +// None. +// +// Returns: +// +// The peer score exposer for the gossipsub adapter. +func (g *GossipSubAdapter) PeerScoreExposer() p2p.PeerScoreExposer { + return g.peerScoreExposer +} + +// ActiveClustersChanged is called when the active clusters of collection nodes changes. +// GossipSubAdapter implements this method to forward the call to the clusterChangeConsumer (rpc inspector), +// which will then update the cluster state of the rpc inspector. +// Args: +// - lst: the list of active clusters +// Returns: +// - void +func (g *GossipSubAdapter) ActiveClustersChanged(lst flow.ChainIDList) { + g.clusterChangeConsumer.ActiveClustersChanged(lst) +} diff --git a/network/p2p/p2pnode/gossipSubAdapterConfig.go b/network/p2p/p2pnode/gossipSubAdapterConfig.go index 5e1081fd704..a2dbe59289f 100644 --- a/network/p2p/p2pnode/gossipSubAdapterConfig.go +++ b/network/p2p/p2pnode/gossipSubAdapterConfig.go @@ -7,68 +7,132 @@ import ( "github.com/libp2p/go-libp2p/core/routing" discoveryrouting "github.com/libp2p/go-libp2p/p2p/discovery/routing" + "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/p2p/inspector" ) // GossipSubAdapterConfig is a wrapper around libp2p pubsub options that // implements the PubSubAdapterConfig interface for the Flow network. type GossipSubAdapterConfig struct { - options []pubsub.Option - inspectors []p2p.GossipSubRPCInspector - scoreTracer p2p.PeerScoreTracer - pubsubTracer p2p.PubSubTracer + options []pubsub.Option + scoreTracer p2p.PeerScoreTracer + scoreOption p2p.ScoreOptionBuilder + pubsubTracer p2p.PubSubTracer + inspectorSuite p2p.GossipSubInspectorSuite // currently only used to manage the lifecycle. } var _ p2p.PubSubAdapterConfig = (*GossipSubAdapterConfig)(nil) +// NewGossipSubAdapterConfig creates a new GossipSubAdapterConfig with the default options. +// Args: +// - base: the base pubsub adapter config +// +// Returns: +// - a new GossipSubAdapterConfig func NewGossipSubAdapterConfig(base *p2p.BasePubSubAdapterConfig) *GossipSubAdapterConfig { return &GossipSubAdapterConfig{ options: defaultPubsubOptions(base), } } +// WithRoutingDiscovery adds a routing discovery option to the config. +// Args: +// - routing: the routing discovery to use +// +// Returns: +// -None func (g *GossipSubAdapterConfig) WithRoutingDiscovery(routing routing.ContentRouting) { g.options = append(g.options, pubsub.WithDiscovery(discoveryrouting.NewRoutingDiscovery(routing))) } +// WithSubscriptionFilter adds a subscription filter option to the config. +// Args: +// - filter: the subscription filter to use +// +// Returns: +// -None func (g *GossipSubAdapterConfig) WithSubscriptionFilter(filter p2p.SubscriptionFilter) { g.options = append(g.options, pubsub.WithSubscriptionFilter(filter)) } +// WithScoreOption adds a score option to the config. +// Args: +// - option: the score option to use +// Returns: +// -None func (g *GossipSubAdapterConfig) WithScoreOption(option p2p.ScoreOptionBuilder) { - g.options = append(g.options, option.BuildFlowPubSubScoreOption()) + params, thresholds := option.BuildFlowPubSubScoreOption() + g.scoreOption = option + g.options = append(g.options, pubsub.WithPeerScore(params, thresholds)) } +// WithMessageIdFunction adds a message ID function option to the config. +// Args: +// - f: the message ID function to use +// Returns: +// -None func (g *GossipSubAdapterConfig) WithMessageIdFunction(f func([]byte) string) { g.options = append(g.options, pubsub.WithMessageIdFn(func(pmsg *pb.Message) string { return f(pmsg.Data) })) } -func (g *GossipSubAdapterConfig) WithAppSpecificRpcInspectors(inspectors ...p2p.GossipSubRPCInspector) { - g.inspectors = inspectors - aggregator := inspector.NewAggregateRPCInspector(inspectors...) - g.options = append(g.options, pubsub.WithAppSpecificRpcInspector(aggregator.Inspect)) +// WithInspectorSuite adds an inspector suite option to the config. +// Args: +// - suite: the inspector suite to use +// Returns: +// -None +func (g *GossipSubAdapterConfig) WithInspectorSuite(suite p2p.GossipSubInspectorSuite) { + g.options = append(g.options, pubsub.WithAppSpecificRpcInspector(suite.InspectFunc())) + g.inspectorSuite = suite } +// WithTracer adds a tracer option to the config. +// Args: +// - tracer: the tracer to use +// Returns: +// -None func (g *GossipSubAdapterConfig) WithTracer(tracer p2p.PubSubTracer) { g.pubsubTracer = tracer g.options = append(g.options, pubsub.WithRawTracer(tracer)) } +// ScoreTracer returns the tracer for the peer score. +// Args: +// - None +// +// Returns: +// - p2p.PeerScoreTracer: the tracer for the peer score. func (g *GossipSubAdapterConfig) ScoreTracer() p2p.PeerScoreTracer { return g.scoreTracer } +// PubSubTracer returns the tracer for the pubsub. +// Args: +// - None +// Returns: +// - p2p.PubSubTracer: the tracer for the pubsub. func (g *GossipSubAdapterConfig) PubSubTracer() p2p.PubSubTracer { return g.pubsubTracer } -func (g *GossipSubAdapterConfig) RPCInspectors() []p2p.GossipSubRPCInspector { - return g.inspectors +// InspectorSuiteComponent returns the component that manages the lifecycle of the inspector suite. +// This is used to start and stop the inspector suite by the PubSubAdapter. +// Args: +// - None +// +// Returns: +// - component.Component: the component that manages the lifecycle of the inspector suite. +func (g *GossipSubAdapterConfig) InspectorSuiteComponent() component.Component { + return g.inspectorSuite } +// WithScoreTracer sets the tracer for the peer score. +// Args: +// - tracer: the tracer for the peer score. +// +// Returns: +// - None func (g *GossipSubAdapterConfig) WithScoreTracer(tracer p2p.PeerScoreTracer) { g.scoreTracer = tracer g.options = append(g.options, pubsub.WithPeerScoreInspect(func(snapshot map[peer.ID]*pubsub.PeerScoreSnapshot) { @@ -77,6 +141,11 @@ func (g *GossipSubAdapterConfig) WithScoreTracer(tracer p2p.PeerScoreTracer) { } // convertPeerScoreSnapshots converts a libp2p pubsub peer score snapshot to a Flow peer score snapshot. +// Args: +// - snapshot: the libp2p pubsub peer score snapshot. +// +// Returns: +// - map[peer.ID]*p2p.PeerScoreSnapshot: the Flow peer score snapshot. func convertPeerScoreSnapshots(snapshot map[peer.ID]*pubsub.PeerScoreSnapshot) map[peer.ID]*p2p.PeerScoreSnapshot { newSnapshot := make(map[peer.ID]*p2p.PeerScoreSnapshot) for id, snap := range snapshot { @@ -92,6 +161,11 @@ func convertPeerScoreSnapshots(snapshot map[peer.ID]*pubsub.PeerScoreSnapshot) m } // convertTopicScoreSnapshot converts a libp2p pubsub topic score snapshot to a Flow topic score snapshot. +// Args: +// - snapshot: the libp2p pubsub topic score snapshot. +// +// Returns: +// - map[string]*p2p.TopicScoreSnapshot: the Flow topic score snapshot. func convertTopicScoreSnapshot(snapshot map[string]*pubsub.TopicScoreSnapshot) map[string]*p2p.TopicScoreSnapshot { newSnapshot := make(map[string]*p2p.TopicScoreSnapshot) for topic, snap := range snapshot { @@ -106,10 +180,37 @@ func convertTopicScoreSnapshot(snapshot map[string]*pubsub.TopicScoreSnapshot) m return newSnapshot } +// TopicScoreParamFunc returns the topic score param function. This function is used to get the topic score params for a topic. +// The topic score params are used to set the topic parameters in GossipSub at the time of joining the topic. +// Args: +// - None +// +// Returns: +// - func(topic *pubsub.Topic) *pubsub.TopicScoreParams: the topic score param function if set, nil otherwise. +// - bool: true if the topic score param function is set, false otherwise. +func (g *GossipSubAdapterConfig) TopicScoreParamFunc() (func(topic *pubsub.Topic) *pubsub.TopicScoreParams, bool) { + if g.scoreOption != nil { + return func(topic *pubsub.Topic) *pubsub.TopicScoreParams { + return g.scoreOption.TopicScoreParams(topic) + }, true + } + + return nil, false +} + +// Build returns the libp2p pubsub options. +// Args: +// - None +// +// Returns: +// - []pubsub.Option: the libp2p pubsub options. +// +// Build is idempotent. func (g *GossipSubAdapterConfig) Build() []pubsub.Option { return g.options } +// defaultPubsubOptions returns the default libp2p pubsub options. These options are used by the Flow network to create a libp2p pubsub. func defaultPubsubOptions(base *p2p.BasePubSubAdapterConfig) []pubsub.Option { return []pubsub.Option{ pubsub.WithMessageSigning(true), diff --git a/network/p2p/p2pnode/internal/cache.go b/network/p2p/p2pnode/internal/cache.go new file mode 100644 index 00000000000..6d8952e6628 --- /dev/null +++ b/network/p2p/p2pnode/internal/cache.go @@ -0,0 +1,192 @@ +package internal + +import ( + "errors" + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" + "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" + "github.com/onflow/flow-go/module/mempool/stdmap" + "github.com/onflow/flow-go/network" +) + +var ( + ErrDisallowCacheEntityNotFound = errors.New("disallow list cache entity not found") +) + +// DisallowListCache is the disallow-list cache. It is used to keep track of the disallow-listed peers and the reasons for it. +type DisallowListCache struct { + c *stdmap.Backend +} + +// NewDisallowListCache creates a new disallow-list cache. The cache is backed by a stdmap.Backend. +// Args: +// - sizeLimit: the size limit of the cache, i.e., the maximum number of records that the cache can hold, recommended size is 100 * number of authorized nodes. +// - logger: the logger used by the cache. +// - collector: the metrics collector used by the cache. +// Returns: +// - *DisallowListCache: the created cache. +func NewDisallowListCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *DisallowListCache { + backData := herocache.NewCache(sizeLimit, + herocache.DefaultOversizeFactor, + // this cache is supposed to keep the disallow-list causes for the authorized (staked) nodes. Since the number of such nodes is + // expected to be small, we do not eject any records from the cache. The cache size must be large enough to hold all + // the spam records of the authorized nodes. Also, this cache is keeping at most one record per peer id, so the + // size of the cache must be at least the number of authorized nodes. + heropool.NoEjection, + logger.With().Str("mempool", "disallow-list-records").Logger(), + collector) + + return &DisallowListCache{ + c: stdmap.NewBackend(stdmap.WithBackData(backData)), + } +} + +// IsDisallowListed determines whether the given peer is disallow-listed for any reason. +// Args: +// - peerID: the peer to check. +// Returns: +// - []network.DisallowListedCause: the list of causes for which the given peer is disallow-listed. If the peer is not disallow-listed for any reason, +// a nil slice is returned. +// - bool: true if the peer is disallow-listed for any reason, false otherwise. +func (d *DisallowListCache) IsDisallowListed(peerID peer.ID) ([]network.DisallowListedCause, bool) { + entity, exists := d.c.ByID(makeId(peerID)) + if !exists { + return nil, false + } + + dEntity := mustBeDisallowListEntity(entity) + if len(dEntity.causes) == 0 { + return nil, false + } + + causes := make([]network.DisallowListedCause, 0, len(dEntity.causes)) + for c := range dEntity.causes { + causes = append(causes, c) + } + return causes, true +} + +// init initializes the disallow-list cache entity for the peerID. +// Args: +// - peerID: the peerID of the peer to be disallow-listed. +// Returns: +// - bool: true if the entity is successfully added to the cache. +// false if the entity already exists in the cache. +func (d *DisallowListCache) init(peerID peer.ID) bool { + return d.c.Add(&disallowListCacheEntity{ + peerID: peerID, + causes: make(map[network.DisallowListedCause]struct{}), + id: makeId(peerID), + }) +} + +// DisallowFor disallow-lists a peer for a cause. +// Args: +// - peerID: the peerID of the peer to be disallow-listed. +// - cause: the cause for disallow-listing the peer. +// Returns: +// - []network.DisallowListedCause: the list of causes for which the peer is disallow-listed. +// - error: if the operation fails, error is irrecoverable. +func (d *DisallowListCache) DisallowFor(peerID peer.ID, cause network.DisallowListedCause) ([]network.DisallowListedCause, error) { + // first, we try to optimistically add the peer to the disallow list. + causes, err := d.disallowListFor(peerID, cause) + switch { + case err == nil: + return causes, nil + case err == ErrDisallowCacheEntityNotFound: + // if the entity not exist, we initialize it and try again. + // Note: there is an edge case where the entity is initialized by another goroutine between the two calls. + // In this case, the init function is invoked twice, but it is not a problem because the underlying + // cache is thread-safe. Hence, we do not need to synchronize the two calls. In such cases, one of the + // two calls returns false, and the other call returns true. We do not care which call returns false, hence, + // we ignore the return value of the init function. + _ = d.init(peerID) + causes, err = d.disallowListFor(peerID, cause) + if err != nil { + // any error after the init is irrecoverable. + return nil, fmt.Errorf("failed to disallow list peer %s for cause %s: %w", peerID, cause, err) + } + return causes, nil + default: + return nil, fmt.Errorf("failed to disallow list peer %s for cause %s: %w", peerID, cause, err) + } +} + +// disallowListFor is a helper function for disallowing a peer for a cause. +// It adds the cause to the disallow list cache entity for the peerID and returns the updated list of causes for the peer. +// Args: +// - peerID: the peerID of the peer to be disallow-listed. +// - cause: the cause for disallow-listing the peer. +// Returns: +// - the updated list of causes for the peer. +// - error if the entity for the peerID is not found in the cache it returns ErrDisallowCacheEntityNotFound, which is a benign error. +func (d *DisallowListCache) disallowListFor(peerID peer.ID, cause network.DisallowListedCause) ([]network.DisallowListedCause, error) { + adjustedEntity, adjusted := d.c.Adjust(makeId(peerID), func(entity flow.Entity) flow.Entity { + dEntity := mustBeDisallowListEntity(entity) + dEntity.causes[cause] = struct{}{} + return dEntity + }) + + if !adjusted { + // if the entity is not found in the cache, we return a benign error. + return nil, ErrDisallowCacheEntityNotFound + } + + dEntity := mustBeDisallowListEntity(adjustedEntity) + updatedCauses := make([]network.DisallowListedCause, 0, len(dEntity.causes)) + for c := range dEntity.causes { + updatedCauses = append(updatedCauses, c) + } + + return updatedCauses, nil +} + +// AllowFor removes a cause from the disallow list cache entity for the peerID. +// Args: +// - peerID: the peerID of the peer to be allow-listed. +// - cause: the cause for allow-listing the peer. +// Returns: +// - the list of causes for which the peer is disallow-listed. +// - error if the entity for the peerID is not found in the cache it returns ErrDisallowCacheEntityNotFound, which is a benign error. +func (d *DisallowListCache) AllowFor(peerID peer.ID, cause network.DisallowListedCause) []network.DisallowListedCause { + adjustedEntity, adjusted := d.c.Adjust(makeId(peerID), func(entity flow.Entity) flow.Entity { + dEntity := mustBeDisallowListEntity(entity) + delete(dEntity.causes, cause) + return dEntity + }) + + if !adjusted { + // if the entity is not found in the cache, we return an empty list. + // we don't return a nil to be consistent with the case that entity is found but the list of causes is empty. + return make([]network.DisallowListedCause, 0) + } + + dEntity := mustBeDisallowListEntity(adjustedEntity) + // returning a deep copy of causes (to avoid being mutated externally). + causes := make([]network.DisallowListedCause, 0, len(dEntity.causes)) + for c := range dEntity.causes { + causes = append(causes, c) + } + return causes +} + +// mustBeDisallowListEntity is a helper function for type assertion of the flow.Entity to disallowListCacheEntity. +// It panics if the type assertion fails. +// Args: +// - entity: the flow.Entity to be type asserted. +// Returns: +// - the disallowListCacheEntity. +func mustBeDisallowListEntity(entity flow.Entity) *disallowListCacheEntity { + dEntity, ok := entity.(*disallowListCacheEntity) + if !ok { + // this should never happen, unless there is a bug. We should crash the node and do not proceed. + panic(fmt.Errorf("disallow list cache entity is not of type disallowListCacheEntity, got: %T", entity)) + } + return dEntity +} diff --git a/network/p2p/p2pnode/internal/cacheEntity.go b/network/p2p/p2pnode/internal/cacheEntity.go new file mode 100644 index 00000000000..e55b0d127b5 --- /dev/null +++ b/network/p2p/p2pnode/internal/cacheEntity.go @@ -0,0 +1,44 @@ +package internal + +import ( + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/network" +) + +// disallowListCacheEntity is the model data type for the disallow list cache. +// It represents a single peer that is disallow-listed and the reasons for it. +// The key for storing this entity is the id field which is the hash of the peerID. +// This means that the entities are deduplicated by their peerID. +type disallowListCacheEntity struct { + peerID peer.ID + causes map[network.DisallowListedCause]struct{} + // id is the hash of the peerID which is used as the key for storing the entity in the cache. + // we cache it internally to avoid hashing the peerID multiple times. + id flow.Identifier +} + +var _ flow.Entity = (*disallowListCacheEntity)(nil) + +// ID returns the hash of the peerID which is used as the key for storing the entity in the cache. +// Returns: +// - the hash of the peerID as a flow.Identifier. +func (d *disallowListCacheEntity) ID() flow.Identifier { + return d.id +} + +// Checksum returns the hash of the peerID, there is no use for this method in the cache. It is implemented to satisfy +// the flow.Entity interface. +// Returns: +// - the hash of the peerID as a flow.Identifier. +func (d *disallowListCacheEntity) Checksum() flow.Identifier { + return d.id +} + +// makeId is a helper function for creating the id field of the disallowListCacheEntity by hashing the peerID. +// Returns: +// - the hash of the peerID as a flow.Identifier. +func makeId(peerID peer.ID) flow.Identifier { + return flow.MakeID([]byte(peerID)) +} diff --git a/network/p2p/p2pnode/internal/cache_test.go b/network/p2p/p2pnode/internal/cache_test.go new file mode 100644 index 00000000000..d4ab02d5c9b --- /dev/null +++ b/network/p2p/p2pnode/internal/cache_test.go @@ -0,0 +1,355 @@ +package internal_test + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/p2p/p2pnode/internal" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestNewDisallowListCache tests the NewDisallowListCache function. It verifies that the returned disallowListCache +// is not nil. +func TestNewDisallowListCache(t *testing.T) { + disallowListCache := internal.NewDisallowListCache(uint32(100), unittest.Logger(), metrics.NewNoopCollector()) + + // Verify that the new disallowListCache is not nil + assert.NotNil(t, disallowListCache) +} + +// TestDisallowFor_SinglePeer tests the DisallowFor function for a single peer. It verifies that the peerID is +// disallow-listed for the given cause and that the cause is returned when the peerID is disallow-listed again. +func TestDisallowFor_SinglePeer(t *testing.T) { + disallowListCache := internal.NewDisallowListCache(uint32(100), unittest.Logger(), metrics.NewNoopCollector()) + require.NotNil(t, disallowListCache) + + // disallowing a peerID for a cause when the peerID doesn't exist in the cache + causes, err := disallowListCache.DisallowFor(peer.ID("peer1"), network.DisallowListedCauseAdmin) + require.NoError(t, err) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAdmin) + + // disallowing a peerID for a cause when the peerID already exists in the cache + causes, err = disallowListCache.DisallowFor(peer.ID("peer1"), network.DisallowListedCauseAlsp) + require.NoError(t, err) + require.Len(t, causes, 2) + require.ElementsMatch(t, causes, []network.DisallowListedCause{network.DisallowListedCauseAdmin, network.DisallowListedCauseAlsp}) + + // disallowing a peerID for a duplicate cause + causes, err = disallowListCache.DisallowFor(peer.ID("peer1"), network.DisallowListedCauseAdmin) + require.NoError(t, err) + require.Len(t, causes, 2) + require.ElementsMatch(t, causes, []network.DisallowListedCause{network.DisallowListedCauseAdmin, network.DisallowListedCauseAlsp}) +} + +// TestDisallowFor_MultiplePeers tests the DisallowFor function for multiple peers. It verifies that the peerIDs are +// disallow-listed for the given cause and that the cause is returned when the peerIDs are disallow-listed again. +func TestDisallowFor_MultiplePeers(t *testing.T) { + disallowListCache := internal.NewDisallowListCache(uint32(100), unittest.Logger(), metrics.NewNoopCollector()) + require.NotNil(t, disallowListCache) + + for i := 0; i <= 10; i++ { + // disallowing a peerID for a cause when the peerID doesn't exist in the cache + causes, err := disallowListCache.DisallowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAdmin) + require.NoError(t, err) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAdmin) + } + + for i := 0; i <= 10; i++ { + // disallowing a peerID for a cause when the peerID already exists in the cache + causes, err := disallowListCache.DisallowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAlsp) + require.NoError(t, err) + require.Len(t, causes, 2) + require.ElementsMatch(t, causes, []network.DisallowListedCause{network.DisallowListedCauseAdmin, network.DisallowListedCauseAlsp}) + } + + for i := 0; i <= 10; i++ { + // getting the disallow-listed causes for a peerID + causes, disallowListed := disallowListCache.IsDisallowListed(peer.ID(fmt.Sprintf("peer-%d", i))) + require.True(t, disallowListed) + require.Len(t, causes, 2) + require.ElementsMatch(t, causes, []network.DisallowListedCause{network.DisallowListedCauseAdmin, network.DisallowListedCauseAlsp}) + } +} + +// TestAllowFor_SinglePeer is a unit test function to verify the behavior of DisallowListCache for a single peer. +// The test checks the following functionalities in sequence: +// 1. Allowing a peerID for a cause when the peerID already exists in the cache. +// 2. Disallowing the peerID for a cause when the peerID doesn't exist in the cache. +// 3. Getting the disallow-listed causes for the peerID. +// 4. Allowing a peerID for a cause when the peerID already exists in the cache. +// 5. Getting the disallow-listed causes for the peerID. +// 6. Disallowing the peerID for a cause. +// 7. Allowing the peerID for a different cause than it is disallowed when the peerID already exists in the cache. +// 8. Disallowing the peerID for another cause. +// 9. Allowing the peerID for the first cause. +// 10. Allowing the peerID for the second cause. +func TestAllowFor_SinglePeer(t *testing.T) { + disallowListCache := internal.NewDisallowListCache(uint32(100), unittest.Logger(), metrics.NewNoopCollector()) + require.NotNil(t, disallowListCache) + peerID := peer.ID("peer1") + + // allowing the peerID for a cause when the peerID already exists in the cache + causes := disallowListCache.AllowFor(peerID, network.DisallowListedCauseAdmin) + require.Len(t, causes, 0) + + // disallowing the peerID for a cause when the peerID doesn't exist in the cache + causes, err := disallowListCache.DisallowFor(peerID, network.DisallowListedCauseAdmin) + require.NoError(t, err) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAdmin) + + // getting the disallow-listed causes for the peerID + causes, disallowListed := disallowListCache.IsDisallowListed(peerID) + require.True(t, disallowListed) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAdmin) + + // allowing a peerID for a cause when the peerID already exists in the cache + causes = disallowListCache.AllowFor(peerID, network.DisallowListedCauseAdmin) + require.NoError(t, err) + require.Len(t, causes, 0) + + // getting the disallow-listed causes for the peerID + causes, disallowListed = disallowListCache.IsDisallowListed(peerID) + require.False(t, disallowListed) + require.Len(t, causes, 0) + + // disallowing the peerID for a cause + causes, err = disallowListCache.DisallowFor(peerID, network.DisallowListedCauseAdmin) + require.NoError(t, err) + require.Len(t, causes, 1) + + // allowing the peerID for a different cause than it is disallowed when the peerID already exists in the cache + causes = disallowListCache.AllowFor(peerID, network.DisallowListedCauseAlsp) + require.NoError(t, err) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAdmin) // the peerID is still disallow-listed for the previous cause + + // disallowing the peerID for another cause + causes, err = disallowListCache.DisallowFor(peerID, network.DisallowListedCauseAlsp) + require.NoError(t, err) + require.Len(t, causes, 2) + require.ElementsMatch(t, causes, []network.DisallowListedCause{network.DisallowListedCauseAdmin, network.DisallowListedCauseAlsp}) + + // allowing the peerID for the first cause + causes = disallowListCache.AllowFor(peerID, network.DisallowListedCauseAdmin) + require.NoError(t, err) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAlsp) // the peerID is still disallow-listed for the previous cause + + // allowing the peerID for the second cause + causes = disallowListCache.AllowFor(peerID, network.DisallowListedCauseAlsp) + require.NoError(t, err) + require.Len(t, causes, 0) +} + +// TestAllowFor_MultiplePeers_Sequentially is a unit test function to test the behavior of DisallowListCache with multiple peers. +// The test checks the following functionalities in sequence: +// 1. Allowing a peerID for a cause when the peerID doesn't exist in the cache. +// 2. Disallowing peers for a cause. +// 3. Getting the disallow-listed causes for a peerID. +// 4. Allowing the peer ids for a cause different than the one they are disallow-listed for. +// 5. Disallowing the peer ids for a different cause. +// 6. Allowing the peer ids for the first cause. +// 7. Allowing the peer ids for the second cause. +func TestAllowFor_MultiplePeers_Sequentially(t *testing.T) { + disallowListCache := internal.NewDisallowListCache(uint32(100), unittest.Logger(), metrics.NewNoopCollector()) + require.NotNil(t, disallowListCache) + + for i := 0; i <= 10; i++ { + // allowing a peerID for a cause when the peerID doesn't exist in the cache + causes := disallowListCache.AllowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAdmin) + require.Len(t, causes, 0) + } + + for i := 0; i <= 10; i++ { + // disallowing peers for a cause + causes, err := disallowListCache.DisallowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAlsp) + require.NoError(t, err) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAlsp) + } + + for i := 0; i <= 10; i++ { + // getting the disallow-listed causes for a peerID + causes, disallowListed := disallowListCache.IsDisallowListed(peer.ID(fmt.Sprintf("peer-%d", i))) + require.True(t, disallowListed) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAlsp) + } + + for i := 0; i <= 10; i++ { + // allowing the peer ids for a cause different than the one they are disallow-listed for + causes := disallowListCache.AllowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAdmin) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAlsp) + } + + for i := 0; i <= 10; i++ { + // disallowing the peer ids for a different cause + causes, err := disallowListCache.DisallowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAdmin) + require.NoError(t, err) + require.Len(t, causes, 2) + require.ElementsMatch(t, causes, []network.DisallowListedCause{network.DisallowListedCauseAdmin, network.DisallowListedCauseAlsp}) + } + + for i := 0; i <= 10; i++ { + // allowing the peer ids for the first cause + causes := disallowListCache.AllowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAdmin) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAlsp) + } + + for i := 0; i <= 10; i++ { + // allowing the peer ids for the second cause + causes := disallowListCache.AllowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAlsp) + require.Len(t, causes, 0) + } +} + +// TestAllowFor_MultiplePeers_Concurrently is a unit test function that verifies the behavior of DisallowListCache +// when multiple peerIDs are added and managed concurrently. This test is designed to confirm that DisallowListCache +// works as expected under concurrent access, an important aspect for a system dealing with multiple connections. +// +// The test runs multiple goroutines simultaneously, each handling a different peerID and performs the following +// operations in the sequence: +// 1. Allowing a peerID for a cause when the peerID doesn't exist in the cache. +// 2. Disallowing peers for a cause. +// 3. Getting the disallow-listed causes for a peerID. +// 4. Allowing the peer ids for a cause different than the one they are disallow-listed for. +// 5. Disallowing the peer ids for a different cause. +// 6. Allowing the peer ids for the first cause. +// 7. Allowing the peer ids for the second cause. +// 8. Getting the disallow-listed causes for a peerID. +// 9. Allowing a peerID for a cause when the peerID doesn't exist in the cache for a new set of peers. +func TestAllowFor_MultiplePeers_Concurrently(t *testing.T) { + disallowListCache := internal.NewDisallowListCache(uint32(100), unittest.Logger(), metrics.NewNoopCollector()) + require.NotNil(t, disallowListCache) + + var wg sync.WaitGroup + for i := 0; i <= 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + // allowing a peerID for a cause when the peerID doesn't exist in the cache + causes := disallowListCache.AllowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAdmin) + require.Len(t, causes, 0) + }(i) + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + for i := 0; i <= 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + // disallowing peers for a cause + causes, err := disallowListCache.DisallowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAlsp) + require.NoError(t, err) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAlsp) + }(i) + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + for i := 0; i <= 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + // getting the disallow-listed causes for a peerID + causes, disallowListed := disallowListCache.IsDisallowListed(peer.ID(fmt.Sprintf("peer-%d", i))) + require.Len(t, causes, 1) + require.True(t, disallowListed) + require.Contains(t, causes, network.DisallowListedCauseAlsp) + }(i) + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + for i := 0; i <= 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + // allowing the peer ids for a cause different than the one they are disallow-listed for + causes := disallowListCache.AllowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAdmin) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAlsp) + }(i) + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + for i := 0; i <= 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + // disallowing the peer ids for a different cause + causes, err := disallowListCache.DisallowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAdmin) + require.NoError(t, err) + require.Len(t, causes, 2) + require.ElementsMatch(t, causes, []network.DisallowListedCause{network.DisallowListedCauseAdmin, network.DisallowListedCauseAlsp}) + }(i) + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + for i := 0; i <= 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + // allowing the peer ids for the first cause + causes := disallowListCache.AllowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAdmin) + require.Len(t, causes, 1) + require.Contains(t, causes, network.DisallowListedCauseAlsp) + }(i) + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + for i := 0; i <= 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + // allowing the peer ids for the second cause + causes := disallowListCache.AllowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAlsp) + require.Len(t, causes, 0) + }(i) + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + for i := 0; i <= 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + // getting the disallow-listed causes for a peerID + causes, disallowListed := disallowListCache.IsDisallowListed(peer.ID(fmt.Sprintf("peer-%d", i))) + require.False(t, disallowListed) + require.Len(t, causes, 0) + }(i) + } + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + for i := 11; i <= 20; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + // allowing a peerID for a cause when the peerID doesn't exist in the cache + causes := disallowListCache.AllowFor(peer.ID(fmt.Sprintf("peer-%d", i)), network.DisallowListedCauseAdmin) + require.Len(t, causes, 0) + }(i) + } +} diff --git a/network/p2p/p2pnode/libp2pNode.go b/network/p2p/p2pnode/libp2pNode.go index 977a5b393d3..38746521430 100644 --- a/network/p2p/p2pnode/libp2pNode.go +++ b/network/p2p/p2pnode/libp2pNode.go @@ -18,12 +18,14 @@ import ( "github.com/libp2p/go-libp2p/core/routing" "github.com/rs/zerolog" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" flownet "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2putils" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/p2pnode/internal" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/utils/logging" ) @@ -46,20 +48,24 @@ const ( findPeerQueryTimeout = 10 * time.Second ) +var _ p2p.LibP2PNode = (*Node)(nil) + // Node is a wrapper around the LibP2P host. type Node struct { component.Component sync.RWMutex - uniMgr p2p.UnicastManager - host host.Host // reference to the libp2p host (https://godoc.org/github.com/libp2p/go-libp2p/core/host) - pubSub p2p.PubSubAdapter - logger zerolog.Logger // used to provide logging - topics map[channels.Topic]p2p.Topic // map of a topic string to an actual topic instance - subs map[channels.Topic]p2p.Subscription // map of a topic string to an actual subscription - routing routing.Routing - pCache p2p.ProtocolPeerCache - peerManager p2p.PeerManager - peerScoreExposer p2p.PeerScoreExposer + uniMgr p2p.UnicastManager + host host.Host // reference to the libp2p host (https://godoc.org/github.com/libp2p/go-libp2p/core/host) + pubSub p2p.PubSubAdapter + logger zerolog.Logger // used to provide logging + topics map[channels.Topic]p2p.Topic // map of a topic string to an actual topic instance + subs map[channels.Topic]p2p.Subscription // map of a topic string to an actual subscription + routing routing.Routing + pCache p2p.ProtocolPeerCache + peerManager p2p.PeerManager + // Cache of temporary disallow-listed peers, when a peer is disallow-listed, the connections to that peer + // are closed and further connections are not allowed till the peer is removed from the disallow-list. + disallowListedCache p2p.DisallowListCache } // NewNode creates a new libp2p node and sets its parameters. @@ -68,18 +74,24 @@ func NewNode( host host.Host, pCache p2p.ProtocolPeerCache, peerManager p2p.PeerManager, + disallowLstCacheCfg *p2p.DisallowListCacheConfig, ) *Node { + lg := logger.With().Str("component", "libp2p-node").Logger() return &Node{ host: host, - logger: logger.With().Str("component", "libp2p-node").Logger(), + logger: lg, topics: make(map[channels.Topic]p2p.Topic), subs: make(map[channels.Topic]p2p.Subscription), pCache: pCache, peerManager: peerManager, + disallowListedCache: internal.NewDisallowListCache( + disallowLstCacheCfg.MaxSize, + logger.With().Str("module", "disallow-list-cache").Logger(), + disallowLstCacheCfg.Metrics), } } -var _ component.Component = (*Node)(nil) +var _ p2p.LibP2PNode = (*Node)(nil) func (n *Node) Start(ctx irrecoverable.SignalerContext) { n.Component.Start(ctx) @@ -346,8 +358,29 @@ func (n *Node) WithDefaultUnicastProtocol(defaultHandler libp2pnet.StreamHandler // WithPeersProvider sets the PeersProvider for the peer manager. // If a peer manager factory is set, this method will set the peer manager's PeersProvider. func (n *Node) WithPeersProvider(peersProvider p2p.PeersProvider) { + // TODO: chore: we should not allow overriding the peers provider if one is already set. if n.peerManager != nil { - n.peerManager.SetPeersProvider(peersProvider) + n.peerManager.SetPeersProvider( + func() peer.IDSlice { + authorizedPeersIds := peersProvider() + allowListedPeerIds := peer.IDSlice{} // subset of authorizedPeersIds that are not disallowed + for _, peerId := range authorizedPeersIds { + // exclude the disallowed peers from the authorized peers list + causes, disallowListed := n.disallowListedCache.IsDisallowListed(peerId) + if disallowListed { + n.logger.Warn(). + Str("peer_id", peerId.String()). + Str("causes", fmt.Sprintf("%v", causes)). + Msg("peer is disallowed for a cause, removing from authorized peers of peer manager") + + // exclude the peer from the authorized peers list + continue + } + allowListedPeerIds = append(allowListedPeerIds, peerId) + } + + return allowListedPeerIds + }) } } @@ -395,25 +428,10 @@ func (n *Node) Routing() routing.Routing { return n.routing } -// SetPeerScoreExposer sets the node's peer score exposer implementation. -// SetPeerScoreExposer may be called at most once. It is an irrecoverable error to call this -// method if the node's peer score exposer has already been set. -func (n *Node) SetPeerScoreExposer(e p2p.PeerScoreExposer) { - if n.peerScoreExposer != nil { - n.logger.Fatal().Msg("peer score exposer already set") - } - - n.peerScoreExposer = e -} - // PeerScoreExposer returns the node's peer score exposer implementation. // If the node's peer score exposer has not been set, the second return value will be false. -func (n *Node) PeerScoreExposer() (p2p.PeerScoreExposer, bool) { - if n.peerScoreExposer == nil { - return nil, false - } - - return n.peerScoreExposer, true +func (n *Node) PeerScoreExposer() p2p.PeerScoreExposer { + return n.pubSub.PeerScoreExposer() } // SetPubSub sets the node's pubsub implementation. @@ -444,3 +462,72 @@ func (n *Node) SetUnicastManager(uniMgr p2p.UnicastManager) { } n.uniMgr = uniMgr } + +// OnDisallowListNotification is called when a new disallow list update notification is distributed. +// Any error on consuming event must handle internally. +// The implementation must be concurrency safe. +// Args: +// +// id: peer ID of the peer being disallow-listed. +// cause: cause of the peer being disallow-listed (only this cause is added to the peer's disallow-listed causes). +// +// Returns: +// +// none +func (n *Node) OnDisallowListNotification(peerId peer.ID, cause flownet.DisallowListedCause) { + causes, err := n.disallowListedCache.DisallowFor(peerId, cause) + if err != nil { + // returned error is fatal. + n.logger.Fatal().Err(err).Str("peer_id", peerId.String()).Msg("failed to add peer to disallow list") + } + + // TODO: this code should further be refactored to also log the Flow id. + n.logger.Warn(). + Str("peer_id", peerId.String()). + Str("notification_cause", cause.String()). + Str("causes", fmt.Sprintf("%v", causes)). + Msg("peer added to disallow list cache") +} + +// OnAllowListNotification is called when a new allow list update notification is distributed. +// Any error on consuming event must handle internally. +// The implementation must be concurrency safe. +// Args: +// +// id: peer ID of the peer being allow-listed. +// cause: cause of the peer being allow-listed (only this cause is removed from the peer's disallow-listed causes). +// +// Returns: +// +// none +func (n *Node) OnAllowListNotification(peerId peer.ID, cause flownet.DisallowListedCause) { + remainingCauses := n.disallowListedCache.AllowFor(peerId, cause) + + n.logger.Info(). + Str("peer_id", peerId.String()). + Str("causes", fmt.Sprintf("%v", cause)). + Str("remaining_causes", fmt.Sprintf("%v", remainingCauses)). + Msg("peer is allow-listed for cause") +} + +// IsDisallowListed determines whether the given peer is disallow-listed for any reason. +// Args: +// - peerID: the peer to check. +// Returns: +// - []network.DisallowListedCause: the list of causes for which the given peer is disallow-listed. If the peer is not disallow-listed for any reason, +// a nil slice is returned. +// - bool: true if the peer is disallow-listed for any reason, false otherwise. +func (n *Node) IsDisallowListed(peerId peer.ID) ([]flownet.DisallowListedCause, bool) { + return n.disallowListedCache.IsDisallowListed(peerId) +} + +// ActiveClustersChanged is called when the active clusters list of the collection clusters has changed. +// The LibP2PNode implementation directly calls the ActiveClustersChanged method of the pubsub implementation, as +// the pubsub implementation is responsible for the actual handling of the event. +// Args: +// - list: the new active clusters list. +// Returns: +// - none +func (n *Node) ActiveClustersChanged(list flow.ChainIDList) { + n.pubSub.ActiveClustersChanged(list) +} diff --git a/network/p2p/p2pnode/libp2pNode_test.go b/network/p2p/p2pnode/libp2pNode_test.go index 3d97096a22a..519a0579163 100644 --- a/network/p2p/p2pnode/libp2pNode_test.go +++ b/network/p2p/p2pnode/libp2pNode_test.go @@ -24,7 +24,6 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" "github.com/onflow/flow-go/network/internal/p2putils" - "github.com/onflow/flow-go/network/internal/testutils" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/p2pnode" p2ptest "github.com/onflow/flow-go/network/p2p/test" @@ -71,11 +70,12 @@ func TestMultiAddress(t *testing.T) { func TestSingleNodeLifeCycle(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := mockmodule.NewIdentityProvider(t) node, _ := p2ptest.NodeFixture( t, unittest.IdentifierFixture(), "test_single_node_life_cycle", + idProvider, ) node.Start(signalerCtx) @@ -113,9 +113,9 @@ func TestAddPeers(t *testing.T) { count := 3 ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) - // create nodes - nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_add_peers", count) + nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_add_peers", count, idProvider) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -135,9 +135,9 @@ func TestRemovePeers(t *testing.T) { count := 3 ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) // create nodes - nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_remove_peers", count) + nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_remove_peers", count, idProvider) peerInfos, errs := utils.PeerInfosFromIDs(identities) assert.Len(t, errs, 0) @@ -171,7 +171,8 @@ func TestConnGater(t *testing.T) { t, sporkID, t.Name(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + idProvider, + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { if !node1Peers.Has(pid) { return fmt.Errorf("peer id not found: %s", pid.String()) } @@ -189,7 +190,8 @@ func TestConnGater(t *testing.T) { node2, identity2 := p2ptest.NodeFixture( t, sporkID, t.Name(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + idProvider, + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { if !node2Peers.Has(pid) { return fmt.Errorf("id not found: %s", pid.String()) } @@ -227,9 +229,9 @@ func TestConnGater(t *testing.T) { func TestNode_HasSubscription(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) sporkID := unittest.IdentifierFixture() - node, _ := p2ptest.NodeFixture(t, sporkID, "test_has_subscription") + node, _ := p2ptest.NodeFixture(t, sporkID, "test_has_subscription", idProvider) p2ptest.StartNode(t, signalerCtx, node, 100*time.Millisecond) defer p2ptest.StopNode(t, node, cancel, 100*time.Millisecond) @@ -260,12 +262,14 @@ func TestCreateStream_SinglePairwiseConnection(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) nodes, ids := p2ptest.NodesFixture(t, sporkId, "test_create_stream_single_pairwise_connection", nodeCount, + idProvider, p2ptest.WithDefaultResourceManager()) + idProvider.SetIdentities(ids) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -332,7 +336,8 @@ func TestCreateStream_SinglePeerDial(t *testing.T) { t, sporkID, t.Name(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + idProvider, + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { // avoid connection gating outbound messages on sender return nil })), @@ -347,7 +352,8 @@ func TestCreateStream_SinglePeerDial(t *testing.T) { t, sporkID, t.Name(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + idProvider, + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { // connection gate all incoming connections forcing the senders unicast manager to perform retries return fmt.Errorf("gate keep") })), @@ -401,6 +407,7 @@ func TestCreateStream_InboundConnResourceLimit(t *testing.T) { t, sporkID, t.Name(), + idProvider, p2ptest.WithDefaultResourceManager(), p2ptest.WithCreateStreamRetryDelay(10*time.Millisecond)) @@ -408,6 +415,7 @@ func TestCreateStream_InboundConnResourceLimit(t *testing.T) { t, sporkID, t.Name(), + idProvider, p2ptest.WithDefaultResourceManager(), p2ptest.WithCreateStreamRetryDelay(10*time.Millisecond)) diff --git a/network/p2p/p2pnode/libp2pStream_test.go b/network/p2p/p2pnode/libp2pStream_test.go index fb184d58ecc..224a913aa8f 100644 --- a/network/p2p/p2pnode/libp2pStream_test.go +++ b/network/p2p/p2pnode/libp2pStream_test.go @@ -11,9 +11,6 @@ import ( "testing" "time" - "github.com/onflow/flow-go/network/p2p" - p2ptest "github.com/onflow/flow-go/network/p2p/test" - "github.com/libp2p/go-libp2p/core" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peerstore" @@ -21,8 +18,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/network/p2p" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" + mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/internal/p2pfixtures" "github.com/onflow/flow-go/network/internal/p2putils" "github.com/onflow/flow-go/network/p2p/p2pnode" @@ -41,13 +42,15 @@ func TestStreamClosing(t *testing.T) { var msgRegex = regexp.MustCompile("^hello[0-9]") handler, streamCloseWG := mockStreamHandlerForMessages(t, ctx, count, msgRegex) - + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) // Creates nodes nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_stream_closing", 2, + idProvider, p2ptest.WithDefaultStreamHandler(handler)) + idProvider.SetIdentities(identities) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -148,13 +151,14 @@ func testCreateStream(t *testing.T, sporkId flow.Identifier, unicasts []protocol count := 2 ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) nodes, identities := p2ptest.NodesFixture(t, sporkId, "test_create_stream", count, + idProvider, p2ptest.WithPreferredUnicasts(unicasts)) - + idProvider.SetIdentities(identities) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -208,13 +212,19 @@ func TestCreateStream_FallBack(t *testing.T) { // Creates two nodes: one with preferred gzip, and other one with default protocol sporkId := unittest.IdentifierFixture() - thisNode, _ := p2ptest.NodeFixture(t, + idProvider := mockmodule.NewIdentityProvider(t) + thisNode, thisID := p2ptest.NodeFixture(t, sporkId, "test_create_stream_fallback", + idProvider, p2ptest.WithPreferredUnicasts([]protocols.ProtocolName{protocols.GzipCompressionUnicast})) - otherNode, otherId := p2ptest.NodeFixture(t, sporkId, "test_create_stream_fallback") - + otherNode, otherId := p2ptest.NodeFixture(t, sporkId, "test_create_stream_fallback", idProvider) + identities := []flow.Identity{thisID, otherId} nodes := []p2p.LibP2PNode{thisNode, otherNode} + for i, node := range nodes { + idProvider.On("ByPeerID", node.Host().ID()).Return(&identities[i], true).Maybe() + + } p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -225,7 +235,7 @@ func TestCreateStream_FallBack(t *testing.T) { require.Equal(t, 0, p2putils.CountStream(thisNode.Host(), otherNode.Host().ID(), preferredProtocolId, network.DirOutbound)) // Now attempt to create another 100 outbound stream to the same destination by calling CreateStream - streamCount := 100 + streamCount := 10 var streams []network.Stream for i := 0; i < streamCount; i++ { pInfo, err := utils.PeerAddressInfo(otherId) @@ -249,12 +259,14 @@ func TestCreateStream_FallBack(t *testing.T) { // reverse loop to close all the streams for i := streamCount - 1; i >= 0; i-- { + fmt.Println("closing stream", i) s := streams[i] wg := sync.WaitGroup{} wg.Add(1) go func() { - err := s.Close() - assert.NoError(t, err) + // not checking the error as per upgrade of libp2p it returns stream reset error. This is not a problem + // as we are closing the stream anyway and counting the number of streams at the end. + _ = s.Close() wg.Done() }() unittest.RequireReturnsBefore(t, wg.Wait, 1*time.Second, "could not close streams on time") @@ -270,11 +282,11 @@ func TestCreateStream_FallBack(t *testing.T) { func TestCreateStreamIsConcurrencySafe(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) // create two nodes - nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_create_stream_is_concurrency_safe", 2) + nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_create_stream_is_concurrency_safe", 2, idProvider) require.Len(t, identities, 2) - + idProvider.SetIdentities(flow.IdentityList{identities[0], identities[1]}) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -318,17 +330,18 @@ func TestNoBackoffWhenCreatingStream(t *testing.T) { ctx2, cancel2 := context.WithCancel(ctx) signalerCtx2 := irrecoverable.NewMockSignalerContext(t, ctx2) - + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) count := 2 // Creates nodes nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_no_backoff_when_create_stream", count, + idProvider, ) node1 := nodes[0] node2 := nodes[1] - + idProvider.SetIdentities(flow.IdentityList{identities[0], identities[1]}) p2ptest.StartNode(t, signalerCtx1, node1, 100*time.Millisecond) p2ptest.StartNode(t, signalerCtx2, node2, 100*time.Millisecond) @@ -392,12 +405,13 @@ func testUnicastOverStream(t *testing.T, opts ...p2ptest.NodeFixtureParameterOpt // Creates nodes sporkId := unittest.IdentifierFixture() - + idProvider := mockmodule.NewIdentityProvider(t) streamHandler1, inbound1 := p2ptest.StreamHandlerFixture(t) node1, id1 := p2ptest.NodeFixture( t, sporkId, t.Name(), + idProvider, append(opts, p2ptest.WithDefaultStreamHandler(streamHandler1))...) streamHandler2, inbound2 := p2ptest.StreamHandlerFixture(t) @@ -405,10 +419,14 @@ func testUnicastOverStream(t *testing.T, opts ...p2ptest.NodeFixtureParameterOpt t, sporkId, t.Name(), + idProvider, append(opts, p2ptest.WithDefaultStreamHandler(streamHandler2))...) - - nodes := []p2p.LibP2PNode{node1, node2} ids := flow.IdentityList{&id1, &id2} + nodes := []p2p.LibP2PNode{node1, node2} + for i, node := range nodes { + idProvider.On("ByPeerID", node.Host().ID()).Return(ids[i], true).Maybe() + + } p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -432,12 +450,13 @@ func TestUnicastOverStream_Fallback(t *testing.T) { // node1: supports only plain unicast protocol // node2: supports plain and gzip sporkId := unittest.IdentifierFixture() - + idProvider := mockmodule.NewIdentityProvider(t) streamHandler1, inbound1 := p2ptest.StreamHandlerFixture(t) node1, id1 := p2ptest.NodeFixture( t, sporkId, t.Name(), + idProvider, p2ptest.WithDefaultStreamHandler(streamHandler1), ) @@ -446,12 +465,17 @@ func TestUnicastOverStream_Fallback(t *testing.T) { t, sporkId, t.Name(), + idProvider, p2ptest.WithDefaultStreamHandler(streamHandler2), p2ptest.WithPreferredUnicasts([]protocols.ProtocolName{protocols.GzipCompressionUnicast}), ) - nodes := []p2p.LibP2PNode{node1, node2} ids := flow.IdentityList{&id1, &id2} + nodes := []p2p.LibP2PNode{node1, node2} + for i, node := range nodes { + idProvider.On("ByPeerID", node.Host().ID()).Return(ids[i], true).Maybe() + + } p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -464,15 +488,16 @@ func TestUnicastOverStream_Fallback(t *testing.T) { func TestCreateStreamTimeoutWithUnresponsiveNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) // creates a regular node nodes, identities := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_create_stream_timeout_with_unresponsive_node", 1, + idProvider, ) require.Len(t, identities, 1) - + idProvider.SetIdentities(identities) p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -505,15 +530,16 @@ func TestCreateStreamTimeoutWithUnresponsiveNode(t *testing.T) { func TestCreateStreamIsConcurrent(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) // create two regular node goodNodes, goodNodeIds := p2ptest.NodesFixture(t, unittest.IdentifierFixture(), "test_create_stream_is_concurrent", 2, + idProvider, ) require.Len(t, goodNodeIds, 2) - + idProvider.SetIdentities(goodNodeIds) p2ptest.StartNodes(t, signalerCtx, goodNodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, goodNodes, cancel, 100*time.Millisecond) diff --git a/network/p2p/p2pnode/protocolPeerCache.go b/network/p2p/p2pnode/protocolPeerCache.go index 41fc42ef4d8..d45f855f80d 100644 --- a/network/p2p/p2pnode/protocolPeerCache.go +++ b/network/p2p/p2pnode/protocolPeerCache.go @@ -98,7 +98,7 @@ func (p *ProtocolPeerCache) consumeSubscription(logger zerolog.Logger, h host.Ho logger.Err(err).Str("peer", evt.Peer.String()).Msg("failed to get protocols for peer") continue } - p.AddProtocols(evt.Peer, protocol.ConvertFromStrings(protocols)) + p.AddProtocols(evt.Peer, protocols) case event.EvtPeerProtocolsUpdated: p.AddProtocols(evt.Peer, evt.Added) p.RemoveProtocols(evt.Peer, evt.Removed) diff --git a/network/p2p/p2pnode/protocolPeerCache_test.go b/network/p2p/p2pnode/protocolPeerCache_test.go index 7ff1896ef56..cc15d6cfc87 100644 --- a/network/p2p/p2pnode/protocolPeerCache_test.go +++ b/network/p2p/p2pnode/protocolPeerCache_test.go @@ -12,11 +12,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + fcrypto "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/network/p2p/p2pbuilder" "github.com/onflow/flow-go/network/p2p/p2pnode" - - fcrypto "github.com/onflow/flow-go/crypto" - "github.com/onflow/flow-go/utils/unittest" ) diff --git a/network/p2p/pubsub.go b/network/p2p/pubsub.go index d2e49420a3e..3e8e030ad8c 100644 --- a/network/p2p/pubsub.go +++ b/network/p2p/pubsub.go @@ -10,18 +10,13 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" + "github.com/onflow/flow-go/engine/collection" "github.com/onflow/flow-go/module/component" ) type ValidationResult int const ( - PublicNetworkEnabled = true - PublicNetworkDisabled = false - - MetricsEnabled = true - MetricsDisabled = false - ValidationAccept ValidationResult = iota ValidationIgnore ValidationReject @@ -32,6 +27,12 @@ type TopicValidatorFunc func(context.Context, peer.ID, *pubsub.Message) Validati // PubSubAdapter is the abstraction of the underlying pubsub logic that is used by the Flow network. type PubSubAdapter interface { component.Component + // CollectionClusterChangesConsumer is the interface for consuming the events of changes in the collection cluster. + // This is used to notify the node of changes in the collection cluster. + // PubSubAdapter implements this interface and consumes the events to be notified of changes in the clustering channels. + // The clustering channels are used by the collection nodes of a cluster to communicate with each other. + // As the cluster (and hence their cluster channels) of collection nodes changes over time (per epoch) the node needs to be notified of these changes. + CollectionClusterChangesConsumer // RegisterTopicValidator registers a validator for topic. RegisterTopicValidator(topic string, topicValidator TopicValidatorFunc) error @@ -52,6 +53,16 @@ type PubSubAdapter interface { // For example, if current peer has subscribed to topics A and B, then ListPeers only return // subscribed peers for topics A and B, and querying for topic C will return an empty list. ListPeers(topic string) []peer.ID + + // PeerScoreExposer returns the peer score exposer for the gossipsub adapter. The exposer is a read-only interface + // for querying peer scores and returns the local scoring table of the underlying gossipsub node. + // The exposer is only available if the gossipsub adapter was configured with a score tracer. + // If the gossipsub adapter was not configured with a score tracer, the exposer will be nil. + // Args: + // None. + // Returns: + // The peer score exposer for the gossipsub adapter. + PeerScoreExposer() PeerScoreExposer } // PubSubAdapterConfig abstracts the configuration for the underlying pubsub implementation. @@ -60,11 +71,11 @@ type PubSubAdapterConfig interface { WithSubscriptionFilter(SubscriptionFilter) WithScoreOption(ScoreOptionBuilder) WithMessageIdFunction(f func([]byte) string) - WithAppSpecificRpcInspectors(...GossipSubRPCInspector) WithTracer(t PubSubTracer) // WithScoreTracer sets the tracer for the underlying pubsub score implementation. // This is used to expose the local scoring table of the GossipSub node to its higher level components. WithScoreTracer(tracer PeerScoreTracer) + WithInspectorSuite(GossipSubInspectorSuite) } // GossipSubControlMetricsObserver funcs used to observe gossipsub related metrics. @@ -88,6 +99,18 @@ type GossipSubRPCInspector interface { Inspect(peer.ID, *pubsub.RPC) error } +// GossipSubMsgValidationRpcInspector abstracts the general behavior of an app specific RPC inspector specifically +// used to inspect and validate incoming. It is used to implement custom message validation logic. It is injected into +// the GossipSubRouter and run on every incoming RPC message before the message is processed by libp2p. If the message +// is invalid the RPC message will be dropped. +// Implementations must: +// - be concurrency safe +// - be non-blocking +type GossipSubMsgValidationRpcInspector interface { + collection.ClusterEvents + GossipSubRPCInspector +} + // Topic is the abstraction of the underlying pubsub topic that is used by the Flow network. type Topic interface { // String returns the topic name as a string. @@ -106,7 +129,10 @@ type Topic interface { // ScoreOptionBuilder abstracts the configuration for the underlying pubsub score implementation. type ScoreOptionBuilder interface { // BuildFlowPubSubScoreOption builds the pubsub score options as pubsub.Option for the Flow network. - BuildFlowPubSubScoreOption() pubsub.Option + BuildFlowPubSubScoreOption() (*pubsub.PeerScoreParams, *pubsub.PeerScoreThresholds) + // TopicScoreParams returns the topic score params for the given topic. + // If the topic score params for the given topic does not exist, it will return the default topic score params. + TopicScoreParams(*pubsub.Topic) *pubsub.TopicScoreParams } // Subscription is the abstraction of the underlying pubsub subscription that is used by the Flow network. diff --git a/network/p2p/scoring/README.md b/network/p2p/scoring/README.md new file mode 100644 index 00000000000..38a758db439 --- /dev/null +++ b/network/p2p/scoring/README.md @@ -0,0 +1,261 @@ +# GossipSub App Specific Score + +This package provides a scoring mechanism for peers in a GossipSub network by computing their application-specific scores. +Application-specific score is part of the GossipSub scoring mechanism, which is used to determine the behavior of peers in the network from +the perspective of their behavior at the application level (i.e., Flow protocol). +The score is determined based on a combination of penalties and rewards related to various factors, such as spamming misbehaviors, staking status, and valid subscriptions. + +## Key Components +1. `GossipSubAppSpecificScoreRegistry`: This struct maintains the necessary information for determining a peer's score. +2. `AppSpecificScoreFunc`: This function is exposed to GossipSub and calculates the application-specific score for a peer based on penalties and rewards. +3. `stakingScore`: This function computes the staking score (reward/penalty) for a peer based on their identity and role. +4. `subscriptionPenalty`: This function calculates the penalty for invalid subscriptions. +5. `OnInvalidControlMessageNotification`: This method updates a peer's penalty when an invalid control message misbehavior is detected, e.g., spamming on a control message. + +## Score Calculation +The application-specific score for a peer is calculated as the sum of the following factors: + +1. Spam Penalty: A penalty applied when a peer conducts a spamming misbehavior (e.g., GRAFT, PRUNE, iHave, or iWant misbehaviors). +2. Staking Penalty: A penalty applied for unknown peers with invalid Flow protocol identities. This ejects them from the GossipSub network. +3. Subscription Penalty: A penalty applied when a peer subscribes to a topic they are not allowed to, based on their role in the Flow network. +4. Staking Reward: A reward applied to well-behaved staked peers (excluding access nodes at the moment) only if they have no penalties from spamming or invalid subscriptions. + +The score is updated every time a peer misbehaves, and the spam penalties decay over time using the default decay function, which applies a geometric decay to the peer's score. + +### Usage +To use the scoring mechanism, create a new `GossipSubAppSpecificScoreRegistry` with the desired configuration, and then obtain the `AppSpecificScoreFunc` to be passed to the GossipSub protocol. + +Example: +```go +config := &GossipSubAppSpecificScoreRegistryConfig{ + // ... configure the required components +} +registry := NewGossipSubAppSpecificScoreRegistry(config) +appSpecificScoreFunc := registry.AppSpecificScoreFunc() + +// Use appSpecificScoreFunc as the score function for GossipSub +``` + +The scoring mechanism can be easily integrated with the GossipSub protocol to ensure that well-behaved peers are prioritized, and misbehaving peers are penalized. See the `ScoreOption` below for more details. + +**Note**: This package was designed specifically for the Flow network and might require adjustments if used in other contexts. + + +## Score Option +`ScoreOption` is a configuration object for the peer scoring system in the Flow network. +It defines several scoring parameters and thresholds that determine the behavior of the network towards its peers. +This includes rewarding well-behaving peers and penalizing misbehaving ones. + +**Note**: `ScoreOption` is passed to the GossipSub as a configuration option at the time of initialization. + +### Usage +To use the `ScoreOption`, you need to create a `ScoreOptionConfig` with the desired settings and then call `NewScoreOption` with that configuration. + +```go +config := NewScoreOptionConfig(logger) +config.SetProvider(identityProvider) +config.SetCacheSize(1000) +config.SetCacheMetrics(metricsCollector) + +// Optional: Set custom app-specific scoring function +config.SetAppSpecificScoreFunction(customAppSpecificScoreFunction) + +scoreOption := NewScoreOption(config) +``` + +### Scoring Parameters +`ScoreOption` provides a set of default scoring parameters and thresholds that can be configured through the `ScoreOptionConfig`. These parameters include: + +1. `AppSpecificScoreWeight`: The weight of the application-specific score in the overall peer score calculation at the GossipSub. +2. `GossipThreshold`: The threshold below which a peer's score will result in ignoring gossips to and from that peer. +3. `PublishThreshold`: The threshold below which a peer's score will result in not propagating self-published messages to that peer. +4. `GraylistThreshold`: The threshold below which a peer's score will result in ignoring incoming RPCs from that peer. +5. `AcceptPXThreshold`: The threshold above which a peer's score will result in accepting PX information with a prune from that peer. PX stands for "Peer Exchange" in the context of libp2p's gossipsub protocol. When a peer sends a PRUNE control message to another peer, it can include a list of other peers as PX information. The purpose of this is to help the pruned peer find new peers to replace the ones that have been pruned from its mesh. When a node receives a PRUNE message containing PX information, it can decide whether to connect to the suggested peers based on its own criteria. In this package, the `DefaultAcceptPXThreshold` is used to determine if the originating peer's penalty score is good enough to accept the PX information. If the originating peer's penalty score exceeds the threshold, the node will consider connecting to the suggested peers. +6. `OpportunisticGraftThreshold`: The threshold below which the median peer score in the mesh may result in selecting more peers with a higher score for opportunistic grafting. + +### Flow Specific Scoring Parameters and Thresholds +# GossipSub Scoring Parameters Explained +1. `DefaultAppSpecificScoreWeight = 1`: This is the default weight for application-specific scoring. It basically tells us how important the application-specific score is in comparison to other scores. +2. `MaxAppSpecificPenalty = -100` and `MinAppSpecificPenalty = -1`: These values define the range for application-specific penalties. A peer can have a maximum penalty of -100 and a minimum penalty of -1. +3. `MaxAppSpecificReward = 100`: This is the maximum reward a peer can earn for good behavior. +4. `DefaultStakedIdentityReward = MaxAppSpecificReward`: This reward is given to peers that contribute positively to the network (i.e., no misbehavior). It’s to encourage them and prioritize them in neighbor selection. +5. `DefaultUnknownIdentityPenalty = MaxAppSpecificPenalty`: This penalty is given to a peer if it's not in the identity list. It's to discourage anonymity. +6. `DefaultInvalidSubscriptionPenalty = MaxAppSpecificPenalty`: This penalty is for peers that subscribe to topics they are not authorized to subscribe to. +7. `DefaultGossipThreshold = -99`: If a peer's penalty goes below this threshold, the peer is ignored for gossip. It means no gossip is sent to or received from that peer. +8. `DefaultPublishThreshold = -99`: If a peer's penalty goes below this threshold, self-published messages will not be sent to this peer. +9. `DefaultGraylistThreshold = -99`: If a peer's penalty goes below this threshold, it is graylisted. This means all incoming messages from this peer are ignored. +10. `DefaultAcceptPXThreshold = 99`: This is a threshold for accepting peers. If a peer sends information and its score is above this threshold, the information is accepted. +11. `DefaultOpportunisticGraftThreshold = MaxAppSpecificReward + 1`: This value is used to selectively connect to new peers if the median score of the current peers drops below this threshold. +12. `defaultScoreCacheSize = 1000`: Sets the default size of the cache used to store the application-specific penalty of peers. +13. `defaultDecayInterval = 1 * time.Minute`: Sets the default interval at which the score of a peer will be decayed. +14. `defaultDecayToZero = 0.01`: This is a threshold below which a decayed score is reset to zero. It prevents the score from decaying to a very small value. +15. `defaultTopicTimeInMeshQuantum` is a parameter in the GossipSub scoring system that represents a fixed time interval used to count the amount of time a peer stays in a topic mesh. It is set to 1 hour, meaning that for each hour a peer remains in a topic mesh, its time-in-mesh counter increases by 1, contributing to its availability score. This is to reward peers that stay in the mesh for longer durations and discourage those that frequently join and leave. +16. `defaultTopicInvalidMessageDeliveriesWeight` is set to -1.0 and is used to penalize peers that send invalid messages by applying it to the square of the number of such messages. A message is considered invalid if it is not properly signed. A peer will be disconnected if it sends around 14 invalid messages within a gossipsub heartbeat interval. +17. `defaultTopicInvalidMessageDeliveriesDecay` is a decay factor set to 0.99. It is used to reduce the number of invalid message deliveries counted against a peer by 1% at each heartbeat interval. This prevents the peer from being disconnected if it stops sending invalid messages. The heartbeat interval in the gossipsub scoring system is set to 1 minute by default. + +## GossipSub Message Delivery Scoring +This section provides an overview of the GossipSub message delivery scoring mechanism used in the Flow network. +It's designed to maintain an efficient, secure and stable peer-to-peer network by scoring each peer based on their message delivery performance. +The system ensures the reliability of message propagation by scoring peers, which discourages malicious behaviors and enhances overall network performance. + +### Comprehensive System Overview +The GossipSub message delivery scoring mechanism used in the Flow network is an integral component of its P2P communication model. +It is designed to monitor and incentivize appropriate network behaviors by attributing scores to peers based on their message delivery performance. +This scoring system is fundamental to ensure that messages are reliably propagated across the network, creating a robust P2P communication infrastructure. + +The scoring system is per topic, which means it tracks the efficiency of peers in delivering messages in each specific topic they are participating in. +These per-topic scores then contribute to an overall score for each peer, providing a comprehensive view of a peer's effectiveness within the network. +In GossipSub, a crucial aspect of a peer's responsibility is to relay messages effectively to other nodes in the network. +The role of the scoring mechanism is to objectively assess a peer's efficiency in delivering these messages. +It takes into account several factors to determine the effectiveness of the peers. + +1. **Message Delivery Rate** - A peer's ability to deliver messages quickly is a vital metric. Slow delivery could lead to network lags and inefficiency. +2. **Message Delivery Volume** - A peer's capacity to deliver a large number of messages accurately and consistently. +3. **Continuity of Performance** - The scoring mechanism tracks not only the rate and volume of the messages but also the consistency in a peer's performance over time. +4. **Prevention of Malicious Behaviors** - The scoring system also helps in mitigating potential network attacks such as spamming and message replay attacks. + +The system utilizes several parameters to maintain and adjust the scores of the peers: +- `defaultTopicMeshMessageDeliveriesDecay`(value: 0.5): This parameter dictates how rapidly a peer's message delivery count decays with time. With a value of 0.5, it indicates a 50% decay at each decay interval. This mechanism ensures that past performances do not disproportionately impact the current score of the peer. +- `defaultTopicMeshMessageDeliveriesCap` (value: 1000): This parameter sets an upper limit on the number of message deliveries that can contribute to the score of a peer in a topic. With a cap set at 1000, it prevents the score from being overly influenced by large volumes of message deliveries, providing a balanced assessment of peer performance. +- `defaultTopicMeshMessageDeliveryThreshold` (value: 0.1 * `defaultTopicMeshMessageDeliveriesCap`): This threshold serves to identify under-performing peers. If a peer's message delivery count is below this threshold in a topic, the peer's score is penalized. This encourages peers to maintain a minimum level of performance. +- `defaultTopicMeshMessageDeliveriesWeight` (value: -0.05 * `MaxAppSpecificReward` / (`defaultTopicMeshMessageDeliveryThreshold` ^ 2) = 5^-4): This weight is applied when penalizing under-performing peers. The penalty is proportional to the square of the difference between the actual message deliveries and the threshold, multiplied by this weight. +- `defaultMeshMessageDeliveriesWindow` (value: `defaultDecayInterval` = 1 minute): This parameter defines the time window within which a message delivery is counted towards the score. This window is set to the decay interval, preventing replay attacks and counting only unique message deliveries. +- `defaultMeshMessageDeliveriesActivation` (value: 2 * `defaultDecayInterval` = 2 minutes): This time interval is the grace period before the scoring system starts tracking a new peer's performance. It accounts for the time it takes for a new peer to fully integrate into the network. + +By continually updating and adjusting the scores of peers based on these parameters, the GossipSub message delivery scoring mechanism ensures a robust, efficient, and secure P2P network. + +### Examples + +#### Scenario 1: Peer A Delivers Messages Within Cap and Above Threshold +Let's assume a Peer A that consistently delivers 500 messages per decay interval. This is within the `defaultTopicMeshMessageDeliveriesCap` (1000) and above the `defaultTopicMeshMessageDeliveryThreshold` (100). +As Peer A's deliveries are above the threshold and within the cap, its score will not be penalized. Instead, it will be maintained, promoting healthy network participation. + +#### Scenario 2: Peer B Delivers Messages Below Threshold +Now, assume Peer B delivers 50 messages per decay interval, below the `defaultTopicMeshMessageDeliveryThreshold` (100). +In this case, the score of Peer B will be penalized because its delivery rate is below the threshold. The penalty is calculated as `-|w| * (actual - threshold)^2`, where `w` is the weight (`defaultTopicMeshMessageDeliveriesWeight`), `actual` is the actual messages delivered (50), and `threshold` is the delivery threshold (100). + +#### Scenario 3: Peer C Delivers Messages Exceeding the Cap +Consider Peer C, which delivers 1500 messages per decay interval, exceeding the `defaultTopicMeshMessageDeliveriesCap` (1000). +In this case, even though Peer C is highly active, its score will not increase further once it hits the cap (1000). This is to avoid overemphasis on high delivery counts, which could skew the scoring system. + +#### Scenario 4: Peer D Joins a Topic Mesh +When a new Peer D joins a topic mesh, it will be given a grace period of `defaultMeshMessageDeliveriesActivation` (2 decay intervals) before its message delivery performance is tracked. This grace period allows the peer to set up and begin receiving messages from the network. +Remember, the parameters and scenarios described here aim to maintain a stable, efficient, and secure peer-to-peer network by carefully tracking and scoring each peer's message delivery performance. + +#### Scenario 5: Message Delivery Decay +To better understand how the message delivery decay (`defaultTopicMeshMessageDeliveriesDecay`) works in the GossipSub protocol, let's examine a hypothetical scenario. +Let's say we have a peer named `Peer A` who is actively participating in `Topic X`. `Peer A` has successfully delivered 800 messages in `Topic X` over a given time period. +**Initial State**: At this point, `Peer A`'s message delivery count for `Topic X` is 800. Now, the decay interval elapses without `Peer A` delivering any new messages in `Topic X`. +**After One Decay Interval**: Given that our `defaultTopicMeshMessageDeliveriesDecay` value is 0.5, after one decay interval, `Peer A`'s message delivery count for `Topic X` will decay by 50%. Therefore, `Peer A`'s count is now: + + 800 (previous message count) * 0.5 (decay factor) = 400 + +**After Two Decay Intervals** +If `Peer A` still hasn't delivered any new messages in `Topic X` during the next decay interval, the decay is applied again, further reducing the message delivery count: + + 400 (current message count) * 0.5 (decay factor) = 200 +And this process will continue at every decay interval, halving `Peer A`'s message delivery count for `Topic X` until `Peer A` delivers new messages in `Topic X` or the count reaches zero. +This decay process ensures that a peer cannot rest on its past deliveries; it must continually contribute to the network to maintain its score. +It helps maintain a lively and dynamic network environment, incentivizing constant active participation from all peers. + +### Scenario 6: Replay Attack +The `defaultMeshMessageDeliveriesWindow` and `defaultMeshMessageDeliveriesActivation` parameters play a crucial role in preventing replay attacks in the GossipSub protocol. Let's illustrate this with an example. +Consider a scenario where we have three peers: `Peer A`, `Peer B`, and `Peer C`. All three peers are active participants in `Topic X`. +**Initial State**: At Time = 0: `Peer A` generates and broadcasts a new message `M` in `Topic X`. `Peer B` and `Peer C` receive this message from `Peer A` and update their message caches accordingly. +**After Few Seconds**: At Time = 30 seconds: `Peer B`, with malicious intent, tries to rebroadcast the same message `M` back into `Topic X`. +Given that our `defaultMeshMessageDeliveriesWindow` value is equal to the decay interval (let's assume 1 minute), `Peer C` would have seen the original message `M` from `Peer A` less than one minute ago. +This is within the `defaultMeshMessageDeliveriesWindow`. Because `Peer A` (the original sender) is different from `Peer B` (the current sender), this delivery will be counted towards `Peer B`'s message delivery score in `Topic X`. +**After One Minute**: At Time = 61 seconds: `Peer B` tries to rebroadcast the same message `M` again. +Now, more than a minute has passed since `Peer C` first saw the message `M` from `Peer A`. This is outside the `defaultMeshMessageDeliveriesWindow`. +Therefore, the message `M` from `Peer B` will not count towards `Peer B`'s message delivery score in `Topic X` and `Peer B` still needs to fill up its threshold of message delivery in order not to be penalized for under-performing. +This effectively discouraging replay attacks of messages older than the `defaultMeshMessageDeliveriesWindow`. +This mechanism, combined with other parameters, helps maintain the security and efficiency of the network by discouraging harmful behaviors such as message replay attacks. + +## Mitigating iHave Broken Promises Attacks in GossipSub Protocol +### What is an iHave Broken Promise Attack? +In the GossipSub protocol, peers gossip information about new messages to a subset of random peers (out of their local mesh) in the form of an "iHave" message which basically tells the receiving peer what messages the sender has. +The receiving peer then replies with an "iWant" message, requesting for the messages it doesn't have. Note that for the peers in local mesh the actual new messages are sent instead of an "iHave" message (i.e., eager push). However, +iHave-iWant protocol is part of a complementary mechanism to ensure that the information is disseminated to the entire network in a timely manner (i.e., lazy pull). + +An "iHave Broken Promise" attack occurs when a peer advertises many "iHave" for a message but doesn't respond to the "iWant" requests for those messages. +This not only hinders the effective dissemination of information but can also strain the network with redundant requests. Hence, we stratify it as a spam behavior mounting a DoS attack on the network. + +### Detecting iHave Broken Promise Attacks +Detecting iHave Broken Promise Attacks is done by the GossipSub itself. On each incoming RPC from a remote node, the local GossipSub node checks if the RPC contains an iHave message. It then samples one (and only one) iHave message +randomly out of the entire set of iHave messages piggybacked on the incoming RPC. If the sampled iHave message is not literally addressed with the actual message, the local GossipSub node considers this as an iHave broken promise and +increases the behavior penalty counter for that remote node. Hence, incrementing the behavior penalty counter for a remote peer is done per RPC containing at least one iHave broken promise and not per iHave message. +Note that the behavior penalty counter also keeps track of GRAFT flood attacks that are done by a remote peer when it advertises many GRAFTs while it is on a PRUNE backoff by the local node. Mitigating iHave broken promise attacks also +mitigates GRAFT flood attacks. + +### Configuring GossipSub Parameters +In order to mitigate the iHave broken promises attacks, GossipSub expects the application layer (i.e., Flow protocol) to properly configure the relevant scoring parameters, notably: + +- `BehaviourPenaltyThreshold` is set to `defaultBehaviourPenaltyThreshold`, i.e., `10`. +- `BehaviourPenaltyWeight` is set to `defaultBehaviourPenaltyWeight`, i.e., `0.01` * `MaxAppSpecificPenalty` +- `BehaviourPenaltyDecay` is set to `defaultBehaviourPenaltyDecay`, i.e., `0.99`. + +#### 1. `defaultBehaviourPenaltyThreshold` +This parameter sets the threshold for when the behavior of a peer is considered bad. Misbehavior is defined as advertising an iHave without responding to the iWants (iHave broken promises), and attempting on GRAFT when the peer is considered for a PRUNE backoff. +If a remote peer sends an RPC that advertises at least one iHave for a message but doesn't respond to the iWant requests for that message within the next `3 seconds`, the peer misbehavior counter is incremented by `1`. This threshold is set to `10`, meaning that we at most tolerate 10 such RPCs containing iHave broken promises. After this, the peer is penalized for every excess RPC containing iHave broken promises. The counter decays by (`0.99`) every decay interval (defaultDecayInterval) i.e., every minute. + +#### 2. `defaultBehaviourPenaltyWeight` +This is the weight applied as a penalty when a peer's misbehavior goes beyond the `defaultBehaviourPenaltyThreshold`. +The penalty is applied to the square of the difference between the misbehavior counter and the threshold, i.e., -|w| * (misbehavior counter - threshold)^2, where `|w|` is the absolute value of the `defaultBehaviourPenaltyWeight`. +Note that `defaultBehaviourPenaltyWeight` is a negative value, meaning that the penalty is applied in the opposite direction of the misbehavior counter. For sake of illustration, we use the notion of `-|w|` to denote that a negative penalty is applied. +We set `defaultBehaviourPenaltyWeight` to `0.01 * MaxAppSpecificPenalty`, meaning a peer misbehaving `10` times more than the threshold (i.e., `10 + 10`) will lose its entire `MaxAppSpecificReward`, which is a reward given to all staked nodes in Flow blockchain. +This also means that a peer misbehaving `sqrt(2) * 10` times more than the threshold will cause the peer score to be dropped below the `MaxAppSpecificPenalty`, which is also below the `GraylistThreshold`, and the peer will be graylisted (i.e., all incoming and outgoing GossipSub RPCs from and to that peer will be rejected). +This means the peer is temporarily disconnected from the network, preventing it from causing further harm. + +#### 3. defaultBehaviourPenaltyDecay +This is the decay interval for the misbehavior counter of a peer. This counter is decayed by the `defaultBehaviourPenaltyDecay` parameter (e.g., `0.99`) per decay interval, which is currently every 1 minute. +This parameter helps to gradually reduce the effect of past misbehaviors and provides a chance for penalized nodes to rejoin the network. A very slow decay rate can help identify and isolate persistent offenders, while also allowing potentially honest nodes that had transient issues to regain their standing in the network. +The duration a peer remains graylisted is governed by the choice of `defaultBehaviourPenaltyWeight` and the decay parameters. +Based on the given configuration, a peer which has misbehaved on `sqrt(2) * 10` RPCs more than the threshold will get graylisted (disconnected at GossipSub level). +With the decay interval set to 1 minute and decay value of 0.99, a graylisted peer due to broken promises would be expected to be reconnected in about 50 minutes. +This is calculated by solving for `x` in the equation `(0.99)^x * (sqrt(2) * 10)^2 * MaxAppSpecificPenalty > GraylistThreshold`. +Simplifying, we find `x` to be approximately `527` decay intervals, or roughly `527` minutes. +This is the estimated time it would take for a severely misbehaving peer to have its penalty decayed enough to exceed the `GraylistThreshold` and thus be reconnected to the network. + +### Example Scenarios +**Scenario 1: Misbehaving Below Threshold** +In this scenario, consider peer `B` that has recently joined the network and is taking part in GossipSub. +This peer advertises to peer `A` many `iHave` messages over an RPC. But when other peer `A` requests these message with `iWant`s it fails to deliver the message within 3 seconds. +This action constitutes an _iHave broken promise_ for a single RPC and peer `A` increases the local behavior penalty counter of peer `B` by 1. +If the peer `B` commits this misbehavior infrequently, such that the total number of these RPCs does not exceed the `defaultBehaviourPenaltyThreshold` (set to 10 in our configuration), +the misbehavior counter for this peer will increment by 1 for each RPC and decays by `1%` evey decay interval (1 minute), but no additional penalty will be applied. +The misbehavior counter decays by a factor of `defaultBehaviourPenaltyDecay` (0.99) every minute, allowing the peer to recover from these minor infractions without significant disruption. + +**Scenario 2: Misbehaving Above Threshold But Below Graylisting** +Now consider that peer `B` frequently sends RPCs advertising many `iHaves` to peer `A` but fails to deliver the promised messages. +If the number of these misbehaviors exceeds our threshold (10 in our configuration), the peer `B` is now penalized by the local GossipSub mechanism of peer `A`. +The amount of the penalty is determined by the `defaultBehaviourPenaltyWeight` (set to 0.01 * MaxAppSpecificPenalty) applied to the square of the difference between the misbehavior counter and the threshold. +This penalty will progressively affect the peer's score, deteriorating its reputation in the local GossipSub scoring system of node `A`, but does not yet result in disconnection or graylisting. +The peer has a chance to amend its behavior before crossing into graylisting territory through stop misbehaving and letting the score to decay. +When peer `B` has a deteriorated score at node `A`, it will be less likely to be selected by node `A` as its local mesh peer (i.e., to directly receive new messages from node `A`), and is deprived of the opportunity to receive new messages earlier through node `A`. + +**Scenario 3: Graylisting** +Now assume that peer `B` peer has been continually misbehaving, with RPCs including iHave broken promises exceeding `sqrt(2) * 10` the threshold. +At this point, the peer's score drops below the `GraylistThreshold` due to the `defaultBehaviorPenaltyWeight` applied to the excess misbehavior. +The peer is then graylisted by peer `A`, i.e., peer `A` rejects all incoming RPCs to and from peer `B` at GossipSub level. +In our configuration, peer `B` will stay disconnected for at least `527` decay intervals or approximately `527` minutes. +This gives a strong disincentive for the peer to continue this behavior and also gives it time to recover and eventually be reconnected to the network. + +## Customization +The scoring mechanism can be easily customized to suit the needs of the Flow network. This includes changing the scoring parameters, thresholds, and the scoring function itself. +You can customize the scoring parameters and thresholds by using the various setter methods provided in the `ScoreOptionConfig` object. Additionally, you can provide a custom app-specific scoring function through the `SetAppSpecificScoreFunction` method. + +**Note**: Usage of non-default app-specific scoring function is not recommended unless you are familiar with the scoring mechanism and the Flow network. It may result in _routing attack vulnerabilities_. It is **always safer** to use the default scoring function unless you know what you are doing. + +Example of setting custom app-specific scoring function: +```go +config.SetAppSpecificScoreFunction(customAppSpecificScoreFunction) +``` + +## Peer Scoring System Integration +The peer scoring system is integrated with the GossipSub protocol through the `ScoreOption` configuration option. +This option is passed to the GossipSub at the time of initialization. +`ScoreOption` can be used to build scoring options for GossipSub protocol with the desired scoring parameters and thresholds. +```go +flowPubSubOption := scoreOption.BuildFlowPubSubScoreOption() +gossipSubOption := scoreOption.BuildGossipSubScoreOption() +``` \ No newline at end of file diff --git a/network/p2p/scoring/app_score_test.go b/network/p2p/scoring/app_score_test.go index 26ab7da5e36..8e2a1ae1bb8 100644 --- a/network/p2p/scoring/app_score_test.go +++ b/network/p2p/scoring/app_score_test.go @@ -33,14 +33,17 @@ func TestFullGossipSubConnectivity(t *testing.T) { // two groups of non-access nodes and one group of access nodes. groupOneNodes, groupOneIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, + idProvider, p2ptest.WithRole(flow.RoleConsensus), - p2ptest.WithPeerScoringEnabled(idProvider)) + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) groupTwoNodes, groupTwoIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, + idProvider, p2ptest.WithRole(flow.RoleCollection), - p2ptest.WithPeerScoringEnabled(idProvider)) + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) accessNodeGroup, accessNodeIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, + idProvider, p2ptest.WithRole(flow.RoleAccess), - p2ptest.WithPeerScoringEnabled(idProvider)) + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) ids := append(append(groupOneIds, groupTwoIds...), accessNodeIds...) nodes := append(append(groupOneNodes, groupTwoNodes...), accessNodeGroup...) @@ -147,22 +150,23 @@ func testGossipSubMessageDeliveryUnderNetworkPartition(t *testing.T, honestPeerS // two (honest) consensus nodes opts := []p2ptest.NodeFixtureParameterOption{p2ptest.WithRole(flow.RoleConsensus)} if honestPeerScoring { - opts = append(opts, p2ptest.WithPeerScoringEnabled(idProvider)) + opts = append(opts, p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) } - con1Node, con1Id := p2ptest.NodeFixture(t, sporkId, t.Name(), opts...) - con2Node, con2Id := p2ptest.NodeFixture(t, sporkId, t.Name(), opts...) + con1Node, con1Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, opts...) + con2Node, con2Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, opts...) // create > 2 * 12 malicious access nodes // 12 is the maximum size of default GossipSub mesh. // We want to make sure that it is unlikely for honest nodes to be in the same mesh (hence messages from // one honest node to the other is routed through the malicious nodes). accessNodeGroup, accessNodeIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 30, + idProvider, p2ptest.WithRole(flow.RoleAccess), - p2ptest.WithPeerScoringEnabled(idProvider), // overrides the default peer scoring parameters to mute GossipSub traffic from/to honest nodes. - p2ptest.WithPeerScoreParamsOption(&p2p.PeerScoringConfig{ + p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ AppSpecificScoreParams: maliciousAppSpecificScore(flow.IdentityList{&con1Id, &con2Id}), - })) + }), + ) allNodes := append([]p2p.LibP2PNode{con1Node, con2Node}, accessNodeGroup...) allIds := append([]*flow.Identity{&con1Id, &con2Id}, accessNodeIds...) @@ -217,7 +221,7 @@ func testGossipSubMessageDeliveryUnderNetworkPartition(t *testing.T, honestPeerS return p2pfixtures.HasSubReceivedMessage(t, ctx1s, proposalMsg, con2Sub) } -// maliciousAppSpecificScore returns a malicious app specific score function that rewards the malicious node and +// maliciousAppSpecificScore returns a malicious app specific penalty function that rewards the malicious node and // punishes the honest nodes. func maliciousAppSpecificScore(honestIds flow.IdentityList) func(peer.ID) float64 { honestIdProvider := id.NewFixedIdentityProvider(honestIds) diff --git a/network/p2p/scoring/decay.go b/network/p2p/scoring/decay.go new file mode 100644 index 00000000000..22665bde690 --- /dev/null +++ b/network/p2p/scoring/decay.go @@ -0,0 +1,40 @@ +package scoring + +import ( + "fmt" + "math" + "time" +) + +// GeometricDecay returns the decayed score based on the decay factor and the time since the last update. +// +// The decayed score is calculated as follows: +// penalty = score * decay^t where t is the time since the last update in seconds. +// Args: +// - score: the score to be decayed. +// - decay: the decay factor, it should be in the range of (0, 1]. +// - lastUpdated: the time when the penalty was last updated. +// Returns: +// - the decayed score. +// - an error if the decay factor is not in the range of (0, 1] or the decayed score is NaN. +// it also returns an error if the last updated time is in the future (to avoid overflow). +// The error is considered irrecoverable (unless the parameters can be adjusted). +func GeometricDecay(score float64, decay float64, lastUpdated time.Time) (float64, error) { + if decay <= 0 || decay > 1 { + return 0.0, fmt.Errorf("decay factor must be in the range (0, 1], got %f", decay) + } + + now := time.Now() + if lastUpdated.After(now) { + return 0.0, fmt.Errorf("last updated time is in the future %v now: %v", lastUpdated, now) + } + + t := time.Since(lastUpdated).Seconds() + decayFactor := math.Pow(decay, t) + + if math.IsNaN(decayFactor) { + return 0.0, fmt.Errorf("decay factor is NaN for %f^%f", decay, t) + } + + return score * decayFactor, nil +} diff --git a/network/p2p/scoring/decay_test.go b/network/p2p/scoring/decay_test.go new file mode 100644 index 00000000000..28ff6dabe7f --- /dev/null +++ b/network/p2p/scoring/decay_test.go @@ -0,0 +1,252 @@ +package scoring_test + +import ( + "fmt" + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/scoring" +) + +// TestGeometricDecay tests the GeometricDecay function. +func TestGeometricDecay(t *testing.T) { + type args struct { + penalty float64 + decay float64 + lastUpdated time.Time + } + tests := []struct { + name string + args args + want float64 + wantErr error + }{ + { + name: "valid penalty, decay, and time", + args: args{ + penalty: 100, + decay: 0.9, + lastUpdated: time.Now().Add(-10 * time.Second), + }, + want: 100 * math.Pow(0.9, 10), + wantErr: nil, + }, + { + name: "zero decay factor", + args: args{ + penalty: 100, + decay: 0, + lastUpdated: time.Now(), + }, + want: 0, + wantErr: fmt.Errorf("decay factor must be in the range [0, 1], got 0"), + }, + { + name: "decay factor of 1", + args: args{ + penalty: 100, + decay: 1, + lastUpdated: time.Now().Add(-10 * time.Second), + }, + want: 100, + wantErr: nil, + }, + { + name: "negative decay factor", + args: args{ + penalty: 100, + decay: -0.5, + lastUpdated: time.Now(), + }, + want: 0, + wantErr: fmt.Errorf("decay factor must be in the range [0, 1], got %f", -0.5), + }, + { + name: "decay factor greater than 1", + args: args{ + penalty: 100, + decay: 1.2, + lastUpdated: time.Now(), + }, + want: 0, + wantErr: fmt.Errorf("decay factor must be in the range [0, 1], got %f", 1.2), + }, + { + name: "large time value causing overflow", + args: args{ + penalty: 100, + decay: 0.999999999999999, + lastUpdated: time.Now().Add(-1e5 * time.Second), + }, + want: 100 * math.Pow(0.999999999999999, 1e5), + wantErr: nil, + }, + { + name: "large decay factor and time value causing underflow", + args: args{ + penalty: 100, + decay: 0.999999, + lastUpdated: time.Now().Add(-1e9 * time.Second), + }, + want: 0, + wantErr: nil, + }, + { + name: "very small decay factor and time value causing underflow", + args: args{ + penalty: 100, + decay: 0.000001, + lastUpdated: time.Now().Add(-1e9 * time.Second), + }, + want: 0, + wantErr: nil, + }, + { + name: "future time value causing an error", + args: args{ + penalty: 100, + decay: 0.999999, + lastUpdated: time.Now().Add(+1e9 * time.Second), + }, + want: 0, + wantErr: fmt.Errorf("last updated time cannot be in the future"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := scoring.GeometricDecay(tt.args.penalty, tt.args.decay, tt.args.lastUpdated) + if tt.wantErr != nil { + assert.Errorf(t, err, tt.wantErr.Error()) + } + assert.Less(t, math.Abs(got-tt.want), 1e-3) + }) + } +} + +// TestDefaultDecayFunction tests the default decay function. +// The default decay function is used when no custom decay function is provided. +// The test evaluates the following cases: +// 1. penalty is non-negative and should not be decayed. +// 2. penalty is negative and above the skipDecayThreshold and lastUpdated is too recent. In this case, the penalty should not be decayed. +// 3. penalty is negative and above the skipDecayThreshold and lastUpdated is too old. In this case, the penalty should not be decayed. +// 4. penalty is negative and below the skipDecayThreshold and lastUpdated is too recent. In this case, the penalty should not be decayed. +// 5. penalty is negative and below the skipDecayThreshold and lastUpdated is too old. In this case, the penalty should be decayed. +func TestDefaultDecayFunction(t *testing.T) { + type args struct { + record p2p.GossipSubSpamRecord + lastUpdated time.Time + } + + type want struct { + record p2p.GossipSubSpamRecord + } + + tests := []struct { + name string + args args + want want + }{ + { + // 1. penalty is non-negative and should not be decayed. + name: "penalty is non-negative", + args: args{ + record: p2p.GossipSubSpamRecord{ + Penalty: 5, + Decay: 0.8, + }, + lastUpdated: time.Now(), + }, + want: want{ + record: p2p.GossipSubSpamRecord{ + Penalty: 5, + Decay: 0.8, + }, + }, + }, + { + // 2. penalty is negative and above the skipDecayThreshold and lastUpdated is too recent. In this case, the penalty should not be decayed, + // since less than a second has passed since last update. + name: "penalty is negative and but above skipDecayThreshold and lastUpdated is too recent", + args: args{ + record: p2p.GossipSubSpamRecord{ + Penalty: -0.09, // -0.09 is above skipDecayThreshold of -0.1 + Decay: 0.8, + }, + lastUpdated: time.Now(), + }, + want: want{ + record: p2p.GossipSubSpamRecord{ + Penalty: 0, // penalty is set to 0 + Decay: 0.8, + }, + }, + }, + { + // 3. penalty is negative and above the skipDecayThreshold and lastUpdated is too old. In this case, the penalty should not be decayed, + // since penalty is between [skipDecayThreshold, 0] and more than a second has passed since last update. + name: "penalty is negative and but above skipDecayThreshold and lastUpdated is too old", + args: args{ + record: p2p.GossipSubSpamRecord{ + Penalty: -0.09, // -0.09 is above skipDecayThreshold of -0.1 + Decay: 0.8, + }, + lastUpdated: time.Now().Add(-10 * time.Second), + }, + want: want{ + record: p2p.GossipSubSpamRecord{ + Penalty: 0, // penalty is set to 0 + Decay: 0.8, + }, + }, + }, + { + // 4. penalty is negative and below the skipDecayThreshold and lastUpdated is too recent. In this case, the penalty should not be decayed, + // since less than a second has passed since last update. + name: "penalty is negative and below skipDecayThreshold but lastUpdated is too recent", + args: args{ + record: p2p.GossipSubSpamRecord{ + Penalty: -5, + Decay: 0.8, + }, + lastUpdated: time.Now(), + }, + want: want{ + record: p2p.GossipSubSpamRecord{ + Penalty: -5, + Decay: 0.8, + }, + }, + }, + { + // 5. penalty is negative and below the skipDecayThreshold and lastUpdated is too old. In this case, the penalty should be decayed. + name: "penalty is negative and below skipDecayThreshold but lastUpdated is too old", + args: args{ + record: p2p.GossipSubSpamRecord{ + Penalty: -15, + Decay: 0.8, + }, + lastUpdated: time.Now().Add(-10 * time.Second), + }, + want: want{ + record: p2p.GossipSubSpamRecord{ + Penalty: -15 * math.Pow(0.8, 10), + Decay: 0.8, + }, + }, + }, + } + + decayFunc := scoring.DefaultDecayFunction() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := decayFunc(tt.args.record, tt.args.lastUpdated) + assert.NoError(t, err) + assert.Less(t, math.Abs(got.Penalty-tt.want.record.Penalty), 10e-3) + assert.Equal(t, got.Decay, tt.want.record.Decay) + }) + } +} diff --git a/network/p2p/scoring/invalid_subscription_error.go b/network/p2p/scoring/invalid_subscription_error.go deleted file mode 100644 index b7b941d49c7..00000000000 --- a/network/p2p/scoring/invalid_subscription_error.go +++ /dev/null @@ -1,26 +0,0 @@ -package scoring - -import ( - "errors" - "fmt" -) - -// InvalidSubscriptionError indicates that a peer has subscribed to a topic that is not allowed for its role. -type InvalidSubscriptionError struct { - topic string // the topic that the peer is subscribed to, but not allowed to. -} - -func NewInvalidSubscriptionError(topic string) error { - return InvalidSubscriptionError{ - topic: topic, - } -} - -func (e InvalidSubscriptionError) Error() string { - return fmt.Sprintf("unauthorized subscription: %s", e.topic) -} - -func IsInvalidSubscriptionError(this error) bool { - var e InvalidSubscriptionError - return errors.As(this, &e) -} diff --git a/network/p2p/scoring/registry.go b/network/p2p/scoring/registry.go new file mode 100644 index 00000000000..9009b86f41a --- /dev/null +++ b/network/p2p/scoring/registry.go @@ -0,0 +1,319 @@ +package scoring + +import ( + "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network/p2p" + netcache "github.com/onflow/flow-go/network/p2p/cache" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" + "github.com/onflow/flow-go/utils/logging" +) + +const ( + // skipDecayThreshold is the threshold for which when the negative penalty is above this value, the decay function will not be called. + // instead, the penalty will be set to 0. This is to prevent the penalty from keeping a small negative value for a long time. + skipDecayThreshold = -0.1 + // defaultDecay is the default decay value for the application specific penalty. + // this value is used when no custom decay value is provided, and decays the penalty by 1% every second. + // assume: + // penalty = -100 (the maximum application specific penalty is -100) + // skipDecayThreshold = -0.1 + // it takes around 459 seconds for the penalty to decay to reach greater than -0.1 and turn into 0. + // x * 0.99 ^ n > -0.1 (assuming negative x). + // 0.99 ^ n > -0.1 / x + // Now we can take the logarithm of both sides (with any base, but let's use base 10 for simplicity). + // log( 0.99 ^ n ) < log( 0.1 / x ) + // Using the properties of logarithms, we can bring down the exponent: + // n * log( 0.99 ) < log( -0.1 / x ) + // And finally, we can solve for n: + // n > log( -0.1 / x ) / log( 0.99 ) + // We can plug in x = -100: + // n > log( -0.1 / -100 ) / log( 0.99 ) + // n > log( 0.001 ) / log( 0.99 ) + // n > -3 / log( 0.99 ) + // n > 458.22 + defaultDecay = 0.99 // default decay value for the application specific penalty. + // graftMisbehaviourPenalty is the penalty applied to the application specific penalty when a peer conducts a graft misbehaviour. + graftMisbehaviourPenalty = -10 + // pruneMisbehaviourPenalty is the penalty applied to the application specific penalty when a peer conducts a prune misbehaviour. + pruneMisbehaviourPenalty = -10 + // iHaveMisbehaviourPenalty is the penalty applied to the application specific penalty when a peer conducts a iHave misbehaviour. + iHaveMisbehaviourPenalty = -10 + // iWantMisbehaviourPenalty is the penalty applied to the application specific penalty when a peer conducts a iWant misbehaviour. + iWantMisbehaviourPenalty = -10 +) + +// GossipSubCtrlMsgPenaltyValue is the penalty value for each control message type. +type GossipSubCtrlMsgPenaltyValue struct { + Graft float64 // penalty value for an individual graft message misbehaviour. + Prune float64 // penalty value for an individual prune message misbehaviour. + IHave float64 // penalty value for an individual iHave message misbehaviour. + IWant float64 // penalty value for an individual iWant message misbehaviour. +} + +// DefaultGossipSubCtrlMsgPenaltyValue returns the default penalty value for each control message type. +func DefaultGossipSubCtrlMsgPenaltyValue() GossipSubCtrlMsgPenaltyValue { + return GossipSubCtrlMsgPenaltyValue{ + Graft: graftMisbehaviourPenalty, + Prune: pruneMisbehaviourPenalty, + IHave: iHaveMisbehaviourPenalty, + IWant: iWantMisbehaviourPenalty, + } +} + +// GossipSubAppSpecificScoreRegistry is the registry for the application specific score of peers in the GossipSub protocol. +// The application specific score is part of the overall score of a peer, and is used to determine the peer's score based +// on its behavior related to the application (Flow protocol). +// This registry holds the view of the local peer of the application specific score of other peers in the network based +// on what it has observed from the network. +// Similar to the GossipSub score, the application specific score is meant to be private to the local peer, and is not +// shared with other peers in the network. +type GossipSubAppSpecificScoreRegistry struct { + logger zerolog.Logger + idProvider module.IdentityProvider + // spamScoreCache currently only holds the control message misbehaviour penalty (spam related penalty). + spamScoreCache p2p.GossipSubSpamRecordCache + penalty GossipSubCtrlMsgPenaltyValue + // initial application specific penalty record, used to initialize the penalty cache entry. + init func() p2p.GossipSubSpamRecord + validator p2p.SubscriptionValidator +} + +// GossipSubAppSpecificScoreRegistryConfig is the configuration for the GossipSubAppSpecificScoreRegistry. +// The configuration is used to initialize the registry. +type GossipSubAppSpecificScoreRegistryConfig struct { + Logger zerolog.Logger + + // Validator is the subscription validator used to validate the subscriptions of peers, and determine if a peer is + // authorized to subscribe to a topic. + Validator p2p.SubscriptionValidator + + // Penalty encapsulates the penalty unit for each control message type misbehaviour. + Penalty GossipSubCtrlMsgPenaltyValue + + // IdProvider is the identity provider used to translate peer ids at the networking layer to Flow identifiers (if + // an authorized peer is found). + IdProvider module.IdentityProvider + + // Init is a factory function that returns a new GossipSubSpamRecord. It is used to initialize the spam record of + // a peer when the peer is first observed by the local peer. + Init func() p2p.GossipSubSpamRecord + + // CacheFactory is a factory function that returns a new GossipSubSpamRecordCache. It is used to initialize the spamScoreCache. + // The cache is used to store the application specific penalty of peers. + CacheFactory func() p2p.GossipSubSpamRecordCache +} + +// NewGossipSubAppSpecificScoreRegistry returns a new GossipSubAppSpecificScoreRegistry. +// Args: +// +// config: the configuration for the registry. +// +// Returns: +// +// a new GossipSubAppSpecificScoreRegistry. +func NewGossipSubAppSpecificScoreRegistry(config *GossipSubAppSpecificScoreRegistryConfig) *GossipSubAppSpecificScoreRegistry { + reg := &GossipSubAppSpecificScoreRegistry{ + logger: config.Logger.With().Str("module", "app_score_registry").Logger(), + spamScoreCache: config.CacheFactory(), + penalty: config.Penalty, + init: config.Init, + validator: config.Validator, + idProvider: config.IdProvider, + } + + return reg +} + +var _ p2p.GossipSubInvCtrlMsgNotifConsumer = (*GossipSubAppSpecificScoreRegistry)(nil) + +// AppSpecificScoreFunc returns the application specific penalty function that is called by the GossipSub protocol to determine the application specific penalty of a peer. +func (r *GossipSubAppSpecificScoreRegistry) AppSpecificScoreFunc() func(peer.ID) float64 { + return func(pid peer.ID) float64 { + appSpecificScore := float64(0) + + lg := r.logger.With().Str("peer_id", pid.String()).Logger() + // (1) spam penalty: the penalty is applied to the application specific penalty when a peer conducts a spamming misbehaviour. + spamRecord, err, spamRecordExists := r.spamScoreCache.Get(pid) + if err != nil { + // the error is considered fatal as it means the cache is not working properly. + // we should not continue with the execution as it may lead to routing attack vulnerability. + r.logger.Fatal().Str("peer_id", pid.String()).Err(err).Msg("could not get application specific penalty for peer") + return appSpecificScore // unreachable, but added to avoid proceeding with the execution if log level is changed. + } + + if spamRecordExists { + lg = lg.With().Float64("spam_penalty", spamRecord.Penalty).Logger() + appSpecificScore += spamRecord.Penalty + } + + // (2) staking score: for staked peers, a default positive reward is applied only if the peer has no penalty on spamming and subscription. + // for unknown peers a negative penalty is applied. + stakingScore, flowId, role := r.stakingScore(pid) + if stakingScore < 0 { + lg = lg.With().Float64("staking_penalty", stakingScore).Logger() + // staking penalty is applied right away. + appSpecificScore += stakingScore + } + + if stakingScore >= 0 { + // (3) subscription penalty: the subscription penalty is applied to the application specific penalty when a + // peer is subscribed to a topic that it is not allowed to subscribe to based on its role. + // Note: subscription penalty can be considered only for staked peers, for non-staked peers, we cannot + // determine the role of the peer. + subscriptionPenalty := r.subscriptionPenalty(pid, flowId, role) + lg = lg.With().Float64("subscription_penalty", subscriptionPenalty).Logger() + if subscriptionPenalty < 0 { + appSpecificScore += subscriptionPenalty + } + } + + // (4) staking reward: for staked peers, a default positive reward is applied only if the peer has no penalty on spamming and subscription. + if stakingScore > 0 && appSpecificScore == float64(0) { + lg = lg.With().Float64("staking_reward", stakingScore).Logger() + appSpecificScore += stakingScore + } + + lg.Trace(). + Float64("total_app_specific_score", appSpecificScore). + Msg("application specific penalty computed") + + return appSpecificScore + } +} + +func (r *GossipSubAppSpecificScoreRegistry) stakingScore(pid peer.ID) (float64, flow.Identifier, flow.Role) { + lg := r.logger.With().Str("peer_id", pid.String()).Logger() + + // checks if peer has a valid Flow protocol identity. + flowId, err := HasValidFlowIdentity(r.idProvider, pid) + if err != nil { + lg.Error(). + Err(err). + Bool(logging.KeySuspicious, true). + Msg("invalid peer identity, penalizing peer") + return DefaultUnknownIdentityPenalty, flow.Identifier{}, 0 + } + + lg = lg.With(). + Hex("flow_id", logging.ID(flowId.NodeID)). + Str("role", flowId.Role.String()). + Logger() + + // checks if peer is an access node, and if so, pushes it to the + // edges of the network by giving the minimum penalty. + if flowId.Role == flow.RoleAccess { + lg.Trace(). + Msg("pushing access node to edge by penalizing with minimum penalty value") + return MinAppSpecificPenalty, flowId.NodeID, flowId.Role + } + + lg.Trace(). + Msg("rewarding well-behaved non-access node peer with maximum reward value") + + return DefaultStakedIdentityReward, flowId.NodeID, flowId.Role +} + +func (r *GossipSubAppSpecificScoreRegistry) subscriptionPenalty(pid peer.ID, flowId flow.Identifier, role flow.Role) float64 { + // checks if peer has any subscription violation. + if err := r.validator.CheckSubscribedToAllowedTopics(pid, role); err != nil { + r.logger.Err(err). + Str("peer_id", pid.String()). + Hex("flow_id", logging.ID(flowId)). + Bool(logging.KeySuspicious, true). + Msg("invalid subscription detected, penalizing peer") + return DefaultInvalidSubscriptionPenalty + } + + return 0 +} + +// OnInvalidControlMessageNotification is called when a new invalid control message notification is distributed. +// Any error on consuming event must handle internally. +// The implementation must be concurrency safe, but can be blocking. +func (r *GossipSubAppSpecificScoreRegistry) OnInvalidControlMessageNotification(notification *p2p.InvCtrlMsgNotif) { + // we use mutex to ensure the method is concurrency safe. + + lg := r.logger.With(). + Str("peer_id", notification.PeerID.String()). + Str("misbehavior_type", notification.MsgType.String()).Logger() + + // try initializing the application specific penalty for the peer if it is not yet initialized. + // this is done to avoid the case where the peer is not yet cached and the application specific penalty is not yet initialized. + // initialization is successful only if the peer is not yet cached. + initialized := r.spamScoreCache.Add(notification.PeerID, r.init()) + if initialized { + lg.Trace().Str("peer_id", notification.PeerID.String()).Msg("application specific penalty initialized for peer") + } + + record, err := r.spamScoreCache.Update(notification.PeerID, func(record p2p.GossipSubSpamRecord) p2p.GossipSubSpamRecord { + switch notification.MsgType { + case p2pmsg.CtrlMsgGraft: + record.Penalty += r.penalty.Graft + case p2pmsg.CtrlMsgPrune: + record.Penalty += r.penalty.Prune + case p2pmsg.CtrlMsgIHave: + record.Penalty += r.penalty.IHave + case p2pmsg.CtrlMsgIWant: + record.Penalty += r.penalty.IWant + default: + // the error is considered fatal as it means that we have an unsupported misbehaviour type, we should crash the node to prevent routing attack vulnerability. + lg.Fatal().Str("misbehavior_type", notification.MsgType.String()).Msg("unknown misbehaviour type") + } + + return record + }) + + if err != nil { + // any returned error from adjust is non-recoverable and fatal, we crash the node. + lg.Fatal().Err(err).Msg("could not adjust application specific penalty for peer") + } + + lg.Debug(). + Float64("app_specific_score", record.Penalty). + Msg("applied misbehaviour penalty and updated application specific penalty") +} + +// DefaultDecayFunction is the default decay function that is used to decay the application specific penalty of a peer. +// It is used if no decay function is provided in the configuration. +// It decays the application specific penalty of a peer if it is negative. +func DefaultDecayFunction() netcache.PreprocessorFunc { + return func(record p2p.GossipSubSpamRecord, lastUpdated time.Time) (p2p.GossipSubSpamRecord, error) { + if record.Penalty >= 0 { + // no need to decay the penalty if it is positive, the reason is currently the app specific penalty + // is only used to penalize peers. Hence, when there is no reward, there is no need to decay the positive penalty, as + // no node can accumulate a positive penalty. + return record, nil + } + + if record.Penalty > skipDecayThreshold { + // penalty is negative but greater than the threshold, we set it to 0. + record.Penalty = 0 + return record, nil + } + + // penalty is negative and below the threshold, we decay it. + penalty, err := GeometricDecay(record.Penalty, record.Decay, lastUpdated) + if err != nil { + return record, fmt.Errorf("could not decay application specific penalty: %w", err) + } + record.Penalty = penalty + return record, nil + } +} + +// InitAppScoreRecordState initializes the gossipsub spam record state for a peer. +// Returns: +// - a gossipsub spam record with the default decay value and 0 penalty. +func InitAppScoreRecordState() p2p.GossipSubSpamRecord { + return p2p.GossipSubSpamRecord{ + Decay: defaultDecay, + Penalty: 0, + } +} diff --git a/network/p2p/scoring/registry_test.go b/network/p2p/scoring/registry_test.go new file mode 100644 index 00000000000..843ec2d87ae --- /dev/null +++ b/network/p2p/scoring/registry_test.go @@ -0,0 +1,481 @@ +package scoring_test + +import ( + "fmt" + "math" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/assert" + testifymock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network/p2p" + netcache "github.com/onflow/flow-go/network/p2p/cache" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" + mockp2p "github.com/onflow/flow-go/network/p2p/mock" + "github.com/onflow/flow-go/network/p2p/scoring" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestNoPenaltyRecord tests that if there is no penalty record for a peer id, the app specific score should be the max +// app specific reward. This is the default reward for a staked peer that has valid subscriptions and has not been +// penalized. +func TestNoPenaltyRecord(t *testing.T) { + peerID := peer.ID("peer-1") + reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + t, + withStakedIdentity(peerID), + withValidSubscriptions(peerID)) + + // initially, the spamRecords should not have the peer id. + assert.False(t, spamRecords.Has(peerID)) + + score := reg.AppSpecificScoreFunc()(peerID) + // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which + // is the default reward for a staked peer that has valid subscriptions. + assert.Equal(t, scoring.MaxAppSpecificReward, score) + + // still the spamRecords should not have the peer id (as there is no spam record for the peer id). + assert.False(t, spamRecords.Has(peerID)) +} + +// TestPeerWithSpamRecord tests the app specific penalty computation of the node when there is a spam record for the peer id. +// It tests the state that a staked peer with a valid role and valid subscriptions has spam records. +// Since the peer has spam records, it should be deprived of the default reward for its staked role, and only have the +// penalty value as the app specific score. +func TestPeerWithSpamRecord(t *testing.T) { + t.Run("graft", func(t *testing.T) { + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) + }) + t.Run("prune", func(t *testing.T) { + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) + }) + t.Run("ihave", func(t *testing.T) { + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) + }) + t.Run("iwant", func(t *testing.T) { + testPeerWithSpamRecord(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) + }) +} + +func testPeerWithSpamRecord(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { + peerID := peer.ID("peer-1") + reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + t, + withStakedIdentity(peerID), + withValidSubscriptions(peerID)) + + // initially, the spamRecords should not have the peer id. + assert.False(t, spamRecords.Has(peerID)) + + // since the peer id does not have a spam record, the app specific score should be the max app specific reward, which + // is the default reward for a staked peer that has valid subscriptions. + score := reg.AppSpecificScoreFunc()(peerID) + assert.Equal(t, scoring.MaxAppSpecificReward, score) + + // report a misbehavior for the peer id. + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: messageType, + Count: 1, + }) + + // the penalty should now be updated in the spamRecords + record, err, ok := spamRecords.Get(peerID) // get the record from the spamRecords. + assert.True(t, ok) + assert.NoError(t, err) + assert.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) // penalty should be updated to -10. + assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) // decay should be initialized to the initial state. + + // this peer has a spam record, with no subscription penalty. Hence, the app specific score should only be the spam penalty, + // and the peer should be deprived of the default reward for its valid staked role. + score = reg.AppSpecificScoreFunc()(peerID) + assert.Less(t, math.Abs(expectedPenalty-score), 10e-3) +} + +func TestSpamRecord_With_UnknownIdentity(t *testing.T) { + t.Run("graft", func(t *testing.T) { + testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) + }) + t.Run("prune", func(t *testing.T) { + testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) + }) + t.Run("ihave", func(t *testing.T) { + testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) + }) + t.Run("iwant", func(t *testing.T) { + testSpamRecordWithUnknownIdentity(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) + }) +} + +// testSpamRecordWithUnknownIdentity tests the app specific penalty computation of the node when there is a spam record for the peer id and +// the peer id has an unknown identity. +func testSpamRecordWithUnknownIdentity(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { + peerID := peer.ID("peer-1") + reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + t, + withUnknownIdentity(peerID), + withValidSubscriptions(peerID)) + + // initially, the spamRecords should not have the peer id. + assert.False(t, spamRecords.Has(peerID)) + + // peer does not have spam record, but has an unknown identity. Hence, the app specific score should be the staking penalty. + score := reg.AppSpecificScoreFunc()(peerID) + require.Equal(t, scoring.DefaultUnknownIdentityPenalty, score) + + // report a misbehavior for the peer id. + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: messageType, + Count: 1, + }) + + // the penalty should now be updated. + record, err, ok := spamRecords.Get(peerID) // get the record from the spamRecords. + assert.True(t, ok) + assert.NoError(t, err) + assert.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) // penalty should be updated to -10, we account for decay. + assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) // decay should be initialized to the initial state. + + // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty + // and the staking penalty. + score = reg.AppSpecificScoreFunc()(peerID) + assert.Less(t, math.Abs(expectedPenalty+scoring.DefaultUnknownIdentityPenalty-score), 10e-3) +} + +func TestSpamRecord_With_SubscriptionPenalty(t *testing.T) { + t.Run("graft", func(t *testing.T) { + testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgGraft, penaltyValueFixtures().Graft) + }) + t.Run("prune", func(t *testing.T) { + testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgPrune, penaltyValueFixtures().Prune) + }) + t.Run("ihave", func(t *testing.T) { + testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgIHave, penaltyValueFixtures().IHave) + }) + t.Run("iwant", func(t *testing.T) { + testSpamRecordWithSubscriptionPenalty(t, p2pmsg.CtrlMsgIWant, penaltyValueFixtures().IWant) + }) +} + +// testSpamRecordWithUnknownIdentity tests the app specific penalty computation of the node when there is a spam record for the peer id and +// the peer id has an invalid subscription as well. +func testSpamRecordWithSubscriptionPenalty(t *testing.T, messageType p2pmsg.ControlMessageType, expectedPenalty float64) { + peerID := peer.ID("peer-1") + reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + t, + withStakedIdentity(peerID), + withInvalidSubscriptions(peerID)) + + // initially, the spamRecords should not have the peer id. + assert.False(t, spamRecords.Has(peerID)) + + // peer does not have spam record, but has invalid subscription. Hence, the app specific score should be subscription penalty. + score := reg.AppSpecificScoreFunc()(peerID) + require.Equal(t, scoring.DefaultInvalidSubscriptionPenalty, score) + + // report a misbehavior for the peer id. + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: messageType, + Count: 1, + }) + + // the penalty should now be updated. + record, err, ok := spamRecords.Get(peerID) // get the record from the spamRecords. + assert.True(t, ok) + assert.NoError(t, err) + assert.Less(t, math.Abs(expectedPenalty-record.Penalty), 10e-3) + assert.Equal(t, scoring.InitAppScoreRecordState().Decay, record.Decay) // decay should be initialized to the initial state. + + // the peer has spam record as well as an unknown identity. Hence, the app specific score should be the spam penalty + // and the staking penalty. + score = reg.AppSpecificScoreFunc()(peerID) + assert.Less(t, math.Abs(expectedPenalty+scoring.DefaultInvalidSubscriptionPenalty-score), 10e-3) +} + +// TestSpamPenaltyDecaysInCache tests that the spam penalty records decay over time in the cache. +func TestSpamPenaltyDecaysInCache(t *testing.T) { + peerID := peer.ID("peer-1") + reg, _ := newGossipSubAppSpecificScoreRegistry(t, + withStakedIdentity(peerID), + withValidSubscriptions(peerID)) + + // report a misbehavior for the peer id. + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: p2pmsg.CtrlMsgPrune, + Count: 1, + }) + + time.Sleep(1 * time.Second) // wait for the penalty to decay. + + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: p2pmsg.CtrlMsgGraft, + Count: 1, + }) + + time.Sleep(1 * time.Second) // wait for the penalty to decay. + + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: p2pmsg.CtrlMsgIHave, + Count: 1, + }) + + time.Sleep(1 * time.Second) // wait for the penalty to decay. + + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: p2pmsg.CtrlMsgIWant, + Count: 1, + }) + + time.Sleep(1 * time.Second) // wait for the penalty to decay. + + // when the app specific penalty function is called for the first time, the decay functionality should be kicked in + // the cache, and the penalty should be updated. Note that since the penalty values are negative, the default staked identity + // reward is not applied. Hence, the penalty is only comprised of the penalties. + score := reg.AppSpecificScoreFunc()(peerID) + // the upper bound is the sum of the penalties without decay. + scoreUpperBound := penaltyValueFixtures().Prune + + penaltyValueFixtures().Graft + + penaltyValueFixtures().IHave + + penaltyValueFixtures().IWant + // the lower bound is the sum of the penalties with decay assuming the decay is applied 4 times to the sum of the penalties. + // in reality, the decay is applied 4 times to the first penalty, then 3 times to the second penalty, and so on. + scoreLowerBound := scoreUpperBound * math.Pow(scoring.InitAppScoreRecordState().Decay, 4) + + // with decay, the penalty should be between the upper and lower bounds. + assert.Greater(t, score, scoreUpperBound) + assert.Less(t, score, scoreLowerBound) +} + +// TestSpamPenaltyDecayToZero tests that the spam penalty decays to zero over time, and when the spam penalty of +// a peer is set back to zero, its app specific penalty is also reset to the initial state. +func TestSpamPenaltyDecayToZero(t *testing.T) { + peerID := peer.ID("peer-1") + reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + t, + withStakedIdentity(peerID), + withValidSubscriptions(peerID), + withInitFunction(func() p2p.GossipSubSpamRecord { + return p2p.GossipSubSpamRecord{ + Decay: 0.02, // we choose a small decay value to speed up the test. + Penalty: 0, + } + })) + + // report a misbehavior for the peer id. + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: p2pmsg.CtrlMsgGraft, + Count: 1, + }) + + // decays happen every second, so we wait for 1 second to make sure the penalty is updated. + time.Sleep(1 * time.Second) + // the penalty should now be updated, it should be still negative but greater than the penalty value (due to decay). + score := reg.AppSpecificScoreFunc()(peerID) + require.Less(t, score, float64(0)) // the penalty should be less than zero. + require.Greater(t, score, penaltyValueFixtures().Graft) // the penalty should be less than the penalty value due to decay. + + require.Eventually(t, func() bool { + // the spam penalty should eventually decay to zero. + r, err, ok := spamRecords.Get(peerID) + return ok && err == nil && r.Penalty == 0.0 + }, 5*time.Second, 100*time.Millisecond) + + require.Eventually(t, func() bool { + // when the spam penalty is decayed to zero, the app specific penalty of the node should reset back to default staking reward. + return reg.AppSpecificScoreFunc()(peerID) == scoring.DefaultStakedIdentityReward + }, 5*time.Second, 100*time.Millisecond) + + // the penalty should now be zero. + record, err, ok := spamRecords.Get(peerID) // get the record from the spamRecords. + assert.True(t, ok) + assert.NoError(t, err) + assert.Equal(t, 0.0, record.Penalty) // penalty should be zero. +} + +// TestPersistingUnknownIdentityPenalty tests that even though the spam penalty is decayed to zero, the unknown identity penalty +// is persisted. This is because the unknown identity penalty is not decayed. +func TestPersistingUnknownIdentityPenalty(t *testing.T) { + peerID := peer.ID("peer-1") + reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + t, + withUnknownIdentity(peerID), // the peer id has an unknown identity. + withValidSubscriptions(peerID), + withInitFunction(func() p2p.GossipSubSpamRecord { + return p2p.GossipSubSpamRecord{ + Decay: 0.02, // we choose a small decay value to speed up the test. + Penalty: 0, + } + })) + + // initially, the app specific score should be the default unknown identity penalty. + require.Equal(t, scoring.DefaultUnknownIdentityPenalty, reg.AppSpecificScoreFunc()(peerID)) + + // report a misbehavior for the peer id. + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: p2pmsg.CtrlMsgGraft, + Count: 1, + }) + + // with reported spam, the app specific score should be the default unknown identity + the spam penalty. + require.Less(t, math.Abs(scoring.DefaultUnknownIdentityPenalty+penaltyValueFixtures().Graft-reg.AppSpecificScoreFunc()(peerID)), 10e-3) + + // decays happen every second, so we wait for 1 second to make sure the penalty is updated. + time.Sleep(1 * time.Second) + // the penalty should now be updated, it should be still negative but greater than the penalty value (due to decay). + score := reg.AppSpecificScoreFunc()(peerID) + require.Less(t, score, float64(0)) // the penalty should be less than zero. + require.Greater(t, score, penaltyValueFixtures().Graft+scoring.DefaultUnknownIdentityPenalty) // the penalty should be less than the penalty value due to decay. + + require.Eventually(t, func() bool { + // the spam penalty should eventually decay to zero. + r, err, ok := spamRecords.Get(peerID) + return ok && err == nil && r.Penalty == 0.0 + }, 5*time.Second, 100*time.Millisecond) + + require.Eventually(t, func() bool { + // when the spam penalty is decayed to zero, the app specific penalty of the node should only contain the unknown identity penalty. + return reg.AppSpecificScoreFunc()(peerID) == scoring.DefaultUnknownIdentityPenalty + }, 5*time.Second, 100*time.Millisecond) + + // the spam penalty should now be zero in spamRecords. + record, err, ok := spamRecords.Get(peerID) // get the record from the spamRecords. + assert.True(t, ok) + assert.NoError(t, err) + assert.Equal(t, 0.0, record.Penalty) // penalty should be zero. +} + +// TestPersistingInvalidSubscriptionPenalty tests that even though the spam penalty is decayed to zero, the invalid subscription penalty +// is persisted. This is because the invalid subscription penalty is not decayed. +func TestPersistingInvalidSubscriptionPenalty(t *testing.T) { + peerID := peer.ID("peer-1") + reg, spamRecords := newGossipSubAppSpecificScoreRegistry( + t, + withStakedIdentity(peerID), + withInvalidSubscriptions(peerID), // the peer id has an invalid subscription. + withInitFunction(func() p2p.GossipSubSpamRecord { + return p2p.GossipSubSpamRecord{ + Decay: 0.02, // we choose a small decay value to speed up the test. + Penalty: 0, + } + })) + + // initially, the app specific score should be the default invalid subscription penalty. + require.Equal(t, scoring.DefaultUnknownIdentityPenalty, reg.AppSpecificScoreFunc()(peerID)) + + // report a misbehavior for the peer id. + reg.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: peerID, + MsgType: p2pmsg.CtrlMsgGraft, + Count: 1, + }) + + // with reported spam, the app specific score should be the default invalid subscription penalty + the spam penalty. + require.Less(t, math.Abs(scoring.DefaultInvalidSubscriptionPenalty+penaltyValueFixtures().Graft-reg.AppSpecificScoreFunc()(peerID)), 10e-3) + + // decays happen every second, so we wait for 1 second to make sure the penalty is updated. + time.Sleep(1 * time.Second) + // the penalty should now be updated, it should be still negative but greater than the penalty value (due to decay). + score := reg.AppSpecificScoreFunc()(peerID) + require.Less(t, score, float64(0)) // the penalty should be less than zero. + require.Greater(t, score, penaltyValueFixtures().Graft+scoring.DefaultInvalidSubscriptionPenalty) // the penalty should be less than the penalty value due to decay. + + require.Eventually(t, func() bool { + // the spam penalty should eventually decay to zero. + r, err, ok := spamRecords.Get(peerID) + return ok && err == nil && r.Penalty == 0.0 + }, 5*time.Second, 100*time.Millisecond) + + require.Eventually(t, func() bool { + // when the spam penalty is decayed to zero, the app specific penalty of the node should only contain the default invalid subscription penalty. + return reg.AppSpecificScoreFunc()(peerID) == scoring.DefaultUnknownIdentityPenalty + }, 5*time.Second, 100*time.Millisecond) + + // the spam penalty should now be zero in spamRecords. + record, err, ok := spamRecords.Get(peerID) // get the record from the spamRecords. + assert.True(t, ok) + assert.NoError(t, err) + assert.Equal(t, 0.0, record.Penalty) // penalty should be zero. +} + +// withStakedIdentity returns a function that sets the identity provider to return an staked identity for the given peer id. +// It is used for testing purposes, and causes the given peer id to benefit from the staked identity reward in GossipSub. +func withStakedIdentity(peerId peer.ID) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + return func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + cfg.IdProvider.(*mock.IdentityProvider).On("ByPeerID", peerId).Return(unittest.IdentityFixture(), true).Maybe() + } +} + +// withValidSubscriptions returns a function that sets the subscription validator to return nil for the given peer id. +// It is used for testing purposes and causes the given peer id to never be penalized for subscribing to invalid topics. +func withValidSubscriptions(peer peer.ID) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + return func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + cfg.Validator.(*mockp2p.SubscriptionValidator).On("CheckSubscribedToAllowedTopics", peer, testifymock.Anything).Return(nil).Maybe() + } +} + +// withUnknownIdentity returns a function that sets the identity provider to return an error for the given peer id. +// It is used for testing purposes, and causes the given peer id to be penalized for not having a staked identity. +func withUnknownIdentity(peer peer.ID) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + return func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + cfg.IdProvider.(*mock.IdentityProvider).On("ByPeerID", peer).Return(nil, false).Maybe() + } +} + +// withInvalidSubscriptions returns a function that sets the subscription validator to return an error for the given peer id. +// It is used for testing purposes and causes the given peer id to be penalized for subscribing to invalid topics. +func withInvalidSubscriptions(peer peer.ID) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + return func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + cfg.Validator.(*mockp2p.SubscriptionValidator).On("CheckSubscribedToAllowedTopics", peer, testifymock.Anything).Return(fmt.Errorf("invalid subscriptions")).Maybe() + } +} + +func withInitFunction(initFunction func() p2p.GossipSubSpamRecord) func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + return func(cfg *scoring.GossipSubAppSpecificScoreRegistryConfig) { + cfg.Init = initFunction + } +} + +// newGossipSubAppSpecificScoreRegistry returns a new instance of GossipSubAppSpecificScoreRegistry with default values +// for the testing purposes. +func newGossipSubAppSpecificScoreRegistry(t *testing.T, opts ...func(*scoring.GossipSubAppSpecificScoreRegistryConfig)) (*scoring.GossipSubAppSpecificScoreRegistry, *netcache.GossipSubSpamRecordCache) { + cache := netcache.NewGossipSubSpamRecordCache(100, unittest.Logger(), metrics.NewNoopCollector(), scoring.DefaultDecayFunction()) + cfg := &scoring.GossipSubAppSpecificScoreRegistryConfig{ + Logger: unittest.Logger(), + Init: scoring.InitAppScoreRecordState, + Penalty: penaltyValueFixtures(), + IdProvider: mock.NewIdentityProvider(t), + Validator: mockp2p.NewSubscriptionValidator(t), + CacheFactory: func() p2p.GossipSubSpamRecordCache { + return cache + }, + } + for _, opt := range opts { + opt(cfg) + } + return scoring.NewGossipSubAppSpecificScoreRegistry(cfg), cache +} + +// penaltyValueFixtures returns a set of penalty values for testing purposes. +// The values are not realistic. The important thing is that they are different from each other. This is to make sure +// that the tests are not passing because of the default values. +func penaltyValueFixtures() scoring.GossipSubCtrlMsgPenaltyValue { + return scoring.GossipSubCtrlMsgPenaltyValue{ + Graft: -100, + Prune: -50, + IHave: -20, + IWant: -10, + } +} diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index fd3ff9ad80d..0ae676005cb 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -1,35 +1,64 @@ package scoring import ( + "fmt" "time" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p" + netcache "github.com/onflow/flow-go/network/p2p/cache" + "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/utils/logging" ) const ( + // DefaultAppSpecificScoreWeight is the default weight for app-specific scores. It is used to scale the app-specific + // scores to the same range as the other scores. At the current version, we don't distinguish between the app-specific + // scores and the other scores, so we set it to 1. DefaultAppSpecificScoreWeight = 1 - MaxAppSpecificPenalty = -100 - MinAppSpecificPenalty = -1 - MaxAppSpecificReward = 100 - // DefaultGossipThreshold when a peer's score drops below this threshold, + // MaxAppSpecificReward is the default reward for well-behaving staked peers. If a peer does not have + // any misbehavior record, e.g., invalid subscription, invalid message, etc., it will be rewarded with this score. + MaxAppSpecificReward = float64(100) + + // MaxAppSpecificPenalty is the maximum penalty for sever offenses that we apply to a remote node score. The score + // mechanism of GossipSub in Flow is designed in a way that all other infractions are penalized with a fraction of + // this value. We have also set the other parameters such as DefaultGraylistThreshold, DefaultGossipThreshold and DefaultPublishThreshold to + // be a bit higher than this, i.e., MaxAppSpecificPenalty + 1. This ensures that a node with a score of MaxAppSpecificPenalty + // will be graylisted (i.e., all incoming and outgoing RPCs are rejected) and will not be able to publish or gossip any messages. + MaxAppSpecificPenalty = -1 * MaxAppSpecificReward + MinAppSpecificPenalty = -1 + + // DefaultStakedIdentityReward is the default reward for staking peers. It is applied to the peer's score when + // the peer does not have any misbehavior record, e.g., invalid subscription, invalid message, etc. + // The purpose is to reward the staking peers for their contribution to the network and prioritize them in neighbor selection. + DefaultStakedIdentityReward = MaxAppSpecificReward + + // DefaultUnknownIdentityPenalty is the default penalty for unknown identity. It is applied to the peer's score when + // the peer is not in the identity list. + DefaultUnknownIdentityPenalty = MaxAppSpecificPenalty + + // DefaultInvalidSubscriptionPenalty is the default penalty for invalid subscription. It is applied to the peer's score when + // the peer subscribes to a topic that it is not authorized to subscribe to. + DefaultInvalidSubscriptionPenalty = MaxAppSpecificPenalty + + // DefaultGossipThreshold when a peer's penalty drops below this threshold, // no gossip is emitted towards that peer and gossip from that peer is ignored. // // Validation Constraint: GossipThreshold >= PublishThreshold && GossipThreshold < 0 // // How we use it: // As current max penalty is -100, we set the threshold to -99 so that all gossips - // to and from peers with score -100 are ignored. - DefaultGossipThreshold = -99 + // to and from peers with penalty -100 are ignored. + DefaultGossipThreshold = MaxAppSpecificPenalty + 1 - // DefaultPublishThreshold when a peer's score drops below this threshold, + // DefaultPublishThreshold when a peer's penalty drops below this threshold, // self-published messages are not propagated towards this peer. // // Validation Constraint: @@ -38,9 +67,9 @@ const ( // How we use it: // As current max penalty is -100, we set the threshold to -99 so that all penalized peers are deprived of // receiving any published messages. - DefaultPublishThreshold = -99 + DefaultPublishThreshold = MaxAppSpecificPenalty + 1 - // DefaultGraylistThreshold when a peer's score drops below this threshold, the peer is graylisted, i.e., + // DefaultGraylistThreshold when a peer's penalty drops below this threshold, the peer is graylisted, i.e., // incoming RPCs from the peer are ignored. // // Validation Constraint: @@ -48,20 +77,20 @@ const ( // // How we use it: // As current max penalty is -100, we set the threshold to -99 so that all penalized peers are graylisted. - DefaultGraylistThreshold = -99 + DefaultGraylistThreshold = MaxAppSpecificPenalty + 1 // DefaultAcceptPXThreshold when a peer sends us PX information with a prune, we only accept it and connect to the supplied - // peers if the originating peer's score exceeds this threshold. + // peers if the originating peer's penalty exceeds this threshold. // // Validation Constraint: must be non-negative. // // How we use it: // As current max reward is 100, we set the threshold to 99 so that we only receive supplied peers from // well-behaved peers. - DefaultAcceptPXThreshold = 99 + DefaultAcceptPXThreshold = MaxAppSpecificReward - 1 - // DefaultOpportunisticGraftThreshold when the median peer score in the mesh drops below this value, - // the peer may select more peers with score above the median to opportunistically graft on the mesh. + // DefaultOpportunisticGraftThreshold when the median peer penalty in the mesh drops below this value, + // the peer may select more peers with penalty above the median to opportunistically graft on the mesh. // // Validation Constraint: must be non-negative. // @@ -73,41 +102,285 @@ const ( // MaxDebugLogs sets the max number of debug/trace log events per second. Logs emitted above // this threshold are dropped. MaxDebugLogs = 50 + + // defaultScoreCacheSize is the default size of the cache used to store the app specific penalty of peers. + defaultScoreCacheSize = 1000 + + // defaultDecayInterval is the default decay interval for the overall score of a peer at the GossipSub scoring + // system. We set it to 1 minute so that it is not too short so that a malicious node can recover from a penalty + // and is not too long so that a well-behaved node can't recover from a penalty. + defaultDecayInterval = 1 * time.Minute + + // defaultDecayToZero is the default decay to zero for the overall score of a peer at the GossipSub scoring system. + // It defines the maximum value below which a peer scoring counter is reset to zero. + // This is to prevent the counter from decaying to a very small value. + // The default value is 0.01, which means that a counter will be reset to zero if it decays to 0.01. + // When a counter hits the DecayToZero threshold, it means that the peer did not exhibit the behavior + // for a long time, and we can reset the counter. + defaultDecayToZero = 0.01 + + // defaultTopicSkipAtomicValidation is the default value for the skip atomic validation flag for topics. + // We set it to true, which means gossipsub parameter validation will not fail if we leave some of the + // topic parameters at their default values, i.e., zero. This is because we are not setting all + // topic parameters at the current implementation. + defaultTopicSkipAtomicValidation = true + + // defaultTopicInvalidMessageDeliveriesWeight this value is applied to the square of the number of invalid message deliveries on a topic. + // It is used to penalize peers that send invalid messages. By an invalid message, we mean a message that is not signed by the + // publisher, or a message that is not signed by the peer that sent it. We set it to -1.0, which means that with around 14 invalid + // message deliveries within a gossipsub heartbeat interval, the peer will be disconnected. + // The supporting math is as follows: + // - each staked (i.e., authorized) peer is rewarded by the fixed reward of 100 (i.e., DefaultStakedIdentityReward). + // - x invalid message deliveries will result in a penalty of x^2 * DefaultTopicInvalidMessageDeliveriesWeight, i.e., -x^2. + // - the peer will be disconnected when its penalty reaches -100 (i.e., MaxAppSpecificPenalty). + // - so, the maximum number of invalid message deliveries that a peer can have before being disconnected is sqrt(200/DefaultTopicInvalidMessageDeliveriesWeight) ~ 14. + defaultTopicInvalidMessageDeliveriesWeight = -1.0 + + // defaultTopicInvalidMessageDeliveriesDecay decay factor used to decay the number of invalid message deliveries. + // The total number of invalid message deliveries is multiplied by this factor at each heartbeat interval to + // decay the number of invalid message deliveries, and prevent the peer from being disconnected if it stops + // sending invalid messages. We set it to 0.99, which means that the number of invalid message deliveries will + // decay by 1% at each heartbeat interval. + // The decay heartbeats are defined by the heartbeat interval of the gossipsub scoring system, which is 1 Minute (defaultDecayInterval). + defaultTopicInvalidMessageDeliveriesDecay = .99 + + // defaultTopicTimeInMeshQuantum is the default time in mesh quantum for the GossipSub scoring system. It is used to gauge + // a discrete time interval for the time in mesh counter. We set it to 1 hour, which means that every one complete hour a peer is + // in a topic mesh, the time in mesh counter will be incremented by 1 and is counted towards the availability score of the peer in that topic mesh. + // The reason of setting it to 1 hour is that we want to reward peers that are in a topic mesh for a long time, and we want to avoid rewarding peers that + // are churners, i.e., peers that join and leave a topic mesh frequently. + defaultTopicTimeInMesh = time.Hour + + // defaultTopicWeight is the default weight of a topic in the GossipSub scoring system. + // The overall score of a peer in a topic mesh is multiplied by the weight of the topic when calculating the overall score of the peer. + // We set it to 1.0, which means that the overall score of a peer in a topic mesh is not affected by the weight of the topic. + defaultTopicWeight = 1.0 + + // defaultTopicMeshMessageDeliveriesDecay is applied to the number of actual message deliveries in a topic mesh + // at each decay interval (i.e., defaultDecayInterval). + // It is used to decay the number of actual message deliveries, and prevents past message + // deliveries from affecting the current score of the peer. + // As the decay interval is 1 minute, we set it to 0.5, which means that the number of actual message + // deliveries will decay by 50% at each decay interval. + defaultTopicMeshMessageDeliveriesDecay = .5 + + // defaultTopicMeshMessageDeliveriesCap is the maximum number of actual message deliveries in a topic + // mesh that is used to calculate the score of a peer in that topic mesh. + // We set it to 1000, which means that the maximum number of actual message deliveries in a + // topic mesh that is used to calculate the score of a peer in that topic mesh is 1000. + // This is to prevent the score of a peer in a topic mesh from being affected by a large number of actual + // message deliveries and also affect the score of the peer in other topic meshes. + // When the total delivered messages in a topic mesh exceeds this value, the score of the peer in that topic + // mesh will not be affected by the actual message deliveries in that topic mesh. + // Moreover, this does not allow the peer to accumulate a large number of actual message deliveries in a topic mesh + // and then start under-performing in that topic mesh without being penalized. + defaultTopicMeshMessageDeliveriesCap = 1000 + + // defaultTopicMeshMessageDeliveriesThreshold is the threshold for the number of actual message deliveries in a + // topic mesh that is used to calculate the score of a peer in that topic mesh. + // If the number of actual message deliveries in a topic mesh is less than this value, + // the peer will be penalized by square of the difference between the actual message deliveries and the threshold, + // i.e., -w * (actual - threshold)^2 where `actual` and `threshold` are the actual message deliveries and the + // threshold, respectively, and `w` is the weight (i.e., defaultTopicMeshMessageDeliveriesWeight). + // We set it to 0.1 * defaultTopicMeshMessageDeliveriesCap, which means that if a peer delivers less tha 10% of the + // maximum number of actual message deliveries in a topic mesh, it will be considered as an under-performing peer + // in that topic mesh. + defaultTopicMeshMessageDeliveryThreshold = 0.1 * defaultTopicMeshMessageDeliveriesCap + + // defaultTopicMeshDeliveriesWeight is the weight for applying penalty when a peer is under-performing in a topic mesh. + // Upon every decay interval, if the number of actual message deliveries is less than the topic mesh message deliveries threshold + // (i.e., defaultTopicMeshMessageDeliveriesThreshold), the peer will be penalized by square of the difference between the actual + // message deliveries and the threshold, multiplied by this weight, i.e., -w * (actual - threshold)^2 where w is the weight, and + // `actual` and `threshold` are the actual message deliveries and the threshold, respectively. + // We set this value to be - 0.05 MaxAppSpecificReward / (defaultTopicMeshMessageDeliveriesThreshold^2). This guarantees that even if a peer + // is not delivering any message in a topic mesh, it will not be disconnected. + // Rather, looses part of the MaxAppSpecificReward that is awarded by our app-specific scoring function to all staked + // nodes by default will be withdrawn, and the peer will be slightly penalized. In other words, under-performing in a topic mesh + // will drop the overall score of a peer by 5% of the MaxAppSpecificReward that is awarded by our app-specific scoring function. + // It means that under-performing in a topic mesh will not cause a peer to be disconnected, but it will cause the peer to lose + // its MaxAppSpecificReward that is awarded by our app-specific scoring function. + // At this point, we do not want to disconnect a peer only because it is under-performing in a topic mesh as it might be + // causing a false positive network partition. + // TODO: we must increase the penalty for under-performing in a topic mesh in the future, and disconnect the peer if it is under-performing. + defaultTopicMeshMessageDeliveriesWeight = -0.05 * MaxAppSpecificReward / (defaultTopicMeshMessageDeliveryThreshold * defaultTopicMeshMessageDeliveryThreshold) + + // defaultMeshMessageDeliveriesWindow is the window size is time interval that we count a delivery of an already + // seen message towards the score of a peer in a topic mesh. The delivery is counted + // by GossipSub only if the previous sender of the message is different from the current sender. + // We set it to the decay interval of the GossipSub scoring system, which is 1 minute. + // It means that if a peer delivers a message that it has already seen less than one minute ago, + // the delivery will be counted towards the score of the peer in a topic mesh only if the previous sender of the message. + // This also prevents replay attacks of messages that are older than one minute. As replayed messages will not + // be counted towards the actual message deliveries of a peer in a topic mesh. + defaultMeshMessageDeliveriesWindow = defaultDecayInterval + + // defaultMeshMessageDeliveryActivation is the time interval that we wait for a new peer that joins a topic mesh + // till start counting the number of actual message deliveries of that peer in that topic mesh. + // We set it to 2 * defaultDecayInterval, which means that we wait for 2 decay intervals before start counting + // the number of actual message deliveries of a peer in a topic mesh. + // With a default decay interval of 1 minute, it means that we wait for 2 minutes before start counting the + // number of actual message deliveries of a peer in a topic mesh. This is to account for + // the time that it takes for a peer to start up and receive messages from other peers in the topic mesh. + defaultMeshMessageDeliveriesActivation = 2 * defaultDecayInterval + + // defaultBehaviorPenaltyThreshold is the threshold when the behavior of a peer is considered as bad by GossipSub. + // Currently, the misbehavior is defined as advertising an iHave without responding to the iWants (iHave broken promises), as well as attempting + // on GRAFT when the peer is considered for a PRUNE backoff, i.e., the local peer does not allow the peer to join the local topic mesh + // for a while, and the remote peer keep attempting on GRAFT (aka GRAFT flood). + // When the misbehavior counter of a peer goes beyond this threshold, the peer is penalized by defaultBehaviorPenaltyWeight (see below) for the excess misbehavior. + // + // An iHave broken promise means that a peer advertises an iHave for a message, but does not respond to the iWant requests for that message. + // For iHave broken promises, the gossipsub scoring works as follows: + // It samples ONLY A SINGLE iHave out of the entire RPC. + // If that iHave is not followed by an actual message within the next 3 seconds, the peer misbehavior counter is incremented by 1. + // + // We set it to 10, meaning that we at most tolerate 10 of such RPCs containing iHave broken promises. After that, the peer is penalized for every + // excess RPC containing iHave broken promises. + // The counter is also decayed by (0.99) every decay interval (defaultDecayInterval) i.e., every minute. + // Note that misbehaviors are counted by GossipSub across all topics (and is different from the Application Layer Misbehaviors that we count through + // the ALSP system). + defaultBehaviourPenaltyThreshold = 10 + + // defaultBehaviorPenaltyWeight is the weight for applying penalty when a peer misbehavior goes beyond the threshold. + // Misbehavior of a peer at gossipsub layer is defined as advertising an iHave without responding to the iWants (broken promises), as well as attempting + // on GRAFT when the peer is considered for a PRUNE backoff, i.e., the local peer does not allow the peer to join the local topic mesh + // This is detected by the GossipSub scoring system, and the peer is penalized by defaultBehaviorPenaltyWeight. + // + // An iHave broken promise means that a peer advertises an iHave for a message, but does not respond to the iWant requests for that message. + // For iHave broken promises, the gossipsub scoring works as follows: + // It samples ONLY A SINGLE iHave out of the entire RPC. + // If that iHave is not followed by an actual message within the next 3 seconds, the peer misbehavior counter is incremented by 1. + // + // The penalty is applied to the square of the difference between the misbehavior counter and the threshold, i.e., -|w| * (misbehavior counter - threshold)^2. + // We set it to 0.01 * MaxAppSpecificPenalty, which means that misbehaving 10 times more than the threshold (i.e., 10 + 10) will cause the peer to lose + // its entire AppSpecificReward that is awarded by our app-specific scoring function to all staked (i.e., authorized) nodes by default. + // Moreover, as the MaxAppSpecificPenalty is -MaxAppSpecificReward, misbehaving sqrt(2) * 10 times more than the threshold will cause the peer score + // to be dropped below the MaxAppSpecificPenalty, which is also below the GraylistThreshold, and the peer will be graylisted (i.e., disconnected). + // + // The math is as follows: -|w| * (misbehavior - threshold)^2 = 0.01 * MaxAppSpecificPenalty * (misbehavior - threshold)^2 < 2 * MaxAppSpecificPenalty + // if misbehavior > threshold + sqrt(2) * 10. + // As shown above, with this choice of defaultBehaviorPenaltyWeight, misbehaving sqrt(2) * 10 times more than the threshold will cause the peer score + // to be dropped below the MaxAppSpecificPenalty, which is also below the GraylistThreshold, and the peer will be graylisted (i.e., disconnected). This weight + // is chosen in a way that with almost a few misbehaviors more than the threshold, the peer will be graylisted. The rationale relies on the fact that + // the misbehavior counter is incremented by 1 for each RPC containing one or more broken promises. Hence, it is per RPC, and not per broken promise. + // Having sqrt(2) * 10 broken promises RPC is a blatant misbehavior, and the peer should be graylisted. With decay interval of 1 minute, and decay value of + // 0.99 we expect a graylisted node due to borken promises to get back in about 527 minutes, i.e., (0.99)^x * (sqrt(2) * 10)^2 * MaxAppSpecificPenalty > GraylistThreshold + // where x is the number of decay intervals that the peer is graylisted. As MaxAppSpecificPenalty and GraylistThresholds are close, we can simplify the inequality + // to (0.99)^x * (sqrt(2) * 10)^2 > 1 --> (0.99)^x * 200 > 1 --> (0.99)^x > 1/200 --> x > log(1/200) / log(0.99) --> x > 527.17 decay intervals, i.e., 527 minutes. + // Note that misbehaviors are counted by GossipSub across all topics (and is different from the Application Layer Misbehaviors that we count through + // the ALSP system that are reported by the engines). + defaultBehaviourPenaltyWeight = 0.01 * MaxAppSpecificPenalty + + // defaultBehaviorPenaltyDecay is the decay interval for the misbehavior counter of a peer. The misbehavior counter is + // incremented by GossipSub for iHave broken promises or the GRAFT flooding attacks (i.e., each GRAFT received from a remote peer while that peer is on a PRUNE backoff). + // + // An iHave broken promise means that a peer advertises an iHave for a message, but does not respond to the iWant requests for that message. + // For iHave broken promises, the gossipsub scoring works as follows: + // It samples ONLY A SINGLE iHave out of the entire RPC. + // If that iHave is not followed by an actual message within the next 3 seconds, the peer misbehavior counter is incremented by 1. + // This means that regardless of how many iHave broken promises an RPC contains, the misbehavior counter is incremented by 1. + // That is why we decay the misbehavior counter very slow, as this counter indicates a severe misbehavior. + // + // The misbehavior counter is decayed per decay interval (i.e., defaultDecayInterval = 1 minute) by GossipSub. + // We set it to 0.99, which means that the misbehavior counter is decayed by 1% per decay interval. + // With the generous threshold that we set (i.e., defaultBehaviourPenaltyThreshold = 10), we take the peers going beyond the threshold as persistent misbehaviors, + // We expect honest peers never to go beyond the threshold, and if they do, we expect them to go back below the threshold quickly. + // + // Note that misbehaviors are counted by GossipSub across all topics (and is different from the Application Layer Misbehaviors that we count through + // the ALSP system that is based on the engines report). + defaultBehaviourPenaltyDecay = 0.99 ) // ScoreOption is a functional option for configuring the peer scoring system. type ScoreOption struct { - logger zerolog.Logger - validator *SubscriptionValidator - idProvider module.IdentityProvider - peerScoreParams *pubsub.PeerScoreParams - peerThresholdParams *pubsub.PeerScoreThresholds - appSpecificScoreFunction func(peer.ID) float64 + logger zerolog.Logger + + peerScoreParams *pubsub.PeerScoreParams + peerThresholdParams *pubsub.PeerScoreThresholds + validator p2p.SubscriptionValidator + appScoreFunc func(peer.ID) float64 } -type PeerScoreParamsOption func(option *ScoreOption) +type ScoreOptionConfig struct { + logger zerolog.Logger + provider module.IdentityProvider + cacheSize uint32 + cacheMetrics module.HeroCacheMetrics + appScoreFunc func(peer.ID) float64 + decayInterval time.Duration // the decay interval, when is set to 0, the default value will be used. + topicParams []func(map[string]*pubsub.TopicScoreParams) + registerNotificationConsumerFunc func(p2p.GossipSubInvCtrlMsgNotifConsumer) +} -func WithAppSpecificScoreFunction(appSpecificScoreFunction func(peer.ID) float64) PeerScoreParamsOption { - return func(s *ScoreOption) { - s.appSpecificScoreFunction = appSpecificScoreFunction +func NewScoreOptionConfig(logger zerolog.Logger, idProvider module.IdentityProvider) *ScoreOptionConfig { + return &ScoreOptionConfig{ + logger: logger, + provider: idProvider, + cacheSize: defaultScoreCacheSize, + cacheMetrics: metrics.NewNoopCollector(), // no metrics by default + topicParams: make([]func(map[string]*pubsub.TopicScoreParams), 0), } } -// WithTopicScoreParams adds the topic score parameters to the peer score parameters. -// It is used to configure the topic score parameters for the pubsub system. -// If there is already a topic score parameter for the given topic, it will be overwritten. -func WithTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) PeerScoreParamsOption { - return func(s *ScoreOption) { - if s.peerScoreParams.Topics == nil { - s.peerScoreParams.Topics = make(map[string]*pubsub.TopicScoreParams) - } - s.peerScoreParams.Topics[topic.String()] = topicScoreParams - } +// SetCacheSize sets the size of the cache used to store the app specific penalty of peers. +// If the cache size is not set, the default value will be used. +// It is safe to call this method multiple times, the last call will be used. +func (c *ScoreOptionConfig) SetCacheSize(size uint32) { + c.cacheSize = size +} + +// SetCacheMetrics sets the cache metrics collector for the penalty option. +// It is used to collect metrics for the app specific penalty cache. If the cache metrics collector is not set, +// a no-op collector will be used. +// It is safe to call this method multiple times, the last call will be used. +func (c *ScoreOptionConfig) SetCacheMetrics(metrics module.HeroCacheMetrics) { + c.cacheMetrics = metrics +} + +// OverrideAppSpecificScoreFunction sets the app specific penalty function for the penalty option. +// It is used to calculate the app specific penalty of a peer. +// If the app specific penalty function is not set, the default one is used. +// Note that it is always safer to use the default one, unless you know what you are doing. +// It is safe to call this method multiple times, the last call will be used. +func (c *ScoreOptionConfig) OverrideAppSpecificScoreFunction(appSpecificScoreFunction func(peer.ID) float64) { + c.appScoreFunc = appSpecificScoreFunction +} + +// OverrideTopicScoreParams overrides the topic score parameters for the given topic. +// It is used to override the default topic score parameters for a specific topic. +// If the topic score parameters are not set, the default ones will be used. +func (c *ScoreOptionConfig) OverrideTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) { + c.topicParams = append(c.topicParams, func(topics map[string]*pubsub.TopicScoreParams) { + topics[topic.String()] = topicScoreParams + }) } -func NewScoreOption(logger zerolog.Logger, idProvider module.IdentityProvider, opts ...PeerScoreParamsOption) *ScoreOption { +// SetRegisterNotificationConsumerFunc sets the function to register the notification consumer for the penalty option. +// ScoreOption uses this function to register the notification consumer for the pubsub system so that it can receive +// notifications of invalid control messages. +func (c *ScoreOptionConfig) SetRegisterNotificationConsumerFunc(f func(p2p.GossipSubInvCtrlMsgNotifConsumer)) { + c.registerNotificationConsumerFunc = f +} + +// OverrideDecayInterval overrides the decay interval for the penalty option. It is used to override the default +// decay interval for the penalty option. The decay interval is the time interval that the decay values are applied and +// peer scores are updated. +// Note: It is always recommended to use the default value unless you know what you are doing. Hence, calling this method +// is not recommended in production. +// Args: +// +// interval: the decay interval. +// +// Returns: +// none +func (c *ScoreOptionConfig) OverrideDecayInterval(interval time.Duration) { + c.decayInterval = interval +} + +// NewScoreOption creates a new penalty option with the given configuration. +func NewScoreOption(cfg *ScoreOptionConfig) *ScoreOption { throttledSampler := logging.BurstSampler(MaxDebugLogs, time.Second) - logger = logger.With(). + logger := cfg.logger.With(). Str("module", "pubsub_score_option"). Logger(). Sample(zerolog.LevelSampler{ @@ -115,29 +388,62 @@ func NewScoreOption(logger zerolog.Logger, idProvider module.IdentityProvider, o DebugSampler: throttledSampler, }) validator := NewSubscriptionValidator() - appSpecificScore := defaultAppSpecificScoreFunction(logger, idProvider, validator) + scoreRegistry := NewGossipSubAppSpecificScoreRegistry(&GossipSubAppSpecificScoreRegistryConfig{ + Logger: logger, + Penalty: DefaultGossipSubCtrlMsgPenaltyValue(), + Validator: validator, + Init: InitAppScoreRecordState, + IdProvider: cfg.provider, + CacheFactory: func() p2p.GossipSubSpamRecordCache { + return netcache.NewGossipSubSpamRecordCache(cfg.cacheSize, cfg.logger, cfg.cacheMetrics, DefaultDecayFunction()) + }, + }) s := &ScoreOption{ - logger: logger, - validator: validator, - idProvider: idProvider, - appSpecificScoreFunction: appSpecificScore, - peerScoreParams: defaultPeerScoreParams(), + logger: logger, + validator: validator, + peerScoreParams: defaultPeerScoreParams(), + appScoreFunc: scoreRegistry.AppSpecificScoreFunc(), + } + + // set the app specific penalty function for the penalty option + // if the app specific penalty function is not set, use the default one + if cfg.appScoreFunc != nil { + s.appScoreFunc = cfg.appScoreFunc + s.logger. + Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Msg("app specific score function is overridden, should never happen in production") } - for _, opt := range opts { - opt(s) + if cfg.decayInterval > 0 { + // overrides the default decay interval if the decay interval is set. + s.peerScoreParams.DecayInterval = cfg.decayInterval + s.logger. + Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Dur("decay_interval_ms", cfg.decayInterval). + Msg("decay interval is overridden, should never happen in production") } - s.peerScoreParams.AppSpecificScore = s.appSpecificScoreFunction + // registers the score registry as the consumer of the invalid control message notifications + if cfg.registerNotificationConsumerFunc != nil { + cfg.registerNotificationConsumerFunc(scoreRegistry) + } + + s.peerScoreParams.AppSpecificScore = s.appScoreFunc + // apply the topic penalty parameters if any. + for _, topicParams := range cfg.topicParams { + topicParams(s.peerScoreParams.Topics) + } return s } -func (s *ScoreOption) SetSubscriptionProvider(provider *SubscriptionProvider) { - s.validator.RegisterSubscriptionProvider(provider) +func (s *ScoreOption) SetSubscriptionProvider(provider *SubscriptionProvider) error { + return s.validator.RegisterSubscriptionProvider(provider) } -func (s *ScoreOption) BuildFlowPubSubScoreOption() pubsub.Option { +func (s *ScoreOption) BuildFlowPubSubScoreOption() (*pubsub.PeerScoreParams, *pubsub.PeerScoreThresholds) { s.preparePeerScoreThresholds() s.logger.Info(). @@ -146,12 +452,15 @@ func (s *ScoreOption) BuildFlowPubSubScoreOption() pubsub.Option { Float64("graylist_threshold", s.peerThresholdParams.GraylistThreshold). Float64("accept_px_threshold", s.peerThresholdParams.AcceptPXThreshold). Float64("opportunistic_graft_threshold", s.peerThresholdParams.OpportunisticGraftThreshold). - Msg("peer score thresholds configured") + Msg("pubsub score thresholds are set") - return pubsub.WithPeerScore( - s.peerScoreParams, - s.peerThresholdParams, - ) + for topic, topicParams := range s.peerScoreParams.Topics { + topicScoreParamLogger := utils.TopicScoreParamsLogger(s.logger, topic, topicParams) + topicScoreParamLogger.Info(). + Msg("pubsub score topic parameters are set for topic") + } + + return s.peerScoreParams, s.peerThresholdParams } func (s *ScoreOption) preparePeerScoreThresholds() { @@ -164,79 +473,70 @@ func (s *ScoreOption) preparePeerScoreThresholds() { } } +// TopicScoreParams returns the topic score parameters for the given topic. If the topic +// score parameters are not set, it returns the default topic score parameters. +// The custom topic parameters are set at the initialization of the score option. +// Args: +// - topic: the topic for which the score parameters are requested. +// Returns: +// - the topic score parameters for the given topic, or the default topic score parameters if +// the topic score parameters are not set. +func (s *ScoreOption) TopicScoreParams(topic *pubsub.Topic) *pubsub.TopicScoreParams { + params, exists := s.peerScoreParams.Topics[topic.String()] + if !exists { + return DefaultTopicScoreParams() + } + return params +} + func defaultPeerScoreParams() *pubsub.PeerScoreParams { + // DO NOT CHANGE THE DEFAULT VALUES, THEY ARE TUNED FOR THE BEST SECURITY PRACTICES. return &pubsub.PeerScoreParams{ + Topics: make(map[string]*pubsub.TopicScoreParams), // we don't set all the parameters, so we skip the atomic validation. // atomic validation fails initialization if any parameter is not set. SkipAtomicValidation: true, - // DecayInterval is the interval over which we decay the effect of past behavior. So that - // a good or bad behavior will not have a permanent effect on the score. - DecayInterval: time.Hour, + // DecayInterval is the interval over which we decay the effect of past behavior, so that + // a good or bad behavior will not have a permanent effect on the penalty. It is also the interval + // that GossipSub uses to refresh the scores of all peers. + DecayInterval: defaultDecayInterval, // DecayToZero defines the maximum value below which a peer scoring counter is reset to zero. // This is to prevent the counter from decaying to a very small value. - // The default value is 0.01, which means that a counter will be reset to zero if it decays to 0.01. // When a counter hits the DecayToZero threshold, it means that the peer did not exhibit the behavior // for a long time, and we can reset the counter. - DecayToZero: 0.01, - // AppSpecificWeight is the weight of the application specific score. + DecayToZero: defaultDecayToZero, + // AppSpecificWeight is the weight of the application specific penalty. AppSpecificWeight: DefaultAppSpecificScoreWeight, + // BehaviourPenaltyThreshold is the threshold above which a peer is penalized for GossipSub-level misbehaviors. + BehaviourPenaltyThreshold: defaultBehaviourPenaltyThreshold, + // BehaviourPenaltyWeight is the weight of the GossipSub-level penalty. + BehaviourPenaltyWeight: defaultBehaviourPenaltyWeight, + // BehaviourPenaltyDecay is the decay of the GossipSub-level penalty (applied every decay interval). + BehaviourPenaltyDecay: defaultBehaviourPenaltyDecay, } } -func (s *ScoreOption) BuildGossipSubScoreOption() pubsub.Option { - s.preparePeerScoreThresholds() +// DefaultTopicScoreParams returns the default score params for topics. +func DefaultTopicScoreParams() *pubsub.TopicScoreParams { + // DO NOT CHANGE THE DEFAULT VALUES, THEY ARE TUNED FOR THE BEST SECURITY PRACTICES. + p := &pubsub.TopicScoreParams{ + TopicWeight: defaultTopicWeight, + SkipAtomicValidation: defaultTopicSkipAtomicValidation, + InvalidMessageDeliveriesWeight: defaultTopicInvalidMessageDeliveriesWeight, + InvalidMessageDeliveriesDecay: defaultTopicInvalidMessageDeliveriesDecay, + TimeInMeshQuantum: defaultTopicTimeInMesh, + MeshMessageDeliveriesWeight: defaultTopicMeshMessageDeliveriesWeight, + MeshMessageDeliveriesDecay: defaultTopicMeshMessageDeliveriesDecay, + MeshMessageDeliveriesCap: defaultTopicMeshMessageDeliveriesCap, + MeshMessageDeliveriesThreshold: defaultTopicMeshMessageDeliveryThreshold, + MeshMessageDeliveriesWindow: defaultMeshMessageDeliveriesWindow, + MeshMessageDeliveriesActivation: defaultMeshMessageDeliveriesActivation, + } - s.logger.Info(). - Float64("gossip_threshold", s.peerThresholdParams.GossipThreshold). - Float64("publish_threshold", s.peerThresholdParams.PublishThreshold). - Float64("graylist_threshold", s.peerThresholdParams.GraylistThreshold). - Float64("accept_px_threshold", s.peerThresholdParams.AcceptPXThreshold). - Float64("opportunistic_graft_threshold", s.peerThresholdParams.OpportunisticGraftThreshold). - Msg("peer score thresholds configured") - - return pubsub.WithPeerScore( - s.peerScoreParams, - s.peerThresholdParams, - ) -} - -func defaultAppSpecificScoreFunction(logger zerolog.Logger, idProvider module.IdentityProvider, validator *SubscriptionValidator) func(peer.ID) float64 { - return func(pid peer.ID) float64 { - lg := logger.With().Str("peer_id", pid.String()).Logger() - - // checks if peer has a valid Flow protocol identity. - flowId, err := HasValidFlowIdentity(idProvider, pid) - if err != nil { - lg.Error(). - Err(err). - Bool(logging.KeySuspicious, true). - Msg("invalid peer identity, penalizing peer") - return MaxAppSpecificPenalty - } - - lg = lg.With(). - Hex("flow_id", logging.ID(flowId.NodeID)). - Str("role", flowId.Role.String()). - Logger() - - // checks if peer has any subscription violation. - if err := validator.CheckSubscribedToAllowedTopics(pid, flowId.Role); err != nil { - lg.Err(err). - Bool(logging.KeySuspicious, true). - Msg("invalid subscription detected, penalizing peer") - return MaxAppSpecificPenalty - } - - // checks if peer is an access node, and if so, pushes it to the - // edges of the network by giving the minimum penalty. - if flowId.Role == flow.RoleAccess { - lg.Trace(). - Msg("pushing access node to edge by penalizing with minimum penalty value") - return MinAppSpecificPenalty - } - - lg.Trace(). - Msg("rewarding well-behaved non-access node peer with maximum reward value") - return MaxAppSpecificReward + if p.MeshMessageDeliveriesWeight >= 0 { + // GossipSub also does a validation, but we want to panic as early as possible. + panic(fmt.Sprintf("invalid mesh message deliveries weight %f", p.MeshMessageDeliveriesWeight)) } + + return p } diff --git a/network/p2p/scoring/scoring_test.go b/network/p2p/scoring/scoring_test.go new file mode 100644 index 00000000000..47e6f27cb57 --- /dev/null +++ b/network/p2p/scoring/scoring_test.go @@ -0,0 +1,151 @@ +package scoring_test + +import ( + "context" + "fmt" + "math/rand" + "testing" + "time" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog" + mocktestify "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/id" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/mock" + flownet "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" + "github.com/onflow/flow-go/network/p2p/p2pconf" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/utils/unittest" +) + +// mockInspectorSuite is a mock implementation of the GossipSubInspectorSuite interface. +// It is used to test the impact of invalid control messages on the scoring and connectivity of nodes in a network. +type mockInspectorSuite struct { + component.Component + t *testing.T + consumer p2p.GossipSubInvCtrlMsgNotifConsumer +} + +// ensures that mockInspectorSuite implements the GossipSubInspectorSuite interface. +var _ p2p.GossipSubInspectorSuite = (*mockInspectorSuite)(nil) + +func (m *mockInspectorSuite) AddInvalidControlMessageConsumer(consumer p2p.GossipSubInvCtrlMsgNotifConsumer) { + require.Nil(m.t, m.consumer) + m.consumer = consumer +} +func (m *mockInspectorSuite) ActiveClustersChanged(_ flow.ChainIDList) { + // no-op +} + +// newMockInspectorSuite creates a new mockInspectorSuite. +// Args: +// - t: the test object used for assertions. +// Returns: +// - a new mockInspectorSuite. +func newMockInspectorSuite(t *testing.T) *mockInspectorSuite { + i := &mockInspectorSuite{ + t: t, + } + + builder := component.NewComponentManagerBuilder() + builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + <-ctx.Done() + }) + + i.Component = builder.Build() + return i +} + +// InspectFunc returns a function that is called when a node receives a control message. +// In this mock implementation, the function does nothing. +func (m *mockInspectorSuite) InspectFunc() func(peer.ID, *pubsub.RPC) error { + return nil +} + +// TestInvalidCtrlMsgScoringIntegration tests the impact of invalid control messages on the scoring and connectivity of nodes in a network. +// It creates a network of 2 nodes, and sends a set of control messages with invalid topic IDs to one of the nodes. +// It then checks that the node receiving the invalid control messages decreases its score for the peer spamming the invalid messages, and +// eventually disconnects from the spamming peer on the gossipsub layer, i.e., messages sent by the spamming peer are no longer +// received by the node. +func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + sporkId := unittest.IdentifierFixture() + idProvider := mock.NewIdentityProvider(t) + + inspectorSuite1 := newMockInspectorSuite(t) + node1, id1 := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(flow.RoleConsensus), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), + p2ptest.OverrideGossipSubRpcInspectorSuiteFactory(func(zerolog.Logger, + flow.Identifier, + *p2pconf.GossipSubRPCInspectorsConfig, + module.GossipSubMetrics, + metrics.HeroCacheMetricsFactory, + flownet.NetworkingType, + module.IdentityProvider) (p2p.GossipSubInspectorSuite, error) { + // override the gossipsub rpc inspector suite factory to return the mock inspector suite + return inspectorSuite1, nil + })) + + node2, id2 := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(flow.RoleConsensus), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) + + ids := flow.IdentityList{&id1, &id2} + nodes := []p2p.LibP2PNode{node1, node2} + + provider := id.NewFixedIdentityProvider(ids) + idProvider.On("ByPeerID", mocktestify.Anything).Return( + func(peerId peer.ID) *flow.Identity { + identity, _ := provider.ByPeerID(peerId) + return identity + }, func(peerId peer.ID) bool { + _, ok := provider.ByPeerID(peerId) + return ok + }) + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + // checks end-to-end message delivery works on GossipSub + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + // now simulates node2 spamming node1 with invalid gossipsub control messages. + for i := 0; i < 30; i++ { + inspectorSuite1.consumer.OnInvalidControlMessageNotification(&p2p.InvCtrlMsgNotif{ + PeerID: node2.Host().ID(), + MsgType: p2pmsg.ControlMessageTypes()[rand.Intn(len(p2pmsg.ControlMessageTypes()))], + Count: 1, + Err: fmt.Errorf("invalid control message"), + }) + } + + // checks no GossipSub message exchange should no longer happen between node1 and node2. + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{node1}, []p2p.LibP2PNode{node2}, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} diff --git a/network/p2p/scoring/subscription_validator.go b/network/p2p/scoring/subscription_validator.go index 0a90b22aaa2..fbffe27752a 100644 --- a/network/p2p/scoring/subscription_validator.go +++ b/network/p2p/scoring/subscription_validator.go @@ -1,6 +1,8 @@ package scoring import ( + "fmt" + "github.com/libp2p/go-libp2p/core/peer" "github.com/onflow/flow-go/model/flow" @@ -8,6 +10,8 @@ import ( p2putils "github.com/onflow/flow-go/network/p2p/utils" ) +// SubscriptionValidator validates that a peer is subscribed to topics that it is allowed to subscribe to. +// It is used to penalize peers that subscribe to topics that they are not allowed to subscribe to in GossipSub. type SubscriptionValidator struct { subscriptionProvider p2p.SubscriptionProvider } @@ -16,19 +20,44 @@ func NewSubscriptionValidator() *SubscriptionValidator { return &SubscriptionValidator{} } -func (v *SubscriptionValidator) RegisterSubscriptionProvider(provider p2p.SubscriptionProvider) { +var _ p2p.SubscriptionValidator = (*SubscriptionValidator)(nil) + +// RegisterSubscriptionProvider registers the subscription provider with the subscription validator. +// This follows a dependency injection pattern. +// Args: +// +// provider: the subscription provider +// +// Returns: +// +// error: if the subscription provider is nil, an error is returned. The error is irrecoverable, i.e., +// it indicates an illegal state in the execution of the code. We expect this error only when there is a bug in the code. +// Such errors should lead to a crash of the node. +func (v *SubscriptionValidator) RegisterSubscriptionProvider(provider p2p.SubscriptionProvider) error { + if v.subscriptionProvider != nil { + return fmt.Errorf("subscription provider already registered") + } v.subscriptionProvider = provider + + return nil } -// CheckSubscribedToAllowedTopics validates all subscriptions a peer has with respect to all Flow topics. -// All errors returned by this method are benign: -// - InvalidSubscriptionError: the peer is subscribed to a topic that is not allowed for its role. +// CheckSubscribedToAllowedTopics checks if a peer is subscribed to topics that it is allowed to subscribe to. +// Args: +// +// pid: the peer ID of the peer to check +// role: the role of the peer to check +// +// Returns: +// error: if the peer is subscribed to topics that it is not allowed to subscribe to, an InvalidSubscriptionError is returned. +// The error is benign, i.e., it does not indicate an illegal state in the execution of the code. We expect this error +// when there are malicious peers in the network. But such errors should not lead to a crash of the node. func (v *SubscriptionValidator) CheckSubscribedToAllowedTopics(pid peer.ID, role flow.Role) error { topics := v.subscriptionProvider.GetSubscribedTopics(pid) for _, topic := range topics { if !p2putils.AllowedSubscription(role, topic) { - return NewInvalidSubscriptionError(topic) + return p2p.NewInvalidSubscriptionError(topic) } } diff --git a/network/p2p/scoring/subscription_validator_test.go b/network/p2p/scoring/subscription_validator_test.go index d0341be891f..549006b3bde 100644 --- a/network/p2p/scoring/subscription_validator_test.go +++ b/network/p2p/scoring/subscription_validator_test.go @@ -33,7 +33,7 @@ func TestSubscriptionValidator_NoSubscribedTopic(t *testing.T) { sp := mockp2p.NewSubscriptionProvider(t) sv := scoring.NewSubscriptionValidator() - sv.RegisterSubscriptionProvider(sp) + require.NoError(t, sv.RegisterSubscriptionProvider(sp)) // mocks peer 1 not subscribed to any topic. peer1 := p2pfixtures.PeerIdFixture(t) @@ -51,7 +51,7 @@ func TestSubscriptionValidator_NoSubscribedTopic(t *testing.T) { func TestSubscriptionValidator_UnknownChannel(t *testing.T) { sp := mockp2p.NewSubscriptionProvider(t) sv := scoring.NewSubscriptionValidator() - sv.RegisterSubscriptionProvider(sp) + require.NoError(t, sv.RegisterSubscriptionProvider(sp)) // mocks peer 1 not subscribed to an unknown topic. peer1 := p2pfixtures.PeerIdFixture(t) @@ -62,7 +62,7 @@ func TestSubscriptionValidator_UnknownChannel(t *testing.T) { for _, role := range flow.Roles() { err := sv.CheckSubscribedToAllowedTopics(peer1, role) require.Error(t, err) - require.True(t, scoring.IsInvalidSubscriptionError(err)) + require.True(t, p2p.IsInvalidSubscriptionError(err)) } } @@ -71,7 +71,7 @@ func TestSubscriptionValidator_UnknownChannel(t *testing.T) { func TestSubscriptionValidator_ValidSubscriptions(t *testing.T) { sp := mockp2p.NewSubscriptionProvider(t) sv := scoring.NewSubscriptionValidator() - sv.RegisterSubscriptionProvider(sp) + require.NoError(t, sv.RegisterSubscriptionProvider(sp)) for _, role := range flow.Roles() { peerId := p2pfixtures.PeerIdFixture(t) @@ -102,7 +102,7 @@ func TestSubscriptionValidator_ValidSubscriptions(t *testing.T) { func TestSubscriptionValidator_SubscribeToAllTopics(t *testing.T) { sp := mockp2p.NewSubscriptionProvider(t) sv := scoring.NewSubscriptionValidator() - sv.RegisterSubscriptionProvider(sp) + require.NoError(t, sv.RegisterSubscriptionProvider(sp)) allChannels := channels.Channels().ExcludePattern(regexp.MustCompile("^(test).*")) sporkID := unittest.IdentifierFixture() @@ -116,7 +116,7 @@ func TestSubscriptionValidator_SubscribeToAllTopics(t *testing.T) { sp.On("GetSubscribedTopics", peerId).Return(allTopics) err := sv.CheckSubscribedToAllowedTopics(peerId, role) require.Error(t, err, role) - require.True(t, scoring.IsInvalidSubscriptionError(err), role) + require.True(t, p2p.IsInvalidSubscriptionError(err), role) } } @@ -125,7 +125,7 @@ func TestSubscriptionValidator_SubscribeToAllTopics(t *testing.T) { func TestSubscriptionValidator_InvalidSubscriptions(t *testing.T) { sp := mockp2p.NewSubscriptionProvider(t) sv := scoring.NewSubscriptionValidator() - sv.RegisterSubscriptionProvider(sp) + require.NoError(t, sv.RegisterSubscriptionProvider(sp)) for _, role := range flow.Roles() { peerId := p2pfixtures.PeerIdFixture(t) @@ -144,7 +144,7 @@ func TestSubscriptionValidator_InvalidSubscriptions(t *testing.T) { sp.On("GetSubscribedTopics", peerId).Return(unauthorizedTopics[:i+1]) err := sv.CheckSubscribedToAllowedTopics(peerId, role) require.Error(t, err, role) - require.True(t, scoring.IsInvalidSubscriptionError(err), role) + require.True(t, p2p.IsInvalidSubscriptionError(err), role) } } } @@ -176,19 +176,22 @@ func TestSubscriptionValidator_Integration(t *testing.T) { idProvider := mock.NewIdentityProvider(t) // one consensus node. conNode, conId := p2ptest.NodeFixture(t, sporkId, t.Name(), + idProvider, p2ptest.WithLogger(unittest.Logger()), - p2ptest.WithPeerScoringEnabled(idProvider), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.WithRole(flow.RoleConsensus)) // two verification node. verNode1, verId1 := p2ptest.NodeFixture(t, sporkId, t.Name(), + idProvider, p2ptest.WithLogger(unittest.Logger()), - p2ptest.WithPeerScoringEnabled(idProvider), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.WithRole(flow.RoleVerification)) verNode2, verId2 := p2ptest.NodeFixture(t, sporkId, t.Name(), + idProvider, p2ptest.WithLogger(unittest.Logger()), - p2ptest.WithPeerScoringEnabled(idProvider), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.WithRole(flow.RoleVerification)) ids := flow.IdentityList{&conId, &verId1, &verId2} diff --git a/network/p2p/subscription.go b/network/p2p/subscription.go index b7d9f4558d3..9d4a117d0bc 100644 --- a/network/p2p/subscription.go +++ b/network/p2p/subscription.go @@ -1,6 +1,13 @@ package p2p -import "github.com/libp2p/go-libp2p/core/peer" +import ( + "errors" + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/model/flow" +) // SubscriptionProvider provides a list of topics a peer is subscribed to. type SubscriptionProvider interface { @@ -12,6 +19,23 @@ type SubscriptionProvider interface { GetSubscribedTopics(pid peer.ID) []string } +// SubscriptionValidator validates the subscription of a peer to a topic. +// It is used to ensure that a peer is only subscribed to topics that it is allowed to subscribe to. +type SubscriptionValidator interface { + // RegisterSubscriptionProvider registers the subscription provider with the subscription validator. + // If there is a subscription provider already registered, it will be replaced by the new one. + RegisterSubscriptionProvider(provider SubscriptionProvider) error + // CheckSubscribedToAllowedTopics checks if a peer is subscribed to topics that it is allowed to subscribe to. + // Args: + // pid: the peer ID of the peer to check + // role: the role of the peer to check + // Returns: + // error: if the peer is subscribed to topics that it is not allowed to subscribe to, an InvalidSubscriptionError is returned. + // The error is benign, i.e., it does not indicate an illegal state in the execution of the code. We expect this error + // when there are malicious peers in the network. But such errors should not lead to a crash of the node. + CheckSubscribedToAllowedTopics(pid peer.ID, role flow.Role) error +} + // TopicProvider provides a low-level abstraction for pubsub to perform topic-related queries. // This abstraction is provided to encapsulate the pubsub implementation details from the rest of the codebase. type TopicProvider interface { @@ -25,3 +49,25 @@ type TopicProvider interface { // subscribed peers for topics A and B, and querying for topic C will return an empty list. ListPeers(topic string) []peer.ID } + +// InvalidSubscriptionError indicates that a peer has subscribed to a topic that is not allowed for its role. +// This error is benign, i.e., it does not indicate an illegal state in the execution of the code. We expect this error +// when there are malicious peers in the network. But such errors should not lead to a crash of the node.32 +type InvalidSubscriptionError struct { + topic string // the topic that the peer is subscribed to, but not allowed to. +} + +func NewInvalidSubscriptionError(topic string) error { + return InvalidSubscriptionError{ + topic: topic, + } +} + +func (e InvalidSubscriptionError) Error() string { + return fmt.Sprintf("unauthorized subscription: %s", e.topic) +} + +func IsInvalidSubscriptionError(this error) bool { + var e InvalidSubscriptionError + return errors.As(this, &e) +} diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index 34d634868e1..50ecb5a568f 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -4,6 +4,8 @@ import ( "bufio" "context" "crypto/rand" + crand "math/rand" + "sync" "testing" "time" @@ -14,24 +16,26 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" "github.com/libp2p/go-libp2p/core/routing" + discoveryBackoff "github.com/libp2p/go-libp2p/p2p/discovery/backoff" mh "github.com/multiformats/go-multihash" "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" + flownet "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" - "github.com/onflow/flow-go/network/internal/testutils" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/connection" p2pdht "github.com/onflow/flow-go/network/p2p/dht" - "github.com/onflow/flow-go/network/p2p/distributor" "github.com/onflow/flow-go/network/p2p/p2pbuilder" - inspectorbuilder "github.com/onflow/flow-go/network/p2p/p2pbuilder/inspector" + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" + "github.com/onflow/flow-go/network/p2p/p2pconf" "github.com/onflow/flow-go/network/p2p/unicast" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" @@ -54,24 +58,37 @@ func NodeFixture( t *testing.T, sporkID flow.Identifier, dhtPrefix string, + idProvider module.IdentityProvider, opts ...NodeFixtureParameterOption, ) (p2p.LibP2PNode, flow.Identity) { - // default parameters - logger := unittest.Logger().Level(zerolog.ErrorLevel) - rpcInspectors, err := inspectorbuilder.NewGossipSubInspectorBuilder(logger, sporkID, inspectorbuilder.DefaultGossipSubRPCInspectorsConfig(), distributor.DefaultGossipSubInspectorNotificationDistributor(logger)).Build() + defaultFlowConfig, err := config.DefaultConfig() require.NoError(t, err) + + logger := unittest.Logger().Level(zerolog.WarnLevel) + require.NotNil(t, idProvider) + connectionGater := NewConnectionGater(idProvider, func(p peer.ID) error { + return nil + }) + require.NotNil(t, connectionGater) parameters := &NodeFixtureParameters{ - HandlerFunc: func(network.Stream) {}, - Unicasts: nil, - Key: NetworkingKeyFixtures(t), - Address: unittest.DefaultAddress, - Logger: logger, - Role: flow.RoleCollection, - CreateStreamRetryDelay: unicast.DefaultRetryDelay, - Metrics: metrics.NewNoopCollector(), - ResourceManager: testutils.NewResourceManager(t), + NetworkingType: flownet.PrivateNetwork, + HandlerFunc: func(network.Stream) {}, + Unicasts: nil, + Key: NetworkingKeyFixtures(t), + Address: unittest.DefaultAddress, + Logger: logger, + Role: flow.RoleCollection, + CreateStreamRetryDelay: unicast.DefaultRetryDelay, + IdProvider: idProvider, + MetricsCfg: &p2pconfig.MetricsConfig{ + HeroCacheFactory: metrics.NewNoopHeroCacheMetricsFactory(), + Metrics: metrics.NewNoopCollector(), + }, + ResourceManager: &network.NullResourceManager{}, GossipSubPeerScoreTracerInterval: 0, // disabled by default - GossipSubRPCInspectors: rpcInspectors, + ConnGater: connectionGater, + PeerManagerConfig: PeerManagerConfigFixture(), // disabled by default + GossipSubRPCInspectorCfg: &defaultFlowConfig.NetworkConfig.GossipSubRPCInspectorsConfig, } for _, opt := range opts { @@ -85,29 +102,40 @@ func NodeFixture( logger = parameters.Logger.With().Hex("node_id", logging.ID(identity.NodeID)).Logger() - connManager, err := connection.NewConnManager(logger, parameters.Metrics, connection.DefaultConnManagerConfig()) + connManager, err := connection.NewConnManager(logger, parameters.MetricsCfg.Metrics, &defaultFlowConfig.NetworkConfig.ConnectionManagerConfig) require.NoError(t, err) builder := p2pbuilder.NewNodeBuilder( logger, - parameters.Metrics, + parameters.MetricsCfg, + parameters.NetworkingType, parameters.Address, parameters.Key, sporkID, - p2pbuilder.DefaultResourceManagerConfig()). + parameters.IdProvider, + &defaultFlowConfig.NetworkConfig.ResourceManagerConfig, + parameters.GossipSubRPCInspectorCfg, + parameters.PeerManagerConfig, + &p2p.DisallowListCacheConfig{ + MaxSize: uint32(1000), + Metrics: metrics.NewNoopCollector(), + }). SetConnectionManager(connManager). SetRoutingSystem(func(c context.Context, h host.Host) (routing.Routing, error) { return p2pdht.NewDHT(c, h, protocol.ID(protocols.FlowDHTProtocolIDPrefix+sporkID.String()+"/"+dhtPrefix), logger, - parameters.Metrics, + parameters.MetricsCfg.Metrics, parameters.DhtOptions..., ) }). SetCreateNode(p2pbuilder.DefaultCreateNodeFunc). SetStreamCreationRetryInterval(parameters.CreateStreamRetryDelay). - SetResourceManager(parameters.ResourceManager). - SetGossipSubRPCInspectors(parameters.GossipSubRPCInspectors...) + SetResourceManager(parameters.ResourceManager) + + if parameters.GossipSubRpcInspectorSuiteFactory != nil { + builder.OverrideDefaultRpcInspectorSuiteFactory(parameters.GossipSubRpcInspectorSuiteFactory) + } if parameters.ResourceManager != nil { builder.SetResourceManager(parameters.ResourceManager) @@ -118,12 +146,7 @@ func NodeFixture( } if parameters.PeerScoringEnabled { - builder.EnableGossipSubPeerScoring(parameters.IdProvider, parameters.PeerScoreConfig) - } - - if parameters.UpdateInterval != 0 { - require.NotNil(t, parameters.PeerProvider) - builder.SetPeerManagerOptions(parameters.ConnectionPruning, parameters.UpdateInterval) + builder.EnableGossipSubScoringWithOverride(parameters.PeerScoringConfigOverride) } if parameters.GossipSubFactory != nil && parameters.GossipSubConfig != nil { @@ -138,13 +161,19 @@ func NodeFixture( builder.SetGossipSubTracer(parameters.PubSubTracer) } + if parameters.UnicastRateLimitDistributor != nil { + builder.SetRateLimiterDistributor(parameters.UnicastRateLimitDistributor) + } + builder.SetGossipSubScoreTracerInterval(parameters.GossipSubPeerScoreTracerInterval) n, err := builder.Build() require.NoError(t, err) - err = n.WithDefaultUnicastProtocol(parameters.HandlerFunc, parameters.Unicasts) - require.NoError(t, err) + if parameters.HandlerFunc != nil { + err = n.WithDefaultUnicastProtocol(parameters.HandlerFunc, parameters.Unicasts) + require.NoError(t, err) + } // get the actual IP and port that have been assigned by the subsystem ip, port, err := n.GetIPPort() @@ -161,29 +190,49 @@ func NodeFixture( type NodeFixtureParameterOption func(*NodeFixtureParameters) type NodeFixtureParameters struct { - HandlerFunc network.StreamHandler - Unicasts []protocols.ProtocolName - Key crypto.PrivateKey - Address string - DhtOptions []dht.Option - Role flow.Role - Logger zerolog.Logger - PeerScoringEnabled bool - IdProvider module.IdentityProvider - PeerScoreConfig *p2p.PeerScoringConfig - ConnectionPruning bool // peer manager parameter - UpdateInterval time.Duration // peer manager parameter - PeerProvider p2p.PeersProvider // peer manager parameter - ConnGater connmgr.ConnectionGater - ConnManager connmgr.ConnManager - GossipSubFactory p2p.GossipSubFactoryFunc - GossipSubConfig p2p.GossipSubAdapterConfigFunc - Metrics module.LibP2PMetrics - ResourceManager network.ResourceManager - PubSubTracer p2p.PubSubTracer - GossipSubPeerScoreTracerInterval time.Duration // intervals at which the peer score is updated and logged. - CreateStreamRetryDelay time.Duration - GossipSubRPCInspectors []p2p.GossipSubRPCInspector + HandlerFunc network.StreamHandler + NetworkingType flownet.NetworkingType + Unicasts []protocols.ProtocolName + Key crypto.PrivateKey + Address string + DhtOptions []dht.Option + Role flow.Role + Logger zerolog.Logger + PeerScoringEnabled bool + IdProvider module.IdentityProvider + PeerScoringConfigOverride *p2p.PeerScoringConfigOverride + PeerManagerConfig *p2pconfig.PeerManagerConfig + PeerProvider p2p.PeersProvider // peer manager parameter + ConnGater p2p.ConnectionGater + ConnManager connmgr.ConnManager + GossipSubFactory p2p.GossipSubFactoryFunc + GossipSubConfig p2p.GossipSubAdapterConfigFunc + MetricsCfg *p2pconfig.MetricsConfig + ResourceManager network.ResourceManager + PubSubTracer p2p.PubSubTracer + GossipSubPeerScoreTracerInterval time.Duration // intervals at which the peer score is updated and logged. + CreateStreamRetryDelay time.Duration + UnicastRateLimitDistributor p2p.UnicastRateLimiterDistributor + GossipSubRpcInspectorSuiteFactory p2p.GossipSubRpcInspectorSuiteFactoryFunc + GossipSubRPCInspectorCfg *p2pconf.GossipSubRPCInspectorsConfig +} + +func WithUnicastRateLimitDistributor(distributor p2p.UnicastRateLimiterDistributor) NodeFixtureParameterOption { + return func(p *NodeFixtureParameters) { + p.UnicastRateLimitDistributor = distributor + } +} + +func OverrideGossipSubRpcInspectorSuiteFactory(factory p2p.GossipSubRpcInspectorSuiteFactoryFunc) NodeFixtureParameterOption { + return func(p *NodeFixtureParameters) { + p.GossipSubRpcInspectorSuiteFactory = factory + } +} + +func OverrideGossipSubRpcInspectorConfig(cfg *p2pconf.GossipSubRPCInspectorsConfig) NodeFixtureParameterOption { + return func(p *NodeFixtureParameters) { + p.GossipSubRPCInspectorCfg = cfg + } } func WithCreateStreamRetryDelay(delay time.Duration) NodeFixtureParameterOption { @@ -192,10 +241,21 @@ func WithCreateStreamRetryDelay(delay time.Duration) NodeFixtureParameterOption } } -func WithPeerScoringEnabled(idProvider module.IdentityProvider) NodeFixtureParameterOption { +// EnablePeerScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. +// Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. +// Anything that is left to nil or zero value in the override will be ignored and the default value will be used. +// Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. +// Default Use Tip: use p2p.PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. +// Args: +// - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use p2p.PeerScoringConfigNoOverride for production or when +// you don't want to override the default peer scoring config. +// +// Returns: +// - NodeFixtureParameterOption: a function that can be passed to the NodeFixture function to enable peer scoring. +func EnablePeerScoringWithOverride(override *p2p.PeerScoringConfigOverride) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { p.PeerScoringEnabled = true - p.IdProvider = idProvider + p.PeerScoringConfigOverride = override } } @@ -211,10 +271,9 @@ func WithDefaultStreamHandler(handler network.StreamHandler) NodeFixtureParamete } } -func WithPeerManagerEnabled(connectionPruning bool, updateInterval time.Duration, peerProvider p2p.PeersProvider) NodeFixtureParameterOption { +func WithPeerManagerEnabled(cfg *p2pconfig.PeerManagerConfig, peerProvider p2p.PeersProvider) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { - p.ConnectionPruning = connectionPruning - p.UpdateInterval = updateInterval + p.PeerManagerConfig = cfg p.PeerProvider = peerProvider } } @@ -243,7 +302,7 @@ func WithDHTOptions(opts ...dht.Option) NodeFixtureParameterOption { } } -func WithConnectionGater(connGater connmgr.ConnectionGater) NodeFixtureParameterOption { +func WithConnectionGater(connGater p2p.ConnectionGater) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { p.ConnGater = connGater } @@ -261,9 +320,9 @@ func WithRole(role flow.Role) NodeFixtureParameterOption { } } -func WithPeerScoreParamsOption(cfg *p2p.PeerScoringConfig) NodeFixtureParameterOption { +func WithPeerScoreParamsOption(cfg *p2p.PeerScoringConfigOverride) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { - p.PeerScoreConfig = cfg + p.PeerScoringConfigOverride = cfg } } @@ -275,7 +334,7 @@ func WithLogger(logger zerolog.Logger) NodeFixtureParameterOption { func WithMetricsCollector(metrics module.NetworkMetrics) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { - p.Metrics = metrics + p.MetricsCfg.Metrics = metrics } } @@ -293,9 +352,53 @@ func WithDefaultResourceManager() NodeFixtureParameterOption { } } +func WithUnicastHandlerFunc(handler network.StreamHandler) NodeFixtureParameterOption { + return func(p *NodeFixtureParameters) { + p.HandlerFunc = handler + } +} + +// PeerManagerConfigFixture is a test fixture that sets the default config for the peer manager. +func PeerManagerConfigFixture(opts ...func(*p2pconfig.PeerManagerConfig)) *p2pconfig.PeerManagerConfig { + cfg := &p2pconfig.PeerManagerConfig{ + ConnectionPruning: true, + UpdateInterval: 1 * time.Second, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + } + for _, opt := range opts { + opt(cfg) + } + return cfg +} + +// WithZeroJitterAndZeroBackoff is a test fixture that sets the default config for the peer manager. +// It uses a backoff connector with zero jitter and zero backoff. +func WithZeroJitterAndZeroBackoff(t *testing.T) func(*p2pconfig.PeerManagerConfig) { + return func(cfg *p2pconfig.PeerManagerConfig) { + cfg.ConnectorFactory = func(host host.Host) (p2p.Connector, error) { + cacheSize := 100 + dialTimeout := time.Minute * 2 + backoff := discoveryBackoff.NewExponentialBackoff( + 1*time.Second, + 1*time.Hour, + func(_, _, _ time.Duration, _ *crand.Rand) time.Duration { + return 0 // no jitter + }, + time.Second, + 1, + 0, + crand.NewSource(crand.Int63()), + ) + backoffConnector, err := discoveryBackoff.NewBackoffConnector(host, cacheSize, dialTimeout, backoff) + require.NoError(t, err) + return backoffConnector, nil + } + } +} + // NodesFixture is a test fixture that creates a number of libp2p nodes with the given callback function for stream handling. // It returns the nodes and their identities. -func NodesFixture(t *testing.T, sporkID flow.Identifier, dhtPrefix string, count int, opts ...NodeFixtureParameterOption) ([]p2p.LibP2PNode, +func NodesFixture(t *testing.T, sporkID flow.Identifier, dhtPrefix string, count int, idProvider module.IdentityProvider, opts ...NodeFixtureParameterOption) ([]p2p.LibP2PNode, flow.IdentityList) { var nodes []p2p.LibP2PNode @@ -303,7 +406,7 @@ func NodesFixture(t *testing.T, sporkID flow.Identifier, dhtPrefix string, count var identities flow.IdentityList for i := 0; i < count; i++ { // create a node on localhost with a random port assigned by the OS - node, identity := NodeFixture(t, sporkID, dhtPrefix, opts...) + node, identity := NodeFixture(t, sporkID, dhtPrefix, idProvider, opts...) nodes = append(nodes, node) identities = append(identities, &identity) } @@ -377,20 +480,71 @@ func LetNodesDiscoverEachOther(t *testing.T, ctx context.Context, nodes []p2p.Li } } -// EnsureConnected ensures that the given nodes are connected to each other. +// TryConnectionAndEnsureConnected tries connecting nodes to each other and ensures that the given nodes are connected to each other. // It fails the test if any of the nodes is not connected to any other node. -func EnsureConnected(t *testing.T, ctx context.Context, nodes []p2p.LibP2PNode) { +func TryConnectionAndEnsureConnected(t *testing.T, ctx context.Context, nodes []p2p.LibP2PNode) { for _, node := range nodes { for _, other := range nodes { if node == other { continue } require.NoError(t, node.Host().Connect(ctx, other.Host().Peerstore().PeerInfo(other.Host().ID()))) + // the other node should be connected to this node require.Equal(t, node.Host().Network().Connectedness(other.Host().ID()), network.Connected) + // at least one connection should be established + require.True(t, len(node.Host().Network().ConnsToPeer(other.Host().ID())) > 0) } } } +// RequireConnectedEventually ensures eventually that the given nodes are already connected to each other. +// It fails the test if any of the nodes is not connected to any other node. +// Args: +// - nodes: the nodes to check +// - tick: the tick duration +// - timeout: the timeout duration +func RequireConnectedEventually(t *testing.T, nodes []p2p.LibP2PNode, tick time.Duration, timeout time.Duration) { + require.Eventually(t, func() bool { + for _, node := range nodes { + for _, other := range nodes { + if node == other { + continue + } + if node.Host().Network().Connectedness(other.Host().ID()) != network.Connected { + return false + } + if len(node.Host().Network().ConnsToPeer(other.Host().ID())) == 0 { + return false + } + } + } + return true + }, timeout, tick) +} + +// RequireEventuallyNotConnected ensures eventually that the given groups of nodes are not connected to each other. +// It fails the test if any of the nodes from groupA is connected to any of the nodes from groupB. +// Args: +// - groupA: the first group of nodes +// - groupB: the second group of nodes +// - tick: the tick duration +// - timeout: the timeout duration +func RequireEventuallyNotConnected(t *testing.T, groupA []p2p.LibP2PNode, groupB []p2p.LibP2PNode, tick time.Duration, timeout time.Duration) { + require.Eventually(t, func() bool { + for _, node := range groupA { + for _, other := range groupB { + if node.Host().Network().Connectedness(other.Host().ID()) == network.Connected { + return false + } + if len(node.Host().Network().ConnsToPeer(other.Host().ID())) > 0 { + return false + } + } + } + return true + }, timeout, tick) +} + // EnsureStreamCreationInBothDirections ensure that between each pair of nodes in the given list, a stream is created in both directions. func EnsureStreamCreationInBothDirections(t *testing.T, ctx context.Context, nodes []p2p.LibP2PNode) { for _, this := range nodes { @@ -407,17 +561,20 @@ func EnsureStreamCreationInBothDirections(t *testing.T, ctx context.Context, nod } // EnsurePubsubMessageExchange ensures that the given connected nodes exchange the given message on the given channel through pubsub. -// Note: EnsureConnected() must be called to connect all nodes before calling this function. -func EnsurePubsubMessageExchange(t *testing.T, ctx context.Context, nodes []p2p.LibP2PNode, messageFactory func() (interface{}, channels.Topic)) { - _, topic := messageFactory() - +// Args: +// - nodes: the nodes to exchange messages +// - ctx: the context- the test will fail if the context expires. +// - topic: the topic to exchange messages on +// - count: the number of messages to exchange from each node. +// - messageFactory: a function that creates a unique message to be published by the node. +// The function should return a different message each time it is called. +// +// Note-1: this function assumes a timeout of 5 seconds for each message to be received. +// Note-2: TryConnectionAndEnsureConnected() must be called to connect all nodes before calling this function. +func EnsurePubsubMessageExchange(t *testing.T, ctx context.Context, nodes []p2p.LibP2PNode, topic channels.Topic, count int, messageFactory func() interface{}) { subs := make([]p2p.Subscription, len(nodes)) for i, node := range nodes { - ps, err := node.Subscribe( - topic, - validator.TopicValidator( - unittest.Logger(), - unittest.AllowAllPeerFilter())) + ps, err := node.Subscribe(topic, validator.TopicValidator(unittest.Logger(), unittest.AllowAllPeerFilter())) require.NoError(t, err) subs[i] = ps } @@ -429,14 +586,52 @@ func EnsurePubsubMessageExchange(t *testing.T, ctx context.Context, nodes []p2p. require.True(t, ok) for _, node := range nodes { + for i := 0; i < count; i++ { + // creates a unique message to be published by the node + msg := messageFactory() + data := p2pfixtures.MustEncodeEvent(t, msg, channel) + require.NoError(t, node.Publish(ctx, topic, data)) + + // wait for the message to be received by all nodes + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + p2pfixtures.SubsMustReceiveMessage(t, ctx, data, subs) + cancel() + } + } +} + +// EnsurePubsubMessageExchangeFromNode ensures that the given node exchanges the given message on the given channel through pubsub with the other nodes. +// Args: +// - node: the node to exchange messages +// +// - ctx: the context- the test will fail if the context expires. +// - sender: the node that sends the message to the other node. +// - receiver: the node that receives the message from the other node. +// - topic: the topic to exchange messages on. +// - count: the number of messages to exchange from `sender` to `receiver`. +// - messageFactory: a function that creates a unique message to be published by the node. +func EnsurePubsubMessageExchangeFromNode(t *testing.T, ctx context.Context, sender p2p.LibP2PNode, receiver p2p.LibP2PNode, topic channels.Topic, count int, messageFactory func() interface{}) { + _, err := sender.Subscribe(topic, validator.TopicValidator(unittest.Logger(), unittest.AllowAllPeerFilter())) + require.NoError(t, err) + + toSub, err := receiver.Subscribe(topic, validator.TopicValidator(unittest.Logger(), unittest.AllowAllPeerFilter())) + require.NoError(t, err) + + // let subscriptions propagate + time.Sleep(1 * time.Second) + + channel, ok := channels.ChannelFromTopic(topic) + require.True(t, ok) + + for i := 0; i < count; i++ { // creates a unique message to be published by the node - msg, _ := messageFactory() + msg := messageFactory() data := p2pfixtures.MustEncodeEvent(t, msg, channel) - require.NoError(t, node.Publish(ctx, topic, data)) + require.NoError(t, sender.Publish(ctx, topic, data)) // wait for the message to be received by all nodes ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - p2pfixtures.SubsMustReceiveMessage(t, ctx, data, subs) + p2pfixtures.SubsMustReceiveMessage(t, ctx, data, []p2p.Subscription{toSub}) cancel() } } @@ -453,3 +648,106 @@ func PeerIdFixture(t *testing.T) peer.ID { return peer.ID(h) } + +// EnsureNotConnectedBetweenGroups ensures no connection exists between the given groups of nodes. +func EnsureNotConnectedBetweenGroups(t *testing.T, ctx context.Context, groupA []p2p.LibP2PNode, groupB []p2p.LibP2PNode) { + // ensure no connection from group A to group B + p2pfixtures.EnsureNotConnected(t, ctx, groupA, groupB) + // ensure no connection from group B to group A + p2pfixtures.EnsureNotConnected(t, ctx, groupB, groupA) +} + +// EnsureNoPubsubMessageExchange ensures that the no pubsub message is exchanged "from" the given nodes "to" the given nodes. +// Args: +// - from: the nodes that send messages to the other group but their message must not be received by the other group. +// +// - to: the nodes that are the target of the messages sent by the other group ("from") but must not receive any message from them. +// - topic: the topic to exchange messages on. +// - count: the number of messages to exchange from each node. +// - messageFactory: a function that creates a unique message to be published by the node. +func EnsureNoPubsubMessageExchange(t *testing.T, ctx context.Context, from []p2p.LibP2PNode, to []p2p.LibP2PNode, topic channels.Topic, count int, messageFactory func() interface{}) { + subs := make([]p2p.Subscription, len(to)) + tv := validator.TopicValidator( + unittest.Logger(), + unittest.AllowAllPeerFilter()) + var err error + for _, node := range from { + _, err = node.Subscribe(topic, tv) + require.NoError(t, err) + } + + for i, node := range to { + s, err := node.Subscribe(topic, tv) + require.NoError(t, err) + subs[i] = s + } + + // let subscriptions propagate + time.Sleep(1 * time.Second) + + wg := &sync.WaitGroup{} + for _, node := range from { + node := node // capture range variable + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + // creates a unique message to be published by the node. + msg := messageFactory() + channel, ok := channels.ChannelFromTopic(topic) + require.True(t, ok) + data := p2pfixtures.MustEncodeEvent(t, msg, channel) + + // ensure the message is NOT received by any of the nodes. + require.NoError(t, node.Publish(ctx, topic, data)) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + p2pfixtures.SubsMustNeverReceiveAnyMessage(t, ctx, subs) + cancel() + wg.Done() + }() + } + } + + // we wait for 5 seconds at most for the messages to be exchanged, hence we wait for a total of 6 seconds here to ensure + // that the goroutines are done in a timely manner. + unittest.RequireReturnsBefore(t, wg.Wait, 6*time.Second, "timed out waiting for messages to be exchanged") +} + +// EnsureNoPubsubExchangeBetweenGroups ensures that no pubsub message is exchanged between the given groups of nodes. +// Args: +// - t: *testing.T instance +// - ctx: context.Context instance +// - groupA: first group of nodes- no message should be exchanged from any node of this group to the other group. +// - groupB: second group of nodes- no message should be exchanged from any node of this group to the other group. +// - topic: pubsub topic- no message should be exchanged on this topic. +// - count: number of messages to be exchanged- no message should be exchanged. +// - messageFactory: function to create a unique message to be published by the node. +func EnsureNoPubsubExchangeBetweenGroups(t *testing.T, ctx context.Context, groupA []p2p.LibP2PNode, groupB []p2p.LibP2PNode, topic channels.Topic, count int, messageFactory func() interface{}) { + // ensure no message exchange from group A to group B + EnsureNoPubsubMessageExchange(t, ctx, groupA, groupB, topic, count, messageFactory) + // ensure no message exchange from group B to group A + EnsureNoPubsubMessageExchange(t, ctx, groupB, groupA, topic, count, messageFactory) +} + +// PeerIdSliceFixture returns a slice of random peer IDs for testing. +// peer ID is the identifier of a node on the libp2p network. +// Args: +// - t: *testing.T instance +// - n: number of peer IDs to generate +// Returns: +// - peer.IDSlice: slice of peer IDs +func PeerIdSliceFixture(t *testing.T, n int) peer.IDSlice { + ids := make([]peer.ID, n) + for i := 0; i < n; i++ { + ids[i] = PeerIdFixture(t) + } + return ids +} + +// NewConnectionGater creates a new connection gater for testing with given allow listing filter. +func NewConnectionGater(idProvider module.IdentityProvider, allowListFilter p2p.PeerFilter) p2p.ConnectionGater { + filters := []p2p.PeerFilter{allowListFilter} + return connection.NewConnGater(unittest.Logger(), + idProvider, + connection.WithOnInterceptPeerDialFilters(filters), + connection.WithOnInterceptSecuredFilters(filters)) +} diff --git a/network/p2p/test/message.go b/network/p2p/test/message.go new file mode 100644 index 00000000000..cb2d762393d --- /dev/null +++ b/network/p2p/test/message.go @@ -0,0 +1,65 @@ +package p2ptest + +import ( + "testing" + + pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/utils/unittest" +) + +// WithFrom is a test helper that returns a function that sets the from field of a pubsub message to the given peer id. +func WithFrom(from peer.ID) func(*pb.Message) { + return func(m *pb.Message) { + m.From = []byte(from) + } +} + +// WithTopic is a test helper that returns a function that sets the topic of a pubsub message to the given topic. +func WithTopic(topic string) func(*pb.Message) { + return func(m *pb.Message) { + m.Topic = &topic + } +} + +// WithoutSignature is a test helper that returns a function that sets the signature of a pubsub message to nil, effectively removing the signature. +func WithoutSignature() func(*pb.Message) { + return func(m *pb.Message) { + m.Signature = nil + } +} + +// WithoutSignerId is a test helper that returns a function that sets the from field of a pubsub message to nil, effectively removing the signer id. +func WithoutSignerId() func(*pb.Message) { + return func(m *pb.Message) { + m.From = nil + } +} + +// PubsubMessageFixture is a test helper that returns a random pubsub message with the given options applied. +// If no options are provided, the message will be random. +// Args: +// +// t: testing.T +// +// opt: variadic list of options to apply to the message +// Returns: +// *pb.Message: pubsub message +func PubsubMessageFixture(t *testing.T, opts ...func(*pb.Message)) *pb.Message { + topic := unittest.RandomStringFixture(t, 10) + + m := &pb.Message{ + Data: unittest.RandomByteSlice(t, 100), + Topic: &topic, + Signature: unittest.RandomByteSlice(t, 100), + From: unittest.RandomByteSlice(t, 100), + Seqno: unittest.RandomByteSlice(t, 100), + } + + for _, opt := range opts { + opt(m) + } + + return m +} diff --git a/network/p2p/test/sporking_test.go b/network/p2p/test/sporking_test.go index 1fa099013f3..6f1b784fc72 100644 --- a/network/p2p/test/sporking_test.go +++ b/network/p2p/test/sporking_test.go @@ -5,16 +5,16 @@ import ( "testing" "time" - "github.com/onflow/flow-go/model/flow" - libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" - "github.com/onflow/flow-go/network" - "github.com/onflow/flow-go/network/message" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/model/flow" + libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/message" + "github.com/onflow/flow-go/network/p2p" p2ptest "github.com/onflow/flow-go/network/p2p/test" @@ -40,7 +40,7 @@ import ( // TestCrosstalkPreventionOnNetworkKeyChange tests that a node from the old chain cannot talk to a node in the new chain // if it's network key is updated while the libp2p protocol ID remains the same func TestCrosstalkPreventionOnNetworkKeyChange(t *testing.T) { - unittest.SkipUnless(t, unittest.TEST_FLAKY, "flaky test - passing in Flaky Test Monitor but keeps failing in CI and keeps blocking many PRs") + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -60,8 +60,10 @@ func TestCrosstalkPreventionOnNetworkKeyChange(t *testing.T) { node1, id1 := p2ptest.NodeFixture(t, sporkId, "test_crosstalk_prevention_on_network_key_change", + idProvider, p2ptest.WithNetworkingPrivateKey(node1key), ) + idProvider.SetIdentities(flow.IdentityList{&id1}) p2ptest.StartNode(t, signalerCtx1, node1, 100*time.Millisecond) defer p2ptest.StopNode(t, node1, cancel1, 100*time.Millisecond) @@ -74,8 +76,11 @@ func TestCrosstalkPreventionOnNetworkKeyChange(t *testing.T) { node2, id2 := p2ptest.NodeFixture(t, sporkId, "test_crosstalk_prevention_on_network_key_change", + idProvider, p2ptest.WithNetworkingPrivateKey(node2key), ) + idProvider.SetIdentities(flow.IdentityList{&id1, &id2}) + p2ptest.StartNode(t, signalerCtx2, node2, 100*time.Millisecond) peerInfo2, err := utils.PeerAddressInfo(id2) @@ -95,9 +100,11 @@ func TestCrosstalkPreventionOnNetworkKeyChange(t *testing.T) { node2, id2New := p2ptest.NodeFixture(t, sporkId, "test_crosstalk_prevention_on_network_key_change", + idProvider, p2ptest.WithNetworkingPrivateKey(node2keyNew), p2ptest.WithNetworkingAddress(id2.Address), ) + idProvider.SetIdentities(flow.IdentityList{&id1, &id2New}) p2ptest.StartNode(t, signalerCtx2a, node2, 100*time.Millisecond) defer p2ptest.StopNode(t, node2, cancel2a, 100*time.Millisecond) @@ -114,6 +121,7 @@ func TestCrosstalkPreventionOnNetworkKeyChange(t *testing.T) { // TestOneToOneCrosstalkPrevention tests that a node from the old chain cannot talk directly to a node in the new chain // if the Flow libp2p protocol ID is updated while the network keys are kept the same. func TestOneToOneCrosstalkPrevention(t *testing.T) { + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -129,7 +137,7 @@ func TestOneToOneCrosstalkPrevention(t *testing.T) { sporkId1 := unittest.IdentifierFixture() // create and start node 1 on localhost and random port - node1, id1 := p2ptest.NodeFixture(t, sporkId1, "test_one_to_one_crosstalk_prevention") + node1, id1 := p2ptest.NodeFixture(t, sporkId1, "test_one_to_one_crosstalk_prevention", idProvider) p2ptest.StartNode(t, signalerCtx1, node1, 100*time.Millisecond) defer p2ptest.StopNode(t, node1, cancel1, 100*time.Millisecond) @@ -138,8 +146,9 @@ func TestOneToOneCrosstalkPrevention(t *testing.T) { require.NoError(t, err) // create and start node 2 on localhost and random port - node2, id2 := p2ptest.NodeFixture(t, sporkId1, "test_one_to_one_crosstalk_prevention") + node2, id2 := p2ptest.NodeFixture(t, sporkId1, "test_one_to_one_crosstalk_prevention", idProvider) + idProvider.SetIdentities(flow.IdentityList{&id1, &id2}) p2ptest.StartNode(t, signalerCtx2, node2, 100*time.Millisecond) // create stream from node 2 to node 1 @@ -153,8 +162,10 @@ func TestOneToOneCrosstalkPrevention(t *testing.T) { node2, id2New := p2ptest.NodeFixture(t, unittest.IdentifierFixture(), // update the flow root id for node 2. node1 is still listening on the old protocol "test_one_to_one_crosstalk_prevention", + idProvider, p2ptest.WithNetworkingAddress(id2.Address), ) + idProvider.SetIdentities(flow.IdentityList{&id1, &id2New}) p2ptest.StartNode(t, signalerCtx2a, node2, 100*time.Millisecond) defer p2ptest.StopNode(t, node2, cancel2a, 100*time.Millisecond) @@ -170,6 +181,7 @@ func TestOneToOneCrosstalkPrevention(t *testing.T) { // TestOneToKCrosstalkPrevention tests that a node from the old chain cannot talk to a node in the new chain via PubSub // if the channel is updated while the network keys are kept the same. func TestOneToKCrosstalkPrevention(t *testing.T) { + idProvider := unittest.NewUpdatableIDProvider(flow.IdentityList{}) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -183,18 +195,20 @@ func TestOneToKCrosstalkPrevention(t *testing.T) { previousSporkId := unittest.IdentifierFixture() // create and start node 1 on localhost and random port - node1, _ := p2ptest.NodeFixture(t, + node1, id1 := p2ptest.NodeFixture(t, previousSporkId, "test_one_to_k_crosstalk_prevention", + idProvider, ) p2ptest.StartNode(t, signalerCtx1, node1, 100*time.Millisecond) defer p2ptest.StopNode(t, node1, cancel1, 100*time.Millisecond) - + idProvider.SetIdentities(flow.IdentityList{&id1}) // create and start node 2 on localhost and random port with the same root block ID node2, id2 := p2ptest.NodeFixture(t, previousSporkId, "test_one_to_k_crosstalk_prevention", + idProvider, ) p2ptest.StartNode(t, signalerCtx2, node2, 100*time.Millisecond) @@ -259,7 +273,7 @@ func testOneToOneMessagingFails(t *testing.T, sourceNode p2p.LibP2PNode, peerInf // assert that stream creation failed assert.Error(t, err) // assert that it failed with the expected error - assert.Regexp(t, ".*failed to negotiate security protocol.*|.*protocol not supported.*", err) + assert.Regexp(t, ".*failed to negotiate security protocol.*|.*protocols not supported.*", err) } func testOneToKMessagingSucceeds(ctx context.Context, diff --git a/network/p2p/test/topic_validator_test.go b/network/p2p/test/topic_validator_test.go index 18229bd2e81..5a7e402b141 100644 --- a/network/p2p/test/topic_validator_test.go +++ b/network/p2p/test/topic_validator_test.go @@ -7,23 +7,26 @@ import ( "testing" "time" - "github.com/onflow/flow-go/network/p2p" - p2ptest "github.com/onflow/flow-go/network/p2p/test" - "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/network/p2p/utils" - - "github.com/onflow/flow-go/network/p2p/translator" + "github.com/stretchr/testify/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" + mockmodule "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/alsp" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" "github.com/onflow/flow-go/network/message" + "github.com/onflow/flow-go/network/mocknetwork" + "github.com/onflow/flow-go/network/p2p" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/network/p2p/translator" + "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/validator" flowpubsub "github.com/onflow/flow-go/network/validator/pubsub" @@ -34,15 +37,16 @@ import ( func TestTopicValidator_Unstaked(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := mockmodule.NewIdentityProvider(t) // create a hooked logger logger, hook := unittest.HookedLogger() sporkId := unittest.IdentifierFixture() - sn1, identity1 := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) - sn2, identity2 := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) - + sn1, identity1 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) + sn2, identity2 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) + idProvider.On("ByPeerID", sn1.Host().ID()).Return(&identity1, true).Maybe() + idProvider.On("ByPeerID", sn2.Host().ID()).Return(&identity2, true).Maybe() nodes := []p2p.LibP2PNode{sn1, sn2} p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -52,12 +56,12 @@ func TestTopicValidator_Unstaked(t *testing.T) { //NOTE: identity2 is not in the ids list simulating an un-staked node ids := flow.IdentityList{&identity1} - translator, err := translator.NewFixedTableIdentityTranslator(ids) + translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) // peer filter used by the topic validator to check if node is staked isStaked := func(pid peer.ID) error { - fid, err := translator.GetFlowID(pid) + fid, err := translatorFixture.GetFlowID(pid) if err != nil { return fmt.Errorf("could not translate the peer_id %s to a Flow identifier: %w", pid.String(), err) } @@ -108,13 +112,14 @@ func TestTopicValidator_Unstaked(t *testing.T) { func TestTopicValidator_PublicChannel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := mockmodule.NewIdentityProvider(t) sporkId := unittest.IdentifierFixture() logger := unittest.Logger() - sn1, _ := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) - sn2, identity2 := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) - + sn1, identity1 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) + sn2, identity2 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) + idProvider.On("ByPeerID", sn1.Host().ID()).Return(&identity1, true).Maybe() + idProvider.On("ByPeerID", sn2.Host().ID()).Return(&identity2, true).Maybe() nodes := []p2p.LibP2PNode{sn1, sn2} p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -166,15 +171,16 @@ func TestTopicValidator_PublicChannel(t *testing.T) { func TestTopicValidator_TopicMismatch(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := mockmodule.NewIdentityProvider(t) // create a hooked logger logger, hook := unittest.HookedLogger() sporkId := unittest.IdentifierFixture() - sn1, _ := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) - sn2, identity2 := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) - + sn1, identity1 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) + sn2, identity2 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) + idProvider.On("ByPeerID", sn1.Host().ID()).Return(&identity1, true).Maybe() + idProvider.On("ByPeerID", sn2.Host().ID()).Return(&identity2, true).Maybe() nodes := []p2p.LibP2PNode{sn1, sn2} p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -218,15 +224,16 @@ func TestTopicValidator_TopicMismatch(t *testing.T) { func TestTopicValidator_InvalidTopic(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := mockmodule.NewIdentityProvider(t) // create a hooked logger logger, hook := unittest.HookedLogger() sporkId := unittest.IdentifierFixture() - sn1, _ := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) - sn2, identity2 := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) - + sn1, identity1 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) + sn2, identity2 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), p2ptest.WithLogger(logger)) + idProvider.On("ByPeerID", sn1.Host().ID()).Return(&identity1, true).Maybe() + idProvider.On("ByPeerID", sn2.Host().ID()).Return(&identity2, true).Maybe() nodes := []p2p.LibP2PNode{sn1, sn2} p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -269,16 +276,17 @@ func TestTopicValidator_InvalidTopic(t *testing.T) { func TestAuthorizedSenderValidator_Unauthorized(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - - // create a hooked logger - logger, hook := unittest.HookedLogger() + idProvider := mockmodule.NewIdentityProvider(t) + logger := unittest.Logger() sporkId := unittest.IdentifierFixture() - sn1, identity1 := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleConsensus)) - sn2, identity2 := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleConsensus)) - an1, identity3 := p2ptest.NodeFixture(t, sporkId, t.Name(), p2ptest.WithRole(flow.RoleAccess)) - + sn1, identity1 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus)) + sn2, identity2 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus)) + an1, identity3 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithRole(flow.RoleAccess)) + idProvider.On("ByPeerID", sn1.Host().ID()).Return(&identity1, true).Maybe() + idProvider.On("ByPeerID", sn2.Host().ID()).Return(&identity2, true).Maybe() + idProvider.On("ByPeerID", an1.Host().ID()).Return(&identity3, true).Maybe() nodes := []p2p.LibP2PNode{sn1, sn2, an1} p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -288,12 +296,22 @@ func TestAuthorizedSenderValidator_Unauthorized(t *testing.T) { ids := flow.IdentityList{&identity1, &identity2, &identity3} - translator, err := translator.NewFixedTableIdentityTranslator(ids) + translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) + violation := &network.Violation{ + Identity: &identity3, + PeerID: an1.Host().ID().String(), + OriginID: identity3.NodeID, + MsgType: "*messages.BlockProposal", + Channel: channel, + Protocol: message.ProtocolTypePubSub, + Err: message.ErrUnauthorizedRole, + } + violationsConsumer := mocknetwork.NewViolationsConsumer(t) + violationsConsumer.On("OnUnAuthorizedSenderError", violation).Once().Return(nil) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { - fid, err := translator.GetFlowID(pid) + fid, err := translatorFixture.GetFlowID(pid) if err != nil { return &flow.Identity{}, false } @@ -369,24 +387,22 @@ func TestAuthorizedSenderValidator_Unauthorized(t *testing.T) { p2pfixtures.SubMustNeverReceiveAnyMessage(t, timedCtx, sub2) unittest.RequireReturnsBefore(t, wg.Wait, 5*time.Second, "could not receive message on time") - - // ensure the correct error is contained in the logged error - require.Contains(t, hook.Logs(), message.ErrUnauthorizedRole.Error()) } // TestAuthorizedSenderValidator_Authorized tests that the authorized sender validator rejects messages being sent on the wrong channel func TestAuthorizedSenderValidator_InvalidMsg(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := mockmodule.NewIdentityProvider(t) // create a hooked logger logger, hook := unittest.HookedLogger() sporkId := unittest.IdentifierFixture() - sn1, identity1 := p2ptest.NodeFixture(t, sporkId, "consensus_1", p2ptest.WithRole(flow.RoleConsensus)) - sn2, identity2 := p2ptest.NodeFixture(t, sporkId, "consensus_2", p2ptest.WithRole(flow.RoleConsensus)) - + sn1, identity1 := p2ptest.NodeFixture(t, sporkId, "consensus_1", idProvider, p2ptest.WithRole(flow.RoleConsensus)) + sn2, identity2 := p2ptest.NodeFixture(t, sporkId, "consensus_2", idProvider, p2ptest.WithRole(flow.RoleConsensus)) + idProvider.On("ByPeerID", sn1.Host().ID()).Return(&identity1, true).Maybe() + idProvider.On("ByPeerID", sn2.Host().ID()).Return(&identity2, true).Maybe() nodes := []p2p.LibP2PNode{sn1, sn2} p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -396,12 +412,16 @@ func TestAuthorizedSenderValidator_InvalidMsg(t *testing.T) { topic := channels.TopicFromChannel(channel, sporkId) ids := flow.IdentityList{&identity1, &identity2} - translator, err := translator.NewFixedTableIdentityTranslator(ids) + translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(identity2.NodeID, alsp.UnAuthorizedSender) + require.NoError(t, err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channel, expectedMisbehaviorReport).Once() + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { - fid, err := translator.GetFlowID(pid) + fid, err := translatorFixture.GetFlowID(pid) if err != nil { return &flow.Identity{}, false } @@ -449,16 +469,18 @@ func TestAuthorizedSenderValidator_InvalidMsg(t *testing.T) { func TestAuthorizedSenderValidator_Ejected(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := mockmodule.NewIdentityProvider(t) // create a hooked logger logger, hook := unittest.HookedLogger() sporkId := unittest.IdentifierFixture() - sn1, identity1 := p2ptest.NodeFixture(t, sporkId, "consensus_1", p2ptest.WithRole(flow.RoleConsensus)) - sn2, identity2 := p2ptest.NodeFixture(t, sporkId, "consensus_2", p2ptest.WithRole(flow.RoleConsensus)) - an1, identity3 := p2ptest.NodeFixture(t, sporkId, "access_1", p2ptest.WithRole(flow.RoleAccess)) - + sn1, identity1 := p2ptest.NodeFixture(t, sporkId, "consensus_1", idProvider, p2ptest.WithRole(flow.RoleConsensus)) + sn2, identity2 := p2ptest.NodeFixture(t, sporkId, "consensus_2", idProvider, p2ptest.WithRole(flow.RoleConsensus)) + an1, identity3 := p2ptest.NodeFixture(t, sporkId, "access_1", idProvider, p2ptest.WithRole(flow.RoleAccess)) + idProvider.On("ByPeerID", sn1.Host().ID()).Return(&identity1, true).Maybe() + idProvider.On("ByPeerID", sn2.Host().ID()).Return(&identity2, true).Maybe() + idProvider.On("ByPeerID", an1.Host().ID()).Return(&identity3, true).Maybe() nodes := []p2p.LibP2PNode{sn1, sn2, an1} p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -467,12 +489,16 @@ func TestAuthorizedSenderValidator_Ejected(t *testing.T) { topic := channels.TopicFromChannel(channel, sporkId) ids := flow.IdentityList{&identity1, &identity2, &identity3} - translator, err := translator.NewFixedTableIdentityTranslator(ids) + translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(identity2.NodeID, alsp.SenderEjected) + require.NoError(t, err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channel, expectedMisbehaviorReport).Once() + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { - fid, err := translator.GetFlowID(pid) + fid, err := translatorFixture.GetFlowID(pid) if err != nil { return &flow.Identity{}, false } @@ -544,13 +570,15 @@ func TestAuthorizedSenderValidator_Ejected(t *testing.T) { func TestAuthorizedSenderValidator_ClusterChannel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) - + idProvider := mockmodule.NewIdentityProvider(t) sporkId := unittest.IdentifierFixture() - ln1, identity1 := p2ptest.NodeFixture(t, sporkId, "collection_1", p2ptest.WithRole(flow.RoleCollection)) - ln2, identity2 := p2ptest.NodeFixture(t, sporkId, "collection_2", p2ptest.WithRole(flow.RoleCollection)) - ln3, identity3 := p2ptest.NodeFixture(t, sporkId, "collection_3", p2ptest.WithRole(flow.RoleCollection)) - + ln1, identity1 := p2ptest.NodeFixture(t, sporkId, "collection_1", idProvider, p2ptest.WithRole(flow.RoleCollection)) + ln2, identity2 := p2ptest.NodeFixture(t, sporkId, "collection_2", idProvider, p2ptest.WithRole(flow.RoleCollection)) + ln3, identity3 := p2ptest.NodeFixture(t, sporkId, "collection_3", idProvider, p2ptest.WithRole(flow.RoleCollection)) + idProvider.On("ByPeerID", ln1.Host().ID()).Return(&identity1, true).Maybe() + idProvider.On("ByPeerID", ln2.Host().ID()).Return(&identity2, true).Maybe() + idProvider.On("ByPeerID", ln3.Host().ID()).Return(&identity3, true).Maybe() nodes := []p2p.LibP2PNode{ln1, ln2, ln3} p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) @@ -559,13 +587,15 @@ func TestAuthorizedSenderValidator_ClusterChannel(t *testing.T) { topic := channels.TopicFromChannel(channel, sporkId) ids := flow.IdentityList{&identity1, &identity2, &identity3} - translator, err := translator.NewFixedTableIdentityTranslator(ids) + translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) logger := unittest.Logger() - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) + defer misbehaviorReportConsumer.AssertNotCalled(t, "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { - fid, err := translator.GetFlowID(pid) + fid, err := translatorFixture.GetFlowID(pid) if err != nil { return &flow.Identity{}, false } diff --git a/network/p2p/tracer/gossipSubMeshTracer.go b/network/p2p/tracer/gossipSubMeshTracer.go index 7cd4dd2b692..fea310a251d 100644 --- a/network/p2p/tracer/gossipSubMeshTracer.go +++ b/network/p2p/tracer/gossipSubMeshTracer.go @@ -13,7 +13,10 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/tracer/internal" "github.com/onflow/flow-go/utils/logging" ) @@ -23,6 +26,12 @@ const ( // MeshLogIntervalWarnMsg is the message logged by the tracer every logInterval if there are unknown peers in the mesh. MeshLogIntervalWarnMsg = "unknown peers in topic mesh peers of local node since last heartbeat" + + // defaultLastHighestIHaveRPCSizeResetInterval is the interval that we reset the tracker of max ihave size sent back + // to a default. We use ihave message max size to determine the health of requested iwants from remote peers. However, + // we don't desire an ihave size anomaly to persist forever, hence, we reset it back to a default every minute. + // The choice of the interval to be a minute is in harmony with the GossipSub decay interval. + defaultLastHighestIHaveRPCSizeResetInterval = time.Minute ) // The GossipSubMeshTracer component in the GossipSub pubsub.RawTracer that is designed to track the local @@ -43,23 +52,47 @@ type GossipSubMeshTracer struct { idProvider module.IdentityProvider loggerInterval time.Duration metrics module.GossipSubLocalMeshMetrics + rpcSentTracker *internal.RPCSentTracker } var _ p2p.PubSubTracer = (*GossipSubMeshTracer)(nil) -func NewGossipSubMeshTracer( - logger zerolog.Logger, - metrics module.GossipSubLocalMeshMetrics, - idProvider module.IdentityProvider, - loggerInterval time.Duration) *GossipSubMeshTracer { +type GossipSubMeshTracerConfig struct { + network.NetworkingType + metrics.HeroCacheMetricsFactory + Logger zerolog.Logger + Metrics module.GossipSubLocalMeshMetrics + IDProvider module.IdentityProvider + LoggerInterval time.Duration + RpcSentTrackerCacheSize uint32 + RpcSentTrackerWorkerQueueCacheSize uint32 + RpcSentTrackerNumOfWorkers int +} +// NewGossipSubMeshTracer creates a new *GossipSubMeshTracer. +// Args: +// - *GossipSubMeshTracerConfig: the mesh tracer config. +// Returns: +// - *GossipSubMeshTracer: new mesh tracer. +func NewGossipSubMeshTracer(config *GossipSubMeshTracerConfig) *GossipSubMeshTracer { + lg := config.Logger.With().Str("component", "gossipsub_topology_tracer").Logger() + rpcSentTracker := internal.NewRPCSentTracker(&internal.RPCSentTrackerConfig{ + Logger: lg, + RPCSentCacheSize: config.RpcSentTrackerCacheSize, + RPCSentCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(config.HeroCacheMetricsFactory, config.NetworkingType), + WorkerQueueCacheCollector: metrics.GossipSubRPCSentTrackerQueueMetricFactory(config.HeroCacheMetricsFactory, config.NetworkingType), + WorkerQueueCacheSize: config.RpcSentTrackerWorkerQueueCacheSize, + NumOfWorkers: config.RpcSentTrackerNumOfWorkers, + LastHighestIhavesSentResetInterval: defaultLastHighestIHaveRPCSizeResetInterval, + }) g := &GossipSubMeshTracer{ RawTracer: NewGossipSubNoopTracer(), topicMeshMap: make(map[string]map[peer.ID]struct{}), - idProvider: idProvider, - metrics: metrics, - logger: logger.With().Str("component", "gossip_sub_topology_tracer").Logger(), - loggerInterval: loggerInterval, + idProvider: config.IDProvider, + metrics: config.Metrics, + logger: lg, + loggerInterval: config.LoggerInterval, + rpcSentTracker: rpcSentTracker, } g.Component = component.NewComponentManagerBuilder(). @@ -67,6 +100,15 @@ func NewGossipSubMeshTracer( ready() g.logLoop(ctx) }). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + lg.Info().Msg("starting rpc sent tracker") + g.rpcSentTracker.Start(ctx) + lg.Info().Msg("rpc sent tracker started") + + <-g.rpcSentTracker.Done() + lg.Info().Msg("rpc sent tracker stopped") + }). Build() return g @@ -139,6 +181,15 @@ func (t *GossipSubMeshTracer) Prune(p peer.ID, topic string) { lg.Info().Hex("flow_id", logging.ID(id.NodeID)).Str("role", id.Role.String()).Msg("pruned peer") } +// SendRPC is called when a RPC is sent. Currently, the GossipSubMeshTracer tracks iHave RPC messages that have been sent. +// This function can be updated to track other control messages in the future as required. +func (t *GossipSubMeshTracer) SendRPC(rpc *pubsub.RPC, _ peer.ID) { + err := t.rpcSentTracker.Track(rpc) + if err != nil { + t.logger.Err(err).Bool(logging.KeyNetworkingSecurity, true).Msg("failed to track sent pubsbub rpc") + } +} + // logLoop logs the mesh peers of the local node for each topic at a regular interval. func (t *GossipSubMeshTracer) logLoop(ctx irrecoverable.SignalerContext) { ticker := time.NewTicker(t.loggerInterval) diff --git a/network/p2p/tracer/gossipSubMeshTracer_test.go b/network/p2p/tracer/gossipSubMeshTracer_test.go index 0659885f929..c8a680df53e 100644 --- a/network/p2p/tracer/gossipSubMeshTracer_test.go +++ b/network/p2p/tracer/gossipSubMeshTracer_test.go @@ -11,8 +11,10 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/atomic" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" @@ -29,6 +31,8 @@ import ( // One of the nodes is running with an unknown peer id, which the identity provider is mocked to return an error and // the mesh tracer should log a warning message. func TestGossipSubMeshTracer(t *testing.T) { + defaultConfig, err := config.DefaultConfig() + require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) sporkId := unittest.IdentifierFixture() @@ -61,11 +65,22 @@ func TestGossipSubMeshTracer(t *testing.T) { // we only need one node with a meshTracer to test the meshTracer. // meshTracer logs at 1 second intervals for sake of testing. collector := mockmodule.NewGossipSubLocalMeshMetrics(t) - meshTracer := tracer.NewGossipSubMeshTracer(logger, collector, idProvider, 1*time.Second) + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: logger, + Metrics: collector, + IDProvider: idProvider, + LoggerInterval: time.Second, + HeroCacheMetricsFactory: metrics.NewNoopHeroCacheMetricsFactory(), + RpcSentTrackerCacheSize: defaultConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + RpcSentTrackerWorkerQueueCacheSize: defaultConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, + RpcSentTrackerNumOfWorkers: defaultConfig.NetworkConfig.GossipSubConfig.RpcSentTrackerNumOfWorkers, + } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) tracerNode, tracerId := p2ptest.NodeFixture( t, sporkId, t.Name(), + idProvider, p2ptest.WithGossipSubTracer(meshTracer), p2ptest.WithRole(flow.RoleConsensus)) @@ -75,6 +90,7 @@ func TestGossipSubMeshTracer(t *testing.T) { t, sporkId, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleConsensus)) idProvider.On("ByPeerID", otherNode1.Host().ID()).Return(&otherId1, true).Maybe() @@ -82,6 +98,7 @@ func TestGossipSubMeshTracer(t *testing.T) { t, sporkId, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleConsensus)) idProvider.On("ByPeerID", otherNode2.Host().ID()).Return(&otherId2, true).Maybe() @@ -90,6 +107,7 @@ func TestGossipSubMeshTracer(t *testing.T) { t, sporkId, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleConsensus)) idProvider.On("ByPeerID", unknownNode.Host().ID()).Return(nil, false).Maybe() diff --git a/network/p2p/tracer/gossipSubScoreTracer.go b/network/p2p/tracer/gossipSubScoreTracer.go index aae023099d7..facdc8bd182 100644 --- a/network/p2p/tracer/gossipSubScoreTracer.go +++ b/network/p2p/tracer/gossipSubScoreTracer.go @@ -224,7 +224,7 @@ func (g *GossipSubScoreTracer) logPeerScore(peerID peer.ID) bool { Str("role", identity.Role.String()).Logger() } - lg = g.logger.With(). + lg = lg.With(). Str("peer_id", peerID.String()). Float64("overall_score", snapshot.Score). Float64("app_specific_score", snapshot.AppSpecificScore). diff --git a/network/p2p/tracer/gossipSubScoreTracer_test.go b/network/p2p/tracer/gossipSubScoreTracer_test.go index 233e3604b6d..2a3ea623eb0 100644 --- a/network/p2p/tracer/gossipSubScoreTracer_test.go +++ b/network/p2p/tracer/gossipSubScoreTracer_test.go @@ -76,15 +76,14 @@ func TestGossipSubScoreTracer(t *testing.T) { t, sporkId, t.Name(), + idProvider, p2ptest.WithMetricsCollector(&mockPeerScoreMetrics{ NoopCollector: metrics.NoopCollector{}, c: scoreMetrics, }), p2ptest.WithLogger(logger), p2ptest.WithPeerScoreTracerInterval(1*time.Second), // set the peer score log interval to 1 second for sake of testing. - p2ptest.WithPeerScoringEnabled(idProvider), // enable peer scoring for sake of testing. - // 4. Sets some fixed scores for the nodes for the sake of testing based on their roles. - p2ptest.WithPeerScoreParamsOption(&p2p.PeerScoringConfig{ + p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ AppSpecificScoreParams: func(pid peer.ID) float64 { id, ok := idProvider.ByPeerID(pid) require.True(t, ok) @@ -130,6 +129,7 @@ func TestGossipSubScoreTracer(t *testing.T) { t, sporkId, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleConsensus)) idProvider.On("ByPeerID", consensusNode.Host().ID()).Return(&consensusId, true).Maybe() @@ -137,6 +137,7 @@ func TestGossipSubScoreTracer(t *testing.T) { t, sporkId, t.Name(), + idProvider, p2ptest.WithRole(flow.RoleAccess)) idProvider.On("ByPeerID", accessNode.Host().ID()).Return(&accessId, true).Maybe() @@ -187,9 +188,7 @@ func TestGossipSubScoreTracer(t *testing.T) { // IP score, and an existing mesh score. assert.Eventually(t, func() bool { // we expect the tracerNode to have the consensusNodes and accessNodes with the correct app scores. - exposer, ok := tracerNode.PeerScoreExposer() - require.True(t, ok) - + exposer := tracerNode.PeerScoreExposer() score, ok := exposer.GetAppScore(consensusNode.Host().ID()) if !ok || score != consensusScore { return false diff --git a/network/p2p/tracer/internal/cache.go b/network/p2p/tracer/internal/cache.go new file mode 100644 index 00000000000..655ddf2179f --- /dev/null +++ b/network/p2p/tracer/internal/cache.go @@ -0,0 +1,83 @@ +package internal + +import ( + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" + "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" + "github.com/onflow/flow-go/module/mempool/stdmap" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" +) + +// rpcCtrlMsgSentCacheConfig configuration for the rpc sent cache. +type rpcCtrlMsgSentCacheConfig struct { + logger zerolog.Logger + sizeLimit uint32 + collector module.HeroCacheMetrics +} + +// rpcSentCache cache that stores rpcSentEntity. These entity's represent RPC control messages sent from the local node. +type rpcSentCache struct { + // c is the underlying cache. + c *stdmap.Backend +} + +// newRPCSentCache creates a new *rpcSentCache. +// Args: +// - config: record cache config. +// Returns: +// - *rpcSentCache: the created cache. +// Note that this cache is intended to track control messages sent by the local node, +// it stores a RPCSendEntity using an Id which should uniquely identifies the message being tracked. +func newRPCSentCache(config *rpcCtrlMsgSentCacheConfig) *rpcSentCache { + backData := herocache.NewCache(config.sizeLimit, + herocache.DefaultOversizeFactor, + heropool.LRUEjection, + config.logger.With().Str("mempool", "gossipsub-rpc-control-messages-sent").Logger(), + config.collector) + return &rpcSentCache{ + c: stdmap.NewBackend(stdmap.WithBackData(backData)), + } +} + +// add initializes the record cached for the given messageEntityID if it does not exist. +// Returns true if the record is initialized, false otherwise (i.e.: the record already exists). +// Args: +// - messageId: the message ID. +// - controlMsgType: the rpc control message type. +// Returns: +// - bool: true if the record is initialized, false otherwise (i.e.: the record already exists). +// Note that if add is called multiple times for the same messageEntityID, the record is initialized only once, and the +// subsequent calls return false and do not change the record (i.e.: the record is not re-initialized). +func (r *rpcSentCache) add(messageId string, controlMsgType p2pmsg.ControlMessageType) bool { + return r.c.Add(newRPCSentEntity(r.rpcSentEntityID(messageId, controlMsgType), controlMsgType)) +} + +// has checks if the RPC message has been cached indicating it has been sent. +// Args: +// - messageId: the message ID. +// - controlMsgType: the rpc control message type. +// Returns: +// - bool: true if the RPC has been cache indicating it was sent from the local node. +func (r *rpcSentCache) has(messageId string, controlMsgType p2pmsg.ControlMessageType) bool { + return r.c.Has(r.rpcSentEntityID(messageId, controlMsgType)) +} + +// size returns the number of records in the cache. +func (r *rpcSentCache) size() uint { + return r.c.Size() +} + +// rpcSentEntityID creates an entity ID from the messageID and control message type. +// Args: +// - messageId: the message ID. +// - controlMsgType: the rpc control message type. +// Returns: +// - flow.Identifier: the entity ID. +func (r *rpcSentCache) rpcSentEntityID(messageId string, controlMsgType p2pmsg.ControlMessageType) flow.Identifier { + return flow.MakeIDFromFingerPrint([]byte(fmt.Sprintf("%s%s", messageId, controlMsgType))) +} diff --git a/network/p2p/tracer/internal/cache_test.go b/network/p2p/tracer/internal/cache_test.go new file mode 100644 index 00000000000..10872b7b7ef --- /dev/null +++ b/network/p2p/tracer/internal/cache_test.go @@ -0,0 +1,118 @@ +package internal + +import ( + "sync" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestCache_Add tests the add method of the rpcSentCache. +// It ensures that the method returns true when a new record is initialized +// and false when an existing record is initialized. +func TestCache_Add(t *testing.T) { + cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) + controlMsgType := p2pmsg.CtrlMsgIHave + messageID1 := unittest.IdentifierFixture().String() + messageID2 := unittest.IdentifierFixture().String() + + // test initializing a record for an ID that doesn't exist in the cache + initialized := cache.add(messageID1, controlMsgType) + require.True(t, initialized, "expected record to be initialized") + require.True(t, cache.has(messageID1, controlMsgType), "expected record to exist") + + // test initializing a record for an ID that already exists in the cache + initialized = cache.add(messageID1, controlMsgType) + require.False(t, initialized, "expected record not to be initialized") + require.True(t, cache.has(messageID1, controlMsgType), "expected record to exist") + + // test initializing a record for another ID + initialized = cache.add(messageID2, controlMsgType) + require.True(t, initialized, "expected record to be initialized") + require.True(t, cache.has(messageID2, controlMsgType), "expected record to exist") +} + +// TestCache_ConcurrentInit tests the concurrent initialization of records. +// The test covers the following scenarios: +// 1. Multiple goroutines initializing records for different ids. +// 2. Ensuring that all records are correctly initialized. +func TestCache_ConcurrentAdd(t *testing.T) { + cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) + controlMsgType := p2pmsg.CtrlMsgIHave + messageIds := unittest.IdentifierListFixture(10) + + var wg sync.WaitGroup + wg.Add(len(messageIds)) + + for _, id := range messageIds { + go func(id flow.Identifier) { + defer wg.Done() + cache.add(id.String(), controlMsgType) + }(id) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that all records are correctly initialized + for _, id := range messageIds { + require.True(t, cache.has(id.String(), controlMsgType)) + } +} + +// TestCache_ConcurrentSameRecordInit tests the concurrent initialization of the same record. +// The test covers the following scenarios: +// 1. Multiple goroutines attempting to initialize the same record concurrently. +// 2. Only one goroutine successfully initializes the record, and others receive false on initialization. +// 3. The record is correctly initialized in the cache and can be retrieved using the Get method. +func TestCache_ConcurrentSameRecordAdd(t *testing.T) { + cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) + controlMsgType := p2pmsg.CtrlMsgIHave + messageID := unittest.IdentifierFixture().String() + const concurrentAttempts = 10 + + var wg sync.WaitGroup + wg.Add(concurrentAttempts) + + successGauge := atomic.Int32{} + + for i := 0; i < concurrentAttempts; i++ { + go func() { + defer wg.Done() + initSuccess := cache.add(messageID, controlMsgType) + if initSuccess { + successGauge.Inc() + } + }() + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that only one goroutine successfully initialized the record + require.Equal(t, int32(1), successGauge.Load()) + + // ensure that the record is correctly initialized in the cache + require.True(t, cache.has(messageID, controlMsgType)) +} + +// cacheFixture returns a new *RecordCache. +func cacheFixture(t *testing.T, sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *rpcSentCache { + config := &rpcCtrlMsgSentCacheConfig{ + sizeLimit: sizeLimit, + logger: logger, + collector: collector, + } + r := newRPCSentCache(config) + // expect cache to be empty + require.Equalf(t, uint(0), r.size(), "cache size must be 0") + require.NotNil(t, r) + return r +} diff --git a/network/p2p/tracer/internal/rpc_send_entity.go b/network/p2p/tracer/internal/rpc_send_entity.go new file mode 100644 index 00000000000..b3f56ce0b55 --- /dev/null +++ b/network/p2p/tracer/internal/rpc_send_entity.go @@ -0,0 +1,37 @@ +package internal + +import ( + "github.com/onflow/flow-go/model/flow" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" +) + +// rpcSentEntity struct representing an RPC control message sent from local node. +// This struct implements the flow.Entity interface and uses messageID field deduplication. +type rpcSentEntity struct { + // messageID the messageID of the rpc control message. + messageID flow.Identifier + // controlMsgType the control message type. + controlMsgType p2pmsg.ControlMessageType +} + +var _ flow.Entity = (*rpcSentEntity)(nil) + +// ID returns the node ID of the sender, which is used as the unique identifier of the entity for maintenance and +// deduplication purposes in the cache. +func (r rpcSentEntity) ID() flow.Identifier { + return r.messageID +} + +// Checksum returns the node ID of the sender, it does not have any purpose in the cache. +// It is implemented to satisfy the flow.Entity interface. +func (r rpcSentEntity) Checksum() flow.Identifier { + return r.messageID +} + +// newRPCSentEntity returns a new rpcSentEntity. +func newRPCSentEntity(id flow.Identifier, controlMessageType p2pmsg.ControlMessageType) rpcSentEntity { + return rpcSentEntity{ + messageID: id, + controlMsgType: controlMessageType, + } +} diff --git a/network/p2p/tracer/internal/rpc_sent_tracker.go b/network/p2p/tracer/internal/rpc_sent_tracker.go new file mode 100644 index 00000000000..5c2c842a48c --- /dev/null +++ b/network/p2p/tracer/internal/rpc_sent_tracker.go @@ -0,0 +1,171 @@ +package internal + +import ( + "crypto/rand" + "fmt" + "sync" + "time" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine/common/worker" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/mempool/queue" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" +) + +const ( + iHaveRPCTrackedLog = "ihave rpc tracked successfully" +) + +// trackableRPC is an internal data structure for "temporarily" storing *pubsub.RPC sent in the queue before they are processed +// by the *RPCSentTracker. +type trackableRPC struct { + // Nonce prevents deduplication in the hero store + Nonce []byte + rpc *pubsub.RPC +} + +// lastHighestIHaveRPCSize tracks the last highest rpc control message size the time stamp it was last updated. +type lastHighestIHaveRPCSize struct { + sync.RWMutex + lastSize int64 + lastUpdate time.Time +} + +// RPCSentTracker tracks RPC messages and the size of the last largest iHave rpc control message sent. +type RPCSentTracker struct { + component.Component + *lastHighestIHaveRPCSize + logger zerolog.Logger + cache *rpcSentCache + workerPool *worker.Pool[trackableRPC] + lastHighestIHaveRPCSizeResetInterval time.Duration +} + +// RPCSentTrackerConfig configuration for the RPCSentTracker. +type RPCSentTrackerConfig struct { + Logger zerolog.Logger + //RPCSentCacheSize size of the *rpcSentCache cache. + RPCSentCacheSize uint32 + // RPCSentCacheCollector metrics collector for the *rpcSentCache cache. + RPCSentCacheCollector module.HeroCacheMetrics + // WorkerQueueCacheCollector metrics factory for the worker pool. + WorkerQueueCacheCollector module.HeroCacheMetrics + // WorkerQueueCacheSize the worker pool herostore cache size. + WorkerQueueCacheSize uint32 + // NumOfWorkers number of workers in the worker pool. + NumOfWorkers int + // LastHighestIhavesSentResetInterval the refresh interval to reset the lastHighestIHaveRPCSize. + LastHighestIhavesSentResetInterval time.Duration +} + +// NewRPCSentTracker returns a new *NewRPCSentTracker. +func NewRPCSentTracker(config *RPCSentTrackerConfig) *RPCSentTracker { + cacheConfig := &rpcCtrlMsgSentCacheConfig{ + sizeLimit: config.RPCSentCacheSize, + logger: config.Logger, + collector: config.RPCSentCacheCollector, + } + + store := queue.NewHeroStore( + config.WorkerQueueCacheSize, + config.Logger, + config.WorkerQueueCacheCollector) + + tracker := &RPCSentTracker{ + logger: config.Logger.With().Str("component", "rpc_sent_tracker").Logger(), + lastHighestIHaveRPCSize: &lastHighestIHaveRPCSize{sync.RWMutex{}, 0, time.Now()}, + cache: newRPCSentCache(cacheConfig), + lastHighestIHaveRPCSizeResetInterval: config.LastHighestIhavesSentResetInterval, + } + tracker.workerPool = worker.NewWorkerPoolBuilder[trackableRPC]( + config.Logger, + store, + tracker.rpcSentWorkerLogic).Build() + + builder := component.NewComponentManagerBuilder() + for i := 0; i < config.NumOfWorkers; i++ { + builder.AddWorker(tracker.workerPool.WorkerLogic()) + } + tracker.Component = builder.Build() + return tracker +} + +// Track submits the control message to the worker queue for async tracking. +// Args: +// - *pubsub.RPC: the rpc sent. +// All errors returned from this function can be considered benign. +func (t *RPCSentTracker) Track(rpc *pubsub.RPC) error { + n, err := nonce() + if err != nil { + return fmt.Errorf("failed to get track rpc work nonce: %w", err) + } + if ok := t.workerPool.Submit(trackableRPC{Nonce: n, rpc: rpc}); !ok { + return fmt.Errorf("failed to track RPC could not submit work to worker pool") + } + return nil +} + +// rpcSentWorkerLogic tracks control messages sent in *pubsub.RPC. +func (t *RPCSentTracker) rpcSentWorkerLogic(work trackableRPC) error { + switch { + case len(work.rpc.GetControl().GetIhave()) > 0: + iHave := work.rpc.GetControl().GetIhave() + t.iHaveRPCSent(iHave) + t.updateLastHighestIHaveRPCSize(int64(len(iHave))) + t.logger.Info().Int("size", len(iHave)).Msg(iHaveRPCTrackedLog) + } + return nil +} + +func (t *RPCSentTracker) updateLastHighestIHaveRPCSize(size int64) { + t.Lock() + defer t.Unlock() + if t.lastSize < size || time.Since(t.lastUpdate) > t.lastHighestIHaveRPCSizeResetInterval { + // The last highest ihave RPC size is updated if the new size is larger than the current size, or if the time elapsed since the last update surpasses the reset interval. + t.lastSize = size + t.lastUpdate = time.Now() + } +} + +// iHaveRPCSent caches a unique entity message ID for each message ID included in each rpc iHave control message. +// Args: +// - []*pb.ControlIHave: list of iHave control messages. +func (t *RPCSentTracker) iHaveRPCSent(iHaves []*pb.ControlIHave) { + controlMsgType := p2pmsg.CtrlMsgIHave + for _, iHave := range iHaves { + for _, messageID := range iHave.GetMessageIDs() { + t.cache.add(messageID, controlMsgType) + } + } +} + +// WasIHaveRPCSent checks if an iHave control message with the provided message ID was sent. +// Args: +// - messageID: the message ID of the iHave RPC. +// Returns: +// - bool: true if the iHave rpc with the provided message ID was sent. +func (t *RPCSentTracker) WasIHaveRPCSent(messageID string) bool { + return t.cache.has(messageID, p2pmsg.CtrlMsgIHave) +} + +// LastHighestIHaveRPCSize returns the last highest size of iHaves sent in an rpc. +func (t *RPCSentTracker) LastHighestIHaveRPCSize() int64 { + t.RLock() + defer t.RUnlock() + return t.lastSize +} + +// nonce returns random string that is used to store unique items in herocache. +func nonce() ([]byte, error) { + b := make([]byte, 16) + _, err := rand.Read(b) + if err != nil { + return nil, err + } + return b, nil +} diff --git a/network/p2p/tracer/internal/rpc_sent_tracker_test.go b/network/p2p/tracer/internal/rpc_sent_tracker_test.go new file mode 100644 index 00000000000..9b571e3d5a6 --- /dev/null +++ b/network/p2p/tracer/internal/rpc_sent_tracker_test.go @@ -0,0 +1,259 @@ +package internal + +import ( + "context" + "os" + "testing" + "time" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/config" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestNewRPCSentTracker ensures *RPCSenTracker is created as expected. +func TestNewRPCSentTracker(t *testing.T) { + tracker := mockTracker(t, time.Minute) + require.NotNil(t, tracker) +} + +// TestRPCSentTracker_IHave ensures *RPCSentTracker tracks sent iHave control messages as expected. +func TestRPCSentTracker_IHave(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + tracker := mockTracker(t, time.Minute) + require.NotNil(t, tracker) + + tracker.Start(signalerCtx) + defer func() { + cancel() + unittest.RequireComponentsDoneBefore(t, time.Second, tracker) + }() + + t.Run("WasIHaveRPCSent should return false for iHave message Id that has not been tracked", func(t *testing.T) { + require.False(t, tracker.WasIHaveRPCSent("message_id")) + }) + + t.Run("WasIHaveRPCSent should return true for iHave message after it is tracked with iHaveRPCSent", func(t *testing.T) { + numOfMsgIds := 100 + testCases := []struct { + messageIDS []string + }{ + {unittest.IdentifierListFixture(numOfMsgIds).Strings()}, + {unittest.IdentifierListFixture(numOfMsgIds).Strings()}, + {unittest.IdentifierListFixture(numOfMsgIds).Strings()}, + {unittest.IdentifierListFixture(numOfMsgIds).Strings()}, + } + iHaves := make([]*pb.ControlIHave, len(testCases)) + for i, testCase := range testCases { + testCase := testCase + iHaves[i] = &pb.ControlIHave{ + MessageIDs: testCase.messageIDS, + } + } + rpc := rpcFixture(withIhaves(iHaves)) + require.NoError(t, tracker.Track(rpc)) + + // eventually we should have tracked numOfMsgIds per single topic + require.Eventually(t, func() bool { + return tracker.cache.size() == uint(len(testCases)*numOfMsgIds) + }, time.Second, 100*time.Millisecond) + + for _, testCase := range testCases { + for _, messageID := range testCase.messageIDS { + require.True(t, tracker.WasIHaveRPCSent(messageID)) + } + } + }) + +} + +// TestRPCSentTracker_DuplicateMessageID ensures the worker pool of the RPC tracker processes req with the same message ID but different nonce. +func TestRPCSentTracker_DuplicateMessageID(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + processedWorkLogs := atomic.NewInt64(0) + hook := zerolog.HookFunc(func(e *zerolog.Event, level zerolog.Level, message string) { + if level == zerolog.InfoLevel { + if message == iHaveRPCTrackedLog { + processedWorkLogs.Inc() + } + } + }) + logger := zerolog.New(os.Stdout).Level(zerolog.InfoLevel).Hook(hook) + + tracker := mockTracker(t, time.Minute) + require.NotNil(t, tracker) + tracker.logger = logger + tracker.Start(signalerCtx) + defer func() { + cancel() + unittest.RequireComponentsDoneBefore(t, time.Second, tracker) + }() + + messageID := unittest.IdentifierFixture().String() + rpc := rpcFixture(withIhaves([]*pb.ControlIHave{{ + MessageIDs: []string{messageID}, + }})) + // track duplicate RPC's each will be processed by a worker + require.NoError(t, tracker.Track(rpc)) + require.NoError(t, tracker.Track(rpc)) + + // eventually we should have processed both RPCs + require.Eventually(t, func() bool { + return processedWorkLogs.Load() == 2 + }, time.Second, 100*time.Millisecond) +} + +// TestRPCSentTracker_ConcurrentTracking ensures that all message IDs in RPC's are tracked as expected when tracked concurrently. +func TestRPCSentTracker_ConcurrentTracking(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + tracker := mockTracker(t, time.Minute) + require.NotNil(t, tracker) + + tracker.Start(signalerCtx) + defer func() { + cancel() + unittest.RequireComponentsDoneBefore(t, time.Second, tracker) + }() + + numOfMsgIds := 100 + numOfRPCs := 100 + rpcs := make([]*pubsub.RPC, numOfRPCs) + for i := 0; i < numOfRPCs; i++ { + i := i + go func() { + rpc := rpcFixture(withIhaves([]*pb.ControlIHave{{MessageIDs: unittest.IdentifierListFixture(numOfMsgIds).Strings()}})) + require.NoError(t, tracker.Track(rpc)) + rpcs[i] = rpc + }() + } + + // eventually we should have tracked numOfMsgIds per single topic + require.Eventually(t, func() bool { + return tracker.cache.size() == uint(numOfRPCs*numOfMsgIds) + }, time.Second, 100*time.Millisecond) + + for _, rpc := range rpcs { + ihaves := rpc.GetControl().GetIhave() + for _, messageID := range ihaves[0].GetMessageIDs() { + require.True(t, tracker.WasIHaveRPCSent(messageID)) + } + } +} + +// TestRPCSentTracker_IHave ensures *RPCSentTracker tracks the last largest iHave size as expected. +func TestRPCSentTracker_LastHighestIHaveRPCSize(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + tracker := mockTracker(t, 3*time.Second) + require.NotNil(t, tracker) + + tracker.Start(signalerCtx) + defer func() { + cancel() + unittest.RequireComponentsDoneBefore(t, time.Second, tracker) + }() + + expectedLastHighestSize := 1000 + // adding a single message ID to the iHave enables us to track the expected cache size by the amount of iHaves. + numOfMessageIds := 1 + testCases := []struct { + rpcFixture *pubsub.RPC + numOfIhaves int + }{ + {rpcFixture(withIhaves(mockIHaveFixture(10, numOfMessageIds))), 10}, + {rpcFixture(withIhaves(mockIHaveFixture(100, numOfMessageIds))), 100}, + {rpcFixture(withIhaves(mockIHaveFixture(expectedLastHighestSize, numOfMessageIds))), expectedLastHighestSize}, + {rpcFixture(withIhaves(mockIHaveFixture(999, numOfMessageIds))), 999}, + {rpcFixture(withIhaves(mockIHaveFixture(23, numOfMessageIds))), 23}, + } + + expectedCacheSize := 0 + for _, testCase := range testCases { + require.NoError(t, tracker.Track(testCase.rpcFixture)) + expectedCacheSize += testCase.numOfIhaves + } + + // eventually we should have tracked numOfMsgIds per single topic + require.Eventually(t, func() bool { + return tracker.cache.size() == uint(expectedCacheSize) + }, time.Second, 100*time.Millisecond) + + require.Equal(t, int64(expectedLastHighestSize), tracker.LastHighestIHaveRPCSize()) + + // after setting sending large RPC lastHighestIHaveRPCSize should reset to 0 after lastHighestIHaveRPCSize reset loop tick + largeIhave := 50000 + require.NoError(t, tracker.Track(rpcFixture(withIhaves(mockIHaveFixture(largeIhave, numOfMessageIds))))) + require.Eventually(t, func() bool { + return tracker.LastHighestIHaveRPCSize() == int64(largeIhave) + }, 1*time.Second, 100*time.Millisecond) + + // we expect lastHighestIHaveRPCSize to be set to the current rpc size being tracked if it hasn't been updated since the configured lastHighestIHaveRPCSizeResetInterval + expectedEventualLastHighest := 8 + require.Eventually(t, func() bool { + require.NoError(t, tracker.Track(rpcFixture(withIhaves(mockIHaveFixture(expectedEventualLastHighest, numOfMessageIds))))) + return tracker.LastHighestIHaveRPCSize() == int64(expectedEventualLastHighest) + }, 4*time.Second, 100*time.Millisecond) +} + +// mockIHaveFixture generate list of iHaves of size n. Each iHave will be created with m number of random message ids. +func mockIHaveFixture(n, m int) []*pb.ControlIHave { + iHaves := make([]*pb.ControlIHave, n) + for i := 0; i < n; i++ { + // topic does not have to be a valid flow topic, for teting purposes we can use a random string + topic := unittest.IdentifierFixture().String() + iHaves[i] = &pb.ControlIHave{ + TopicID: &topic, + MessageIDs: unittest.IdentifierListFixture(m).Strings(), + } + } + return iHaves +} + +func mockTracker(t *testing.T, lastHighestIhavesSentResetInterval time.Duration) *RPCSentTracker { + cfg, err := config.DefaultConfig() + require.NoError(t, err) + tracker := NewRPCSentTracker(&RPCSentTrackerConfig{ + Logger: zerolog.Nop(), + RPCSentCacheSize: cfg.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + RPCSentCacheCollector: metrics.NewNoopCollector(), + WorkerQueueCacheCollector: metrics.NewNoopCollector(), + WorkerQueueCacheSize: cfg.NetworkConfig.GossipSubConfig.RPCSentTrackerQueueCacheSize, + NumOfWorkers: 1, + LastHighestIhavesSentResetInterval: lastHighestIhavesSentResetInterval, + }) + return tracker +} + +type rpcFixtureOpt func(*pubsub.RPC) + +func withIhaves(iHave []*pb.ControlIHave) rpcFixtureOpt { + return func(rpc *pubsub.RPC) { + rpc.Control.Ihave = iHave + } +} + +func rpcFixture(opts ...rpcFixtureOpt) *pubsub.RPC { + rpc := &pubsub.RPC{ + RPC: pb.RPC{ + Control: &pb.ControlMessage{}, + }, + } + for _, opt := range opts { + opt(rpc) + } + return rpc +} diff --git a/network/p2p/unicast/manager.go b/network/p2p/unicast/manager.go index f45c2ce7bcd..d8af81ed49d 100644 --- a/network/p2p/unicast/manager.go +++ b/network/p2p/unicast/manager.go @@ -69,12 +69,12 @@ func NewUnicastManager(logger zerolog.Logger, // as the core handler for other unicast protocols, e.g., compressions. func (m *Manager) WithDefaultHandler(defaultHandler libp2pnet.StreamHandler) { defaultProtocolID := protocols.FlowProtocolID(m.sporkId) - m.defaultHandler = defaultHandler - if len(m.protocols) > 0 { panic("default handler must be set only once before any unicast registration") } + m.defaultHandler = defaultHandler + m.protocols = []protocols.Protocol{ &PlainStream{ protocolId: defaultProtocolID, diff --git a/network/p2p/unicast/ratelimit/bandwidth_rate_limiter_test.go b/network/p2p/unicast/ratelimit/bandwidth_rate_limiter_test.go index 16df3b62f78..0016ae49f63 100644 --- a/network/p2p/unicast/ratelimit/bandwidth_rate_limiter_test.go +++ b/network/p2p/unicast/ratelimit/bandwidth_rate_limiter_test.go @@ -73,7 +73,7 @@ func TestBandWidthRateLimiter_IsRateLimited(t *testing.T) { burst := 1000 // setup bandwidth rate limiter - bandwidthRateLimiter := NewBandWidthRateLimiter(limit, burst, 1) + bandwidthRateLimiter := NewBandWidthRateLimiter(limit, burst, time.Second) // for the duration of a simulated second we will send 3 messages. Each message is about // 400 bytes, the 3rd message will put our limiter over the 1000 byte limit at 1200 bytes. Thus diff --git a/network/p2p/utils/logger.go b/network/p2p/utils/logger.go new file mode 100644 index 00000000000..b535d567ccd --- /dev/null +++ b/network/p2p/utils/logger.go @@ -0,0 +1,33 @@ +package utils + +import ( + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/rs/zerolog" +) + +// TopicScoreParamsLogger is a helper function that returns a logger with the topic score params added as fields. +// Args: +// logger: zerolog.Logger - logger to add fields to +// topicName: string - name of the topic +// params: pubsub.TopicScoreParams - topic score params +func TopicScoreParamsLogger(logger zerolog.Logger, topicName string, topicParams *pubsub.TopicScoreParams) zerolog.Logger { + return logger.With().Str("topic", topicName). + Bool("atomic_validation", topicParams.SkipAtomicValidation). + Float64("topic_weight", topicParams.TopicWeight). + Float64("time_in_mesh_weight", topicParams.TimeInMeshWeight). + Dur("time_in_mesh_quantum", topicParams.TimeInMeshQuantum). + Float64("time_in_mesh_cap", topicParams.TimeInMeshCap). + Float64("first_message_deliveries_weight", topicParams.FirstMessageDeliveriesWeight). + Float64("first_message_deliveries_decay", topicParams.FirstMessageDeliveriesDecay). + Float64("first_message_deliveries_cap", topicParams.FirstMessageDeliveriesCap). + Float64("mesh_message_deliveries_weight", topicParams.MeshMessageDeliveriesWeight). + Float64("mesh_message_deliveries_decay", topicParams.MeshMessageDeliveriesDecay). + Float64("mesh_message_deliveries_cap", topicParams.MeshMessageDeliveriesCap). + Float64("mesh_message_deliveries_threshold", topicParams.MeshMessageDeliveriesThreshold). + Dur("mesh_message_deliveries_window", topicParams.MeshMessageDeliveriesWindow). + Dur("mesh_message_deliveries_activation", topicParams.MeshMessageDeliveriesActivation). + Float64("mesh_failure_penalty_weight", topicParams.MeshFailurePenaltyWeight). + Float64("mesh_failure_penalty_decay", topicParams.MeshFailurePenaltyDecay). + Float64("invalid_message_deliveries_weight", topicParams.InvalidMessageDeliveriesWeight). + Float64("invalid_message_deliveries_decay", topicParams.InvalidMessageDeliveriesDecay).Logger() +} diff --git a/network/p2p/utils/ratelimiter/rate_limiter.go b/network/p2p/utils/ratelimiter/rate_limiter.go index 46ddc456db4..fa29ef0d5b4 100644 --- a/network/p2p/utils/ratelimiter/rate_limiter.go +++ b/network/p2p/utils/ratelimiter/rate_limiter.go @@ -39,7 +39,7 @@ func NewRateLimiter(limit rate.Limit, burst int, lockoutDuration time.Duration, limiterMap: internal.NewLimiterMap(rateLimiterTTL, cleanUpTickInterval), limit: limit, burst: burst, - rateLimitLockoutDuration: lockoutDuration * time.Second, + rateLimitLockoutDuration: lockoutDuration, } for _, opt := range opts { diff --git a/network/p2p/utils/ratelimiter/rate_limiter_test.go b/network/p2p/utils/ratelimiter/rate_limiter_test.go index 6b45857ae52..8864011263d 100644 --- a/network/p2p/utils/ratelimiter/rate_limiter_test.go +++ b/network/p2p/utils/ratelimiter/rate_limiter_test.go @@ -23,7 +23,7 @@ func TestRateLimiter_Allow(t *testing.T) { require.NoError(t, err) // setup rate limiter - rateLimiter := NewRateLimiter(limit, burst, 1) + rateLimiter := NewRateLimiter(limit, burst, time.Second) require.True(t, rateLimiter.Allow(peerID, 0)) @@ -49,7 +49,7 @@ func TestRateLimiter_IsRateLimited(t *testing.T) { require.NoError(t, err) // setup rate limiter - rateLimiter := NewRateLimiter(limit, burst, 1) + rateLimiter := NewRateLimiter(limit, burst, time.Second) require.False(t, rateLimiter.IsRateLimited(peerID)) require.True(t, rateLimiter.Allow(peerID, 0)) diff --git a/network/proxy/conduit.go b/network/proxy/conduit.go index 4e9d2478380..377087dc005 100644 --- a/network/proxy/conduit.go +++ b/network/proxy/conduit.go @@ -12,6 +12,8 @@ type ProxyConduit struct { targetNodeID flow.Identifier } +var _ network.Conduit = (*ProxyConduit)(nil) + func (c *ProxyConduit) Publish(event interface{}, targetIDs ...flow.Identifier) error { return c.Conduit.Publish(event, c.targetNodeID) } diff --git a/network/queue/messageQueue_test.go b/network/queue/messageQueue_test.go index 159ce7506cb..5fd7cf86839 100644 --- a/network/queue/messageQueue_test.go +++ b/network/queue/messageQueue_test.go @@ -217,7 +217,7 @@ func createMessages(messageCnt int, priorityFunc queue.MessagePriorityFunc) map[ } func randomPriority(_ interface{}) (queue.Priority, error) { - rand.Seed(time.Now().UnixNano()) + p := rand.Intn(int(queue.HighPriority-queue.LowPriority+1)) + int(queue.LowPriority) return queue.Priority(p), nil } diff --git a/network/slashing/consumer.go b/network/slashing/consumer.go index aaac28fccc5..3ba8d656c21 100644 --- a/network/slashing/consumer.go +++ b/network/slashing/consumer.go @@ -7,35 +7,34 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/alsp" "github.com/onflow/flow-go/utils/logging" ) const ( - unknown = "unknown" - unExpectedValidationError = "unexpected_validation_error" - unAuthorizedSenderViolation = "unauthorized_sender" - unknownMsgTypeViolation = "unknown_message_type" - invalidMsgViolation = "invalid_message" - senderEjectedViolation = "sender_ejected" - unauthorizedUnicastOnChannel = "unauthorized_unicast_on_channel" + unknown = "unknown" ) // Consumer is a struct that logs a message for any slashable offenses. // This struct will be updated in the future when slashing is implemented. type Consumer struct { - log zerolog.Logger - metrics module.NetworkSecurityMetrics + log zerolog.Logger + metrics module.NetworkSecurityMetrics + misbehaviorReportConsumer network.MisbehaviorReportConsumer } // NewSlashingViolationsConsumer returns a new Consumer. -func NewSlashingViolationsConsumer(log zerolog.Logger, metrics module.NetworkSecurityMetrics) *Consumer { +func NewSlashingViolationsConsumer(log zerolog.Logger, metrics module.NetworkSecurityMetrics, misbehaviorReportConsumer network.MisbehaviorReportConsumer) *Consumer { return &Consumer{ - log: log.With().Str("module", "network_slashing_consumer").Logger(), - metrics: metrics, + log: log.With().Str("module", "network_slashing_consumer").Logger(), + metrics: metrics, + misbehaviorReportConsumer: misbehaviorReportConsumer, } } -func (c *Consumer) logOffense(networkOffense string, violation *Violation) { +// logOffense logs the slashing violation with details. +func (c *Consumer) logOffense(misbehavior network.Misbehavior, violation *network.Violation) { // if violation fails before the message is decoded the violation.MsgType will be unknown if len(violation.MsgType) == 0 { violation.MsgType = unknown @@ -51,7 +50,7 @@ func (c *Consumer) logOffense(networkOffense string, violation *Violation) { e := c.log.Error(). Str("peer_id", violation.PeerID). - Str("networking_offense", networkOffense). + Str("misbehavior", misbehavior.String()). Str("message_type", violation.MsgType). Str("channel", violation.Channel.String()). Str("protocol", violation.Protocol.String()). @@ -62,37 +61,77 @@ func (c *Consumer) logOffense(networkOffense string, violation *Violation) { e.Msg(fmt.Sprintf("potential slashable offense: %s", violation.Err)) // capture unauthorized message count metric - c.metrics.OnUnauthorizedMessage(role, violation.MsgType, violation.Channel.String(), networkOffense) + c.metrics.OnUnauthorizedMessage(role, violation.MsgType, violation.Channel.String(), misbehavior.String()) } -// OnUnAuthorizedSenderError logs an error for unauthorized sender error. -func (c *Consumer) OnUnAuthorizedSenderError(violation *Violation) { - c.logOffense(unAuthorizedSenderViolation, violation) +// reportMisbehavior reports the slashing violation to the alsp misbehavior report manager. When violation identity +// is nil this indicates the misbehavior occurred either on a public network and the identity of the sender is unknown +// we can skip reporting the misbehavior. +// Args: +// - misbehavior: the network misbehavior. +// - violation: the slashing violation. +// Any error encountered while creating the misbehavior report is considered irrecoverable and will result in a fatal log. +func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation *network.Violation) { + if violation.Identity == nil { + c.log.Debug(). + Bool(logging.KeySuspicious, true). + Str("peerID", violation.PeerID). + Msg("violation identity unknown (or public) skipping misbehavior reporting") + c.metrics.OnViolationReportSkipped() + return + } + report, err := alsp.NewMisbehaviorReport(violation.Identity.NodeID, misbehavior) + if err != nil { + // failing to create the misbehavior report is unlikely. If an error is encountered while + // creating the misbehavior report it indicates a bug and processing can not proceed. + c.log.Fatal(). + Err(err). + Str("peerID", violation.PeerID). + Msg("failed to create misbehavior report") + } + c.misbehaviorReportConsumer.ReportMisbehaviorOnChannel(violation.Channel, report) +} + +// OnUnAuthorizedSenderError logs an error for unauthorized sender error and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnUnAuthorizedSenderError(violation *network.Violation) { + c.logOffense(alsp.UnAuthorizedSender, violation) + c.reportMisbehavior(alsp.UnAuthorizedSender, violation) } -// OnUnknownMsgTypeError logs an error for unknown message type error. -func (c *Consumer) OnUnknownMsgTypeError(violation *Violation) { - c.logOffense(unknownMsgTypeViolation, violation) +// OnUnknownMsgTypeError logs an error for unknown message type error and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnUnknownMsgTypeError(violation *network.Violation) { + c.logOffense(alsp.UnknownMsgType, violation) + c.reportMisbehavior(alsp.UnknownMsgType, violation) } // OnInvalidMsgError logs an error for messages that contained payloads that could not -// be unmarshalled into the message type denoted by message code byte. -func (c *Consumer) OnInvalidMsgError(violation *Violation) { - c.logOffense(invalidMsgViolation, violation) +// be unmarshalled into the message type denoted by message code byte and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnInvalidMsgError(violation *network.Violation) { + c.logOffense(alsp.InvalidMessage, violation) + c.reportMisbehavior(alsp.InvalidMessage, violation) +} + +// OnSenderEjectedError logs an error for sender ejected error and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnSenderEjectedError(violation *network.Violation) { + c.logOffense(alsp.SenderEjected, violation) + c.reportMisbehavior(alsp.SenderEjected, violation) } -// OnSenderEjectedError logs an error for sender ejected error. -func (c *Consumer) OnSenderEjectedError(violation *Violation) { - c.logOffense(senderEjectedViolation, violation) +// OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnUnauthorizedUnicastOnChannel(violation *network.Violation) { + c.logOffense(alsp.UnauthorizedUnicastOnChannel, violation) + c.reportMisbehavior(alsp.UnauthorizedUnicastOnChannel, violation) } -// OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast. -func (c *Consumer) OnUnauthorizedUnicastOnChannel(violation *Violation) { - c.logOffense(unauthorizedUnicastOnChannel, violation) +// OnUnauthorizedPublishOnChannel logs an error for messages unauthorized to be sent via pubsub. +func (c *Consumer) OnUnauthorizedPublishOnChannel(violation *network.Violation) { + c.logOffense(alsp.UnauthorizedPublishOnChannel, violation) + c.reportMisbehavior(alsp.UnauthorizedPublishOnChannel, violation) } // OnUnexpectedError logs an error for unexpected errors. This indicates message validation -// has failed for an unknown reason and could potentially be n slashable offense. -func (c *Consumer) OnUnexpectedError(violation *Violation) { - c.logOffense(unExpectedValidationError, violation) +// has failed for an unknown reason and could potentially be n slashable offense and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnUnexpectedError(violation *network.Violation) { + c.logOffense(alsp.UnExpectedValidationError, violation) + c.reportMisbehavior(alsp.UnExpectedValidationError, violation) } diff --git a/network/stub/network.go b/network/stub/network.go index ef99b3e39aa..5990e245944 100644 --- a/network/stub/network.go +++ b/network/stub/network.go @@ -41,6 +41,9 @@ func WithConduitFactory(factory network.ConduitFactory) func(*Network) { } } +var _ network.Network = (*Network)(nil) +var _ network.Adapter = (*Network)(nil) + // NewNetwork create a mocked Network. // The committee has the identity of the node already, so only `committee` is needed // in order for a mock hub to find each other. @@ -157,7 +160,11 @@ func (n *Network) PublishOnChannel(channel channels.Channel, event interface{}, // Engines attached to the same channel on other nodes. The targeted nodes are selected based on the selector. // In this test helper implementation, multicast uses submit method under the hood. func (n *Network) MulticastOnChannel(channel channels.Channel, event interface{}, num uint, targetIDs ...flow.Identifier) error { - targetIDs = flow.Sample(num, targetIDs...) + var err error + targetIDs, err = flow.Sample(num, targetIDs...) + if err != nil { + return fmt.Errorf("sampling failed: %w", err) + } return n.submit(channel, event, targetIDs...) } @@ -302,3 +309,7 @@ func (n *Network) StartConDev(updateInterval time.Duration, recursive bool) { func (n *Network) StopConDev() { close(n.qCD) } + +func (n *Network) ReportMisbehaviorOnChannel(_ channels.Channel, _ network.MisbehaviorReport) { + // no-op for stub network. +} diff --git a/network/test/blob_service_test.go b/network/test/blob_service_test.go index bcce039fa35..c0979244ad8 100644 --- a/network/test/blob_service_test.go +++ b/network/test/blob_service_test.go @@ -3,7 +3,6 @@ package test import ( "context" "fmt" - "os" "testing" "time" @@ -11,12 +10,14 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/sync" blockstore "github.com/ipfs/go-ipfs-blockstore" - "github.com/rs/zerolog" "github.com/stretchr/testify/suite" "go.uber.org/atomic" + "github.com/onflow/flow-go/network/mocknetwork" + "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/dht" - + p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" + p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/utils/unittest" "github.com/onflow/flow-go/model/flow" @@ -26,7 +27,6 @@ import ( "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/testutils" - "github.com/onflow/flow-go/network/mocknetwork" ) // conditionalTopology is a topology that behaves like the underlying topology when the condition is true, @@ -71,8 +71,6 @@ func (suite *BlobServiceTestSuite) putBlob(ds datastore.Batching, blob blobs.Blo func (suite *BlobServiceTestSuite) SetupTest() { suite.numNodes = 3 - logger := zerolog.New(os.Stdout) - // Bitswap listens to connect events but doesn't iterate over existing connections, and fixing this without // race conditions is tricky given the way the code is architected. As a result, libP2P hosts must first listen // on Bitswap before connecting to each other, otherwise their Bitswap requests may never reach each other. @@ -84,22 +82,21 @@ func (suite *BlobServiceTestSuite) SetupTest() { signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) - ids, nodes, mws, networks, _ := testutils.GenerateIDsMiddlewaresNetworks( - suite.T(), + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(suite.T(), suite.numNodes, - logger, - unittest.NetworkCodec(), - mocknetwork.NewViolationsConsumer(suite.T()), - testutils.WithDHT("blob_service_test", dht.AsServer()), - testutils.WithPeerUpdateInterval(time.Second), - ) - suite.networks = networks - - testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, networks, 100*time.Millisecond) + p2ptest.WithDHTOptions(dht.AsServer()), + p2ptest.WithPeerManagerEnabled(&p2pconfig.PeerManagerConfig{ + UpdateInterval: 1 * time.Second, + ConnectionPruning: true, + ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), + }, nil)) + mws, _ := testutils.MiddlewareFixtures(suite.T(), ids, nodes, testutils.MiddlewareConfigFixture(suite.T()), mocknetwork.NewViolationsConsumer(suite.T())) + suite.networks = testutils.NetworksFixture(suite.T(), ids, mws) + testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, suite.networks, 100*time.Millisecond) blobExchangeChannel := channels.Channel("blob-exchange") - for i, net := range networks { + for i, net := range suite.networks { ds := sync.MutexWrap(datastore.NewMapDatastore()) suite.datastores = append(suite.datastores, ds) blob := blobs.NewBlob([]byte(fmt.Sprintf("foo%v", i))) @@ -107,7 +104,7 @@ func (suite *BlobServiceTestSuite) SetupTest() { suite.putBlob(ds, blob) blobService, err := net.RegisterBlobService(blobExchangeChannel, ds) suite.Require().NoError(err) - <-blobService.Ready() + unittest.RequireCloseBefore(suite.T(), blobService.Ready(), 100*time.Millisecond, "blob service not ready") suite.blobServices = append(suite.blobServices, blobService) } diff --git a/network/test/echoengine_test.go b/network/test/echoengine_test.go index d04c1a6007c..55732b64d17 100644 --- a/network/test/echoengine_test.go +++ b/network/test/echoengine_test.go @@ -3,16 +3,12 @@ package test import ( "context" "fmt" - "os" "strings" "sync" "testing" "time" - "github.com/onflow/flow-go/network/p2p" - "github.com/ipfs/go-log" - "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -24,6 +20,7 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/testutils" "github.com/onflow/flow-go/network/mocknetwork" + "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/utils/unittest" ) @@ -48,7 +45,6 @@ func TestStubEngineTestSuite(t *testing.T) { func (suite *EchoEngineTestSuite) SetupTest() { const count = 2 - logger := zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) log.SetAllLoggers(log.LevelError) ctx, cancel := context.WithCancel(context.Background()) @@ -58,14 +54,9 @@ func (suite *EchoEngineTestSuite) SetupTest() { // both nodes should be of the same role to get connected on epidemic dissemination var nodes []p2p.LibP2PNode - suite.ids, nodes, suite.mws, suite.nets, _ = testutils.GenerateIDsMiddlewaresNetworks( - suite.T(), - count, - logger, - unittest.NetworkCodec(), - mocknetwork.NewViolationsConsumer(suite.T()), - ) - + suite.ids, nodes, _ = testutils.LibP2PNodeForMiddlewareFixture(suite.T(), count) + suite.mws, _ = testutils.MiddlewareFixtures(suite.T(), suite.ids, nodes, testutils.MiddlewareConfigFixture(suite.T()), mocknetwork.NewViolationsConsumer(suite.T())) + suite.nets = testutils.NetworksFixture(suite.T(), suite.ids, suite.mws) testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, suite.nets, 100*time.Millisecond) } diff --git a/network/test/epochtransition_test.go b/network/test/epochtransition_test.go index 8b7c0a655bd..34a037a90e7 100644 --- a/network/test/epochtransition_test.go +++ b/network/test/epochtransition_test.go @@ -135,7 +135,7 @@ func (suite *MutableIdentityTableSuite) signalIdentityChanged() { func (suite *MutableIdentityTableSuite) SetupTest() { suite.testNodes = newTestNodeList() suite.removedTestNodes = newTestNodeList() - rand.Seed(time.Now().UnixNano()) + nodeCount := 10 suite.logger = zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) log.SetAllLoggers(log.LevelError) @@ -181,13 +181,9 @@ func (suite *MutableIdentityTableSuite) addNodes(count int) { signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) // create the ids, middlewares and networks - ids, nodes, mws, nets, _ := testutils.GenerateIDsMiddlewaresNetworks( - suite.T(), - count, - suite.logger, - unittest.NetworkCodec(), - mocknetwork.NewViolationsConsumer(suite.T()), - ) + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(suite.T(), count) + mws, _ := testutils.MiddlewareFixtures(suite.T(), ids, nodes, testutils.MiddlewareConfigFixture(suite.T()), mocknetwork.NewViolationsConsumer(suite.T())) + nets := testutils.NetworksFixture(suite.T(), ids, mws) suite.cancels = append(suite.cancels, cancel) testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, nets, 100*time.Millisecond) diff --git a/network/test/meshengine_test.go b/network/test/meshengine_test.go index 221bba44bc2..55a95994d45 100644 --- a/network/test/meshengine_test.go +++ b/network/test/meshengine_test.go @@ -11,8 +11,6 @@ import ( "testing" "time" - "github.com/onflow/flow-go/network/p2p" - "github.com/ipfs/go-log" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/rs/zerolog" @@ -20,10 +18,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/onflow/flow-go/network/mocknetwork" - "github.com/onflow/flow-go/network/p2p/middleware" - "github.com/onflow/flow-go/network/p2p/p2pnode" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/libp2p/message" @@ -32,6 +26,10 @@ import ( "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/testutils" + "github.com/onflow/flow-go/network/mocknetwork" + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/middleware" + "github.com/onflow/flow-go/network/p2p/p2pnode" "github.com/onflow/flow-go/utils/unittest" ) @@ -74,15 +72,9 @@ func (suite *MeshEngineTestSuite) SetupTest() { signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) var nodes []p2p.LibP2PNode - suite.ids, nodes, suite.mws, suite.nets, obs = testutils.GenerateIDsMiddlewaresNetworks( - suite.T(), - count, - logger, - unittest.NetworkCodec(), - mocknetwork.NewViolationsConsumer(suite.T()), - testutils.WithIdentityOpts(unittest.WithAllRoles()), - ) - + suite.ids, nodes, obs = testutils.LibP2PNodeForMiddlewareFixture(suite.T(), count) + suite.mws, _ = testutils.MiddlewareFixtures(suite.T(), suite.ids, nodes, testutils.MiddlewareConfigFixture(suite.T()), mocknetwork.NewViolationsConsumer(suite.T())) + suite.nets = testutils.NetworksFixture(suite.T(), suite.ids, suite.mws) testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, suite.nets, 100*time.Millisecond) for _, observableConnMgr := range obs { diff --git a/network/test/middleware_test.go b/network/test/middleware_test.go index 7f8884e8ee7..1b42df088aa 100644 --- a/network/test/middleware_test.go +++ b/network/test/middleware_test.go @@ -38,7 +38,6 @@ import ( p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/network/p2p/utils/ratelimiter" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/utils/unittest" ) @@ -80,12 +79,11 @@ type MiddlewareTestSuite struct { ids []*flow.Identity metrics *metrics.NoopCollector // no-op performance monitoring simulation logger zerolog.Logger - providers []*testutils.UpdatableIDProvider + providers []*unittest.UpdatableIDProvider - mwCancel context.CancelFunc - mwCtx irrecoverable.SignalerContext - - slashingViolationsConsumer slashing.ViolationsConsumer + mwCancel context.CancelFunc + mwCtx irrecoverable.SignalerContext + slashingViolationsConsumer network.ViolationsConsumer } // TestMiddlewareTestSuit runs all the test methods in this test suit @@ -110,13 +108,8 @@ func (m *MiddlewareTestSuite) SetupTest() { } m.slashingViolationsConsumer = mocknetwork.NewViolationsConsumer(m.T()) - - m.ids, m.nodes, m.mws, obs, m.providers = testutils.GenerateIDsAndMiddlewares(m.T(), - m.size, - m.logger, - unittest.NetworkCodec(), - m.slashingViolationsConsumer) - + m.ids, m.nodes, obs = testutils.LibP2PNodeForMiddlewareFixture(m.T(), m.size) + m.mws, m.providers = testutils.MiddlewareFixtures(m.T(), m.ids, m.nodes, testutils.MiddlewareConfigFixture(m.T()), m.slashingViolationsConsumer) for _, observableConnMgr := range obs { observableConnMgr.Subscribe(&ob) } @@ -166,9 +159,8 @@ func (m *MiddlewareTestSuite) TestUpdateNodeAddresses() { irrecoverableCtx := irrecoverable.NewMockSignalerContext(m.T(), ctx) // create a new staked identity - ids, libP2PNodes, _ := testutils.GenerateIDs(m.T(), m.logger, 1) - - mws, providers := testutils.GenerateMiddlewares(m.T(), m.logger, ids, libP2PNodes, unittest.NetworkCodec(), m.slashingViolationsConsumer) + ids, libP2PNodes, _ := testutils.LibP2PNodeForMiddlewareFixture(m.T(), 1) + mws, providers := testutils.MiddlewareFixtures(m.T(), ids, libP2PNodes, testutils.MiddlewareConfigFixture(m.T()), m.slashingViolationsConsumer) require.Len(m.T(), ids, 1) require.Len(m.T(), providers, 1) require.Len(m.T(), mws, 1) @@ -224,7 +216,7 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Messages() { // burst per interval burst := 5 - messageRateLimiter := ratelimiter.NewRateLimiter(limit, burst, 3) + messageRateLimiter := ratelimiter.NewRateLimiter(limit, burst, 3*time.Second) // we only expect messages from the first middleware on the test suite expectedPID, err := unittest.PeerIDFromFlowID(m.ids[0]) @@ -248,30 +240,27 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Messages() { opts := []ratelimit.RateLimitersOption{ratelimit.WithMessageRateLimiter(messageRateLimiter), ratelimit.WithNotifier(distributor), ratelimit.WithDisabledRateLimiting(false)} rateLimiters := ratelimit.NewRateLimiters(opts...) - idProvider := testutils.NewUpdatableIDProvider(m.ids) - // create a new staked identity - connGater := testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { - if messageRateLimiter.IsRateLimited(pid) { - return fmt.Errorf("rate-limited peer") - } - return nil - }) - ids, libP2PNodes, _ := testutils.GenerateIDs(m.T(), - m.logger, + idProvider := unittest.NewUpdatableIDProvider(m.ids) + + ids, libP2PNodes, _ := testutils.LibP2PNodeForMiddlewareFixture(m.T(), 1, - testutils.WithUnicastRateLimiterDistributor(distributor), - testutils.WithConnectionGater(connGater)) + p2ptest.WithUnicastRateLimitDistributor(distributor), + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { + if messageRateLimiter.IsRateLimited(pid) { + return fmt.Errorf("rate-limited peer") + } + return nil + }))) idProvider.SetIdentities(append(m.ids, ids...)) // create middleware - mws, providers := testutils.GenerateMiddlewares(m.T(), - m.logger, + mws, providers := testutils.MiddlewareFixtures(m.T(), ids, libP2PNodes, - unittest.NetworkCodec(), + testutils.MiddlewareConfigFixture(m.T()), m.slashingViolationsConsumer, - testutils.WithUnicastRateLimiters(rateLimiters), - testutils.WithPeerManagerFilters(testutils.IsRateLimitedPeerFilter(messageRateLimiter))) + middleware.WithUnicastRateLimiters(rateLimiters), + middleware.WithPeerManagerFilters([]p2p.PeerFilter{testutils.IsRateLimitedPeerFilter(messageRateLimiter)})) require.Len(m.T(), ids, 1) require.Len(m.T(), providers, 1) @@ -317,7 +306,7 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Messages() { // return true only if the node is a direct peer of the other, after rate limiting this direct // peer should be removed by the peer manager. p2ptest.LetNodesDiscoverEachOther(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0], m.nodes[0]}, flow.IdentityList{ids[0], m.ids[0]}) - p2ptest.EnsureConnected(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0], m.nodes[0]}) + p2ptest.TryConnectionAndEnsureConnected(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0], m.nodes[0]}) // with the rate limit configured to 5 msg/sec we send 10 messages at once and expect the rate limiter // to be invoked at-least once. We send 10 messages due to the flakiness that is caused by async stream @@ -342,7 +331,7 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Messages() { time.Sleep(1 * time.Second) // ensure connection to rate limited peer is pruned - p2pfixtures.EnsureNotConnectedBetweenGroups(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0]}, []p2p.LibP2PNode{m.nodes[0]}) + p2ptest.EnsureNotConnectedBetweenGroups(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0]}, []p2p.LibP2PNode{m.nodes[0]}) p2pfixtures.EnsureNoStreamCreationBetweenGroups(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0]}, []p2p.LibP2PNode{m.nodes[0]}) // eventually the rate limited node should be able to reconnect and send messages @@ -380,7 +369,7 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Bandwidth() { require.NoError(m.T(), err) // setup bandwidth rate limiter - bandwidthRateLimiter := ratelimit.NewBandWidthRateLimiter(limit, burst, 4) + bandwidthRateLimiter := ratelimit.NewBandWidthRateLimiter(limit, burst, 4*time.Second) // the onRateLimit call back we will use to keep track of how many times a rate limit happens // after 5 rate limits we will close ch. @@ -403,32 +392,29 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Bandwidth() { opts := []ratelimit.RateLimitersOption{ratelimit.WithBandwidthRateLimiter(bandwidthRateLimiter), ratelimit.WithNotifier(distributor), ratelimit.WithDisabledRateLimiting(false)} rateLimiters := ratelimit.NewRateLimiters(opts...) - idProvider := testutils.NewUpdatableIDProvider(m.ids) - // create connection gater, connection gater will refuse connections from rate limited nodes - connGater := testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { - if bandwidthRateLimiter.IsRateLimited(pid) { - return fmt.Errorf("rate-limited peer") - } - - return nil - }) + idProvider := unittest.NewUpdatableIDProvider(m.ids) // create a new staked identity - ids, libP2PNodes, _ := testutils.GenerateIDs(m.T(), - m.logger, + ids, libP2PNodes, _ := testutils.LibP2PNodeForMiddlewareFixture(m.T(), 1, - testutils.WithUnicastRateLimiterDistributor(distributor), - testutils.WithConnectionGater(connGater)) + p2ptest.WithUnicastRateLimitDistributor(distributor), + p2ptest.WithConnectionGater(p2ptest.NewConnectionGater(idProvider, func(pid peer.ID) error { + // create connection gater, connection gater will refuse connections from rate limited nodes + if bandwidthRateLimiter.IsRateLimited(pid) { + return fmt.Errorf("rate-limited peer") + } + + return nil + }))) idProvider.SetIdentities(append(m.ids, ids...)) // create middleware - mws, providers := testutils.GenerateMiddlewares(m.T(), - m.logger, + mws, providers := testutils.MiddlewareFixtures(m.T(), ids, libP2PNodes, - unittest.NetworkCodec(), + testutils.MiddlewareConfigFixture(m.T()), m.slashingViolationsConsumer, - testutils.WithUnicastRateLimiters(rateLimiters), - testutils.WithPeerManagerFilters(testutils.IsRateLimitedPeerFilter(bandwidthRateLimiter))) + middleware.WithUnicastRateLimiters(rateLimiters), + middleware.WithPeerManagerFilters([]p2p.PeerFilter{testutils.IsRateLimitedPeerFilter(bandwidthRateLimiter)})) require.Len(m.T(), ids, 1) require.Len(m.T(), providers, 1) require.Len(m.T(), mws, 1) @@ -494,7 +480,7 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Bandwidth() { time.Sleep(1 * time.Second) // ensure connection to rate limited peer is pruned - p2pfixtures.EnsureNotConnectedBetweenGroups(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0]}, []p2p.LibP2PNode{m.nodes[0]}) + p2ptest.EnsureNotConnectedBetweenGroups(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0]}, []p2p.LibP2PNode{m.nodes[0]}) p2pfixtures.EnsureNoStreamCreationBetweenGroups(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0]}, []p2p.LibP2PNode{m.nodes[0]}) // eventually the rate limited node should be able to reconnect and send messages @@ -519,7 +505,7 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Bandwidth() { require.Equal(m.T(), uint64(1), rateLimits.Load()) } -func (m *MiddlewareTestSuite) createOverlay(provider *testutils.UpdatableIDProvider) *mocknetwork.Overlay { +func (m *MiddlewareTestSuite) createOverlay(provider *unittest.UpdatableIDProvider) *mocknetwork.Overlay { overlay := &mocknetwork.Overlay{} overlay.On("Identities").Maybe().Return(func() flow.IdentityList { return provider.Identities(filter.Any) diff --git a/network/test/unicast_authorization_test.go b/network/test/unicast_authorization_test.go index 6fe4d0b8b58..197c5f4a5a2 100644 --- a/network/test/unicast_authorization_test.go +++ b/network/test/unicast_authorization_test.go @@ -24,7 +24,6 @@ import ( "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/middleware" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/validator" "github.com/onflow/flow-go/utils/unittest" ) @@ -47,7 +46,7 @@ type UnicastAuthorizationTestSuite struct { // receiverID the identity on the mw sending the message receiverID *flow.Identity // providers id providers generated at beginning of a test run - providers []*testutils.UpdatableIDProvider + providers []*unittest.UpdatableIDProvider // cancel is the cancel func from the context that was used to start the middlewares in a test run cancel context.CancelFunc // waitCh is the channel used to wait for the middleware to perform authorization and invoke the slashing @@ -73,9 +72,10 @@ func (u *UnicastAuthorizationTestSuite) TearDownTest() { } // setupMiddlewaresAndProviders will setup 2 middlewares that will be used as a sender and receiver in each suite test. -func (u *UnicastAuthorizationTestSuite) setupMiddlewaresAndProviders(slashingViolationsConsumer slashing.ViolationsConsumer) { - ids, libP2PNodes, _ := testutils.GenerateIDs(u.T(), u.logger, 2) - mws, providers := testutils.GenerateMiddlewares(u.T(), u.logger, ids, libP2PNodes, unittest.NetworkCodec(), slashingViolationsConsumer) +func (u *UnicastAuthorizationTestSuite) setupMiddlewaresAndProviders(slashingViolationsConsumer network.ViolationsConsumer) { + ids, libP2PNodes, _ := testutils.LibP2PNodeForMiddlewareFixture(u.T(), 2) + cfg := testutils.MiddlewareConfigFixture(u.T()) + mws, providers := testutils.MiddlewareFixtures(u.T(), ids, libP2PNodes, cfg, slashingViolationsConsumer) require.Len(u.T(), ids, 2) require.Len(u.T(), providers, 2) require.Len(u.T(), mws, 2) @@ -123,7 +123,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnstakedPeer() require.NoError(u.T(), err) var nilID *flow.Identity - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: nilID, // because the peer will be unverified this identity will be nil PeerID: expectedSenderPeerID.String(), MsgType: "", // message will not be decoded before OnSenderEjectedError is logged, we won't log message type @@ -134,7 +134,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnstakedPeer() slashingViolationsConsumer.On( "OnUnAuthorizedSenderError", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -185,8 +185,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_EjectedPeer() { expectedSenderPeerID, err := unittest.PeerIDFromFlowID(u.senderID) require.NoError(u.T(), err) - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: u.senderID, // we expect this method to be called with the ejected identity + OriginID: u.senderID.NodeID, PeerID: expectedSenderPeerID.String(), MsgType: "", // message will not be decoded before OnSenderEjectedError is logged, we won't log message type Channel: channels.TestNetworkChannel, // message will not be decoded before OnSenderEjectedError is logged, we won't log peer ID @@ -196,7 +197,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_EjectedPeer() { slashingViolationsConsumer.On( "OnSenderEjectedError", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -244,8 +245,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedPee expectedSenderPeerID, err := unittest.PeerIDFromFlowID(u.senderID) require.NoError(u.T(), err) - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: u.senderID, + OriginID: u.senderID.NodeID, PeerID: expectedSenderPeerID.String(), MsgType: "*message.TestMessage", Channel: channels.ConsensusCommittee, @@ -256,7 +258,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedPee slashingViolationsConsumer.On( "OnUnAuthorizedSenderError", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -307,7 +309,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnknownMsgCode( invalidMessageCode := codec.MessageCode(byte('X')) var nilID *flow.Identity - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: nilID, PeerID: expectedSenderPeerID.String(), MsgType: "", @@ -319,7 +321,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnknownMsgCode( slashingViolationsConsumer.On( "OnUnknownMsgTypeError", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -376,8 +378,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_WrongMsgCode() modifiedMessageCode := codec.CodeDKGMessage - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: u.senderID, + OriginID: u.senderID.NodeID, PeerID: expectedSenderPeerID.String(), MsgType: "*messages.DKGMessage", Channel: channels.TestNetworkChannel, @@ -388,7 +391,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_WrongMsgCode() slashingViolationsConsumer.On( "OnUnAuthorizedSenderError", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -501,8 +504,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedUni expectedSenderPeerID, err := unittest.PeerIDFromFlowID(u.senderID) require.NoError(u.T(), err) - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: u.senderID, + OriginID: u.senderID.NodeID, PeerID: expectedSenderPeerID.String(), MsgType: "*messages.BlockProposal", Channel: channels.ConsensusCommittee, @@ -513,7 +517,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedUni slashingViolationsConsumer.On( "OnUnauthorizedUnicastOnChannel", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -564,7 +568,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_ReceiverHasNoSu expectedSenderPeerID, err := unittest.PeerIDFromFlowID(u.senderID) require.NoError(u.T(), err) - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: nil, PeerID: expectedSenderPeerID.String(), MsgType: "*message.TestMessage", @@ -576,7 +580,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_ReceiverHasNoSu slashingViolationsConsumer.On( "OnUnauthorizedUnicastOnChannel", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) diff --git a/network/validator/authorized_sender_validator.go b/network/validator/authorized_sender_validator.go index 0af21b45e39..6841d69a9e6 100644 --- a/network/validator/authorized_sender_validator.go +++ b/network/validator/authorized_sender_validator.go @@ -8,11 +8,11 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/codec" "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/slashing" ) var ( @@ -25,12 +25,12 @@ type GetIdentityFunc func(peer.ID) (*flow.Identity, bool) // AuthorizedSenderValidator performs message authorization validation. type AuthorizedSenderValidator struct { log zerolog.Logger - slashingViolationsConsumer slashing.ViolationsConsumer + slashingViolationsConsumer network.ViolationsConsumer getIdentity GetIdentityFunc } // NewAuthorizedSenderValidator returns a new AuthorizedSenderValidator -func NewAuthorizedSenderValidator(log zerolog.Logger, slashingViolationsConsumer slashing.ViolationsConsumer, getIdentity GetIdentityFunc) *AuthorizedSenderValidator { +func NewAuthorizedSenderValidator(log zerolog.Logger, slashingViolationsConsumer network.ViolationsConsumer, getIdentity GetIdentityFunc) *AuthorizedSenderValidator { return &AuthorizedSenderValidator{ log: log.With().Str("component", "authorized_sender_validator").Logger(), slashingViolationsConsumer: slashingViolationsConsumer, @@ -61,14 +61,14 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan // something terrible went wrong. identity, ok := av.getIdentity(from) if !ok { - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), Channel: channel, Protocol: protocol, Err: ErrIdentityUnverified} + violation := &network.Violation{PeerID: from.String(), Channel: channel, Protocol: protocol, Err: ErrIdentityUnverified} av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) return "", ErrIdentityUnverified } msgCode, err := codec.MessageCodeFromPayload(payload) if err != nil { - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), Channel: channel, Protocol: protocol, Err: err} av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) return "", err } @@ -77,28 +77,32 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan switch { case err == nil: return msgType, nil - case message.IsUnknownMsgTypeErr(err): - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + case message.IsUnknownMsgTypeErr(err) || codec.IsErrUnknownMsgCode(err): + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) return msgType, err case errors.Is(err, message.ErrUnauthorizedMessageOnChannel) || errors.Is(err, message.ErrUnauthorizedRole): - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) return msgType, err case errors.Is(err, ErrSenderEjected): - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} av.slashingViolationsConsumer.OnSenderEjectedError(violation) return msgType, err case errors.Is(err, message.ErrUnauthorizedUnicastOnChannel): - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} av.slashingViolationsConsumer.OnUnauthorizedUnicastOnChannel(violation) return msgType, err + case errors.Is(err, message.ErrUnauthorizedPublishOnChannel): + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + av.slashingViolationsConsumer.OnUnauthorizedPublishOnChannel(violation) + return msgType, err default: // this condition should never happen and indicates there's a bug // don't crash as a result of external inputs since that creates a DoS vector // collect slashing data because this could potentially lead to slashing err = fmt.Errorf("unexpected error during message validation: %w", err) - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} av.slashingViolationsConsumer.OnUnexpectedError(violation) return msgType, err } diff --git a/network/validator/authorized_sender_validator_test.go b/network/validator/authorized_sender_validator_test.go index 966ae5ba127..8a9cd138cbb 100644 --- a/network/validator/authorized_sender_validator_test.go +++ b/network/validator/authorized_sender_validator_test.go @@ -6,16 +6,20 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/model/flow" + libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/alsp" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/codec" "github.com/onflow/flow-go/network/message" + "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/utils/unittest" @@ -43,7 +47,7 @@ type TestAuthorizedSenderValidatorSuite struct { unauthorizedUnicastOnChannel []TestCase authorizedUnicastOnChannel []TestCase log zerolog.Logger - slashingViolationsConsumer slashing.ViolationsConsumer + slashingViolationsConsumer network.ViolationsConsumer allMsgConfigs []message.MsgAuthConfig codec network.Codec } @@ -54,7 +58,6 @@ func (s *TestAuthorizedSenderValidatorSuite) SetupTest() { s.initializeInvalidMessageOnChannelTestCases() s.initializeUnicastOnChannelTestCases() s.log = unittest.Logger() - s.slashingViolationsConsumer = slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector()) s.codec = unittest.NetworkCodec() } @@ -64,37 +67,64 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedSen for _, c := range s.authorizedSenderTestCases { str := fmt.Sprintf("role (%s) should be authorized to send message type (%s) on channel (%s)", c.Identity.Role, c.MessageStr, c.Channel) s.Run(str, func() { - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) - + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) + validateUnicast := authorizedSenderValidator.Validate + validatePubsub := authorizedSenderValidator.PubSubMessageValidator(c.Channel) pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - + switch { // ensure according to the message auth config, if a message is authorized to be sent via unicast it - // is accepted or rejected. - msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) - if c.Protocols.Contains(message.ProtocolTypeUnicast) { + // is accepted. + case c.Protocols.Contains(message.ProtocolTypeUnicast): + msgType, err := validateUnicast(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) + if c.Protocols.Contains(message.ProtocolTypeUnicast) { + require.NoError(s.T(), err) + require.Equal(s.T(), c.MessageStr, msgType) + } + // ensure according to the message auth config, if a message is authorized to be sent via pubsub it + // is accepted. + case c.Protocols.Contains(message.ProtocolTypePubSub): + payload, err := s.codec.Encode(c.Message) require.NoError(s.T(), err) - require.Equal(s.T(), c.MessageStr, msgType) - } else { - require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) - require.Equal(s.T(), c.MessageStr, msgType) - } - - payload, err := s.codec.Encode(c.Message) - require.NoError(s.T(), err) - m := &message.Message{ - ChannelID: c.Channel.String(), - Payload: payload, - } - validatePubsub := authorizedSenderValidator.PubSubMessageValidator(c.Channel) - pubsubResult := validatePubsub(pid, m) - if !c.Protocols.Contains(message.ProtocolTypePubSub) { - require.Equal(s.T(), p2p.ValidationReject, pubsubResult) - } else { + m := &message.Message{ + ChannelID: c.Channel.String(), + Payload: payload, + } + pubsubResult := validatePubsub(pid, m) require.Equal(s.T(), p2p.ValidationAccept, pubsubResult) + default: + s.T().Fatal("authconfig does not contain any protocols") } }) } + + s.Run("test messages should be allowed to be sent via both protocols unicast/pubsub on test channel", func() { + identity, _ := unittest.IdentityWithNetworkingKeyFixture(unittest.WithRole(flow.RoleCollection)) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + getIdentityFunc := s.getIdentity(identity) + pid, err := unittest.PeerIDFromFlowID(identity) + require.NoError(s.T(), err) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) + + msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeEcho.Uint8()}, channels.TestNetworkChannel, message.ProtocolTypeUnicast) + require.NoError(s.T(), err) + require.Equal(s.T(), "*message.TestMessage", msgType) + + payload, err := s.codec.Encode(&libp2pmessage.TestMessage{}) + require.NoError(s.T(), err) + m := &message.Message{ + ChannelID: channels.TestNetworkChannel.String(), + Payload: payload, + } + validatePubsub := authorizedSenderValidator.PubSubMessageValidator(channels.TestNetworkChannel) + pubsubResult := validatePubsub(pid, m) + require.Equal(s.T(), p2p.ValidationAccept, pubsubResult) + }) } // TestValidatorCallback_UnAuthorizedSender checks that AuthorizedSenderValidator.Validate return's p2p.ValidationReject @@ -105,8 +135,12 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedS s.Run(str, func() { pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(c.Identity.NodeID, alsp.UnAuthorizedSender) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) payload, err := s.codec.Encode(c.Message) require.NoError(s.T(), err) @@ -129,8 +163,10 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedUni s.Run(str, func() { pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) require.NoError(s.T(), err) @@ -147,8 +183,12 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedU s.Run(str, func() { pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(c.Identity.NodeID, alsp.UnauthorizedUnicastOnChannel) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) @@ -165,8 +205,12 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedM s.Run(str, func() { pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(c.Identity.NodeID, alsp.UnAuthorizedSender) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Twice() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, message.ErrUnauthorizedMessageOnChannel) @@ -195,10 +239,22 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ClusterPrefix pid, err := unittest.PeerIDFromFlowID(identity) require.NoError(s.T(), err) - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(identity.NodeID, alsp.UnauthorizedUnicastOnChannel) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.SyncCluster(clusterID), expectedMisbehaviorReport).Once() + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.ConsensusCluster(clusterID), expectedMisbehaviorReport).Once() + + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) + + // validate collection sync cluster SyncRequest is not allowed to be sent on channel via unicast + msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCluster(clusterID), message.ProtocolTypeUnicast) + require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) + require.Equal(s.T(), "*messages.SyncRequest", msgType) // ensure ClusterBlockProposal not allowed to be sent on channel via unicast - msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeClusterBlockProposal.Uint8()}, channels.ConsensusCluster(clusterID), message.ProtocolTypeUnicast) + msgType, err = authorizedSenderValidator.Validate(pid, []byte{codec.CodeClusterBlockProposal.Uint8()}, channels.ConsensusCluster(clusterID), message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) require.Equal(s.T(), "*messages.ClusterBlockProposal", msgType) @@ -213,11 +269,6 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ClusterPrefix pubsubResult := validateCollConsensusPubsub(pid, m) require.Equal(s.T(), p2p.ValidationAccept, pubsubResult) - // validate collection sync cluster SyncRequest is not allowed to be sent on channel via unicast - msgType, err = authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCluster(clusterID), message.ProtocolTypeUnicast) - require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) - require.Equal(s.T(), "*messages.SyncRequest", msgType) - // ensure SyncRequest is allowed to be sent via pubsub by authorized sender payload, err = s.codec.Encode(&messages.SyncRequest{}) require.NoError(s.T(), err) @@ -239,7 +290,12 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai pid, err := unittest.PeerIDFromFlowID(identity) require.NoError(s.T(), err) - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(identity.NodeID, alsp.SenderEjected) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.SyncCommittee, expectedMisbehaviorReport).Twice() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCommittee, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, ErrSenderEjected) @@ -263,7 +319,12 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai pid, err := unittest.PeerIDFromFlowID(identity) require.NoError(s.T(), err) - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(identity.NodeID, alsp.UnknownMsgType) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.ConsensusCommittee, expectedMisbehaviorReport).Twice() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) validatePubsub := authorizedSenderValidator.PubSubMessageValidator(channels.ConsensusCommittee) // unknown message types are rejected @@ -291,7 +352,11 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai pid, err := unittest.PeerIDFromFlowID(identity) require.NoError(s.T(), err) - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + // we cannot penalize a peer if identity is not known, in this case we don't expect any misbehavior reports to be reported + defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCommittee, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, ErrIdentityUnverified) @@ -314,17 +379,21 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnauthorizedP for _, c := range s.authorizedUnicastOnChannel { str := fmt.Sprintf("message type (%s) is not authorized to be sent via libp2p publish", c.MessageStr) s.Run(str, func() { + // skip test message check + if c.MessageStr == "*message.TestMessage" { + return + } pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(c.Identity.NodeID, alsp.UnauthorizedPublishOnChannel) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypePubSub) - if c.MessageStr == "*message.TestMessage" { - require.NoError(s.T(), err) - } else { - require.ErrorIs(s.T(), err, message.ErrUnauthorizedPublishOnChannel) - require.Equal(s.T(), c.MessageStr, msgType) - } + require.ErrorIs(s.T(), err, message.ErrUnauthorizedPublishOnChannel) + require.Equal(s.T(), c.MessageStr, msgType) }) } } diff --git a/network/slashing/violations_consumer.go b/network/violations_consumer.go similarity index 61% rename from network/slashing/violations_consumer.go rename to network/violations_consumer.go index cf1f8ea7d85..6c3de412c77 100644 --- a/network/slashing/violations_consumer.go +++ b/network/violations_consumer.go @@ -1,4 +1,4 @@ -package slashing +package network import ( "github.com/onflow/flow-go/model/flow" @@ -6,24 +6,30 @@ import ( "github.com/onflow/flow-go/network/message" ) +// ViolationsConsumer logs reported slashing violation errors and reports those violations as misbehavior's to the ALSP +// misbehavior report manager. Any errors encountered while reporting the misbehavior are considered irrecoverable and +// will result in a fatal level log. type ViolationsConsumer interface { - // OnUnAuthorizedSenderError logs an error for unauthorized sender error + // OnUnAuthorizedSenderError logs an error for unauthorized sender error. OnUnAuthorizedSenderError(violation *Violation) - // OnUnknownMsgTypeError logs an error for unknown message type error + // OnUnknownMsgTypeError logs an error for unknown message type error. OnUnknownMsgTypeError(violation *Violation) // OnInvalidMsgError logs an error for messages that contained payloads that could not // be unmarshalled into the message type denoted by message code byte. OnInvalidMsgError(violation *Violation) - // OnSenderEjectedError logs an error for sender ejected error + // OnSenderEjectedError logs an error for sender ejected error. OnSenderEjectedError(violation *Violation) - // OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast + // OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast. OnUnauthorizedUnicastOnChannel(violation *Violation) - // OnUnexpectedError logs an error for unknown errors + // OnUnauthorizedPublishOnChannel logs an error for messages unauthorized to be sent via pubsub. + OnUnauthorizedPublishOnChannel(violation *Violation) + + // OnUnexpectedError logs an error for unknown errors. OnUnexpectedError(violation *Violation) } diff --git a/state/cluster/badger/mutator.go b/state/cluster/badger/mutator.go index a5c39142f00..a4d867f4a8a 100644 --- a/state/cluster/badger/mutator.go +++ b/state/cluster/badger/mutator.go @@ -11,8 +11,10 @@ import ( "github.com/onflow/flow-go/model/cluster" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/trace" "github.com/onflow/flow-go/state" + clusterstate "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/state/fork" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/operation" @@ -26,6 +28,8 @@ type MutableState struct { payloads storage.ClusterPayloads } +var _ clusterstate.MutableState = (*MutableState)(nil) + func NewMutableState(state *State, tracer module.Tracer, headers storage.Headers, payloads storage.ClusterPayloads) (*MutableState, error) { mutableState := &MutableState{ State: state, @@ -36,202 +40,308 @@ func NewMutableState(state *State, tracer module.Tracer, headers storage.Headers return mutableState, nil } -// Extend validates that the given cluster block passes compliance rules, then inserts -// it to the cluster state. -// TODO (Ramtin) pass context here -// Expected errors during normal operations: -// - state.OutdatedExtensionError if the candidate block is outdated (e.g. orphaned) -// - state.InvalidExtensionError if the candidate block is invalid -func (m *MutableState) Extend(block *cluster.Block) error { - - blockID := block.ID() +// extendContext encapsulates all state information required in order to validate a candidate cluster block. +type extendContext struct { + candidate *cluster.Block // the proposed candidate cluster block + finalizedClusterBlock *flow.Header // the latest finalized cluster block + finalizedConsensusHeight uint64 // the latest finalized height on the main chain + epochFirstHeight uint64 // the first height of this cluster's operating epoch + epochLastHeight uint64 // the last height of this cluster's operating epoch (may be unknown) + epochHasEnded bool // whether this cluster's operating epoch has ended (whether the above field is known) +} - span, ctx := m.tracer.StartCollectionSpan(context.Background(), blockID, trace.COLClusterStateMutatorExtend) - defer span.End() +// getExtendCtx reads all required information from the database in order to validate +// a candidate cluster block. +// No errors are expected during normal operation. +func (m *MutableState) getExtendCtx(candidate *cluster.Block) (extendContext, error) { + var ctx extendContext + ctx.candidate = candidate err := m.State.db.View(func(tx *badger.Txn) error { - - setupSpan, _ := m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendSetup) - - header := block.Header - payload := block.Payload - - // check chain ID - if header.ChainID != m.State.clusterID { - return state.NewInvalidExtensionErrorf("new block chain ID (%s) does not match configured (%s)", block.Header.ChainID, m.State.clusterID) - } - - // check for a specified reference block - // we also implicitly check this later, but can fail fast here - if payload.ReferenceBlockID == flow.ZeroID { - return state.NewInvalidExtensionError("new block has empty reference block ID") - } - - // get the chain ID, which determines which cluster state to query - chainID := header.ChainID - // get the latest finalized cluster block and latest finalized consensus height - var finalizedClusterBlock flow.Header - err := procedure.RetrieveLatestFinalizedClusterHeader(chainID, &finalizedClusterBlock)(tx) + ctx.finalizedClusterBlock = new(flow.Header) + err := procedure.RetrieveLatestFinalizedClusterHeader(candidate.Header.ChainID, ctx.finalizedClusterBlock)(tx) if err != nil { return fmt.Errorf("could not retrieve finalized cluster head: %w", err) } - var finalizedConsensusHeight uint64 - err = operation.RetrieveFinalizedHeight(&finalizedConsensusHeight)(tx) + err = operation.RetrieveFinalizedHeight(&ctx.finalizedConsensusHeight)(tx) if err != nil { return fmt.Errorf("could not retrieve finalized height on consensus chain: %w", err) } - // get the header of the parent of the new block - parent, err := m.headers.ByBlockID(header.ParentID) + err = operation.RetrieveEpochFirstHeight(m.State.epoch, &ctx.epochFirstHeight)(tx) if err != nil { - return fmt.Errorf("could not retrieve latest finalized header: %w", err) + return fmt.Errorf("could not get operating epoch first height: %w", err) } - - // extending block must have correct parent view - if header.ParentView != parent.View { - return state.NewInvalidExtensionErrorf("candidate build with inconsistent parent view (candidate: %d, parent %d)", - header.ParentView, parent.View) + err = operation.RetrieveEpochLastHeight(m.State.epoch, &ctx.epochLastHeight)(tx) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + ctx.epochHasEnded = false + return nil + } + return fmt.Errorf("unexpected failure to retrieve final height of operating epoch: %w", err) } + ctx.epochHasEnded = true + return nil + }) + if err != nil { + return extendContext{}, fmt.Errorf("could not read required state information for Extend checks: %w", err) + } + return ctx, nil +} - // the extending block must increase height by 1 from parent - if header.Height != parent.Height+1 { - return state.NewInvalidExtensionErrorf("extending block height (%d) must be parent height + 1 (%d)", - block.Header.Height, parent.Height) - } +// Extend introduces the given block into the cluster state as a pending +// without modifying the current finalized state. +// The block's parent must have already been successfully inserted. +// TODO(ramtin) pass context here +// Expected errors during normal operations: +// - state.OutdatedExtensionError if the candidate block is outdated (e.g. orphaned) +// - state.UnverifiableExtensionError if the reference block is _not_ a known finalized block +// - state.InvalidExtensionError if the candidate block is invalid +func (m *MutableState) Extend(candidate *cluster.Block) error { + parentSpan, ctx := m.tracer.StartCollectionSpan(context.Background(), candidate.ID(), trace.COLClusterStateMutatorExtend) + defer parentSpan.End() - // ensure that the extending block connects to the finalized state, we - // do this by tracing back until we see a parent block that is the - // latest finalized block, or reach height below the finalized boundary + span, _ := m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendCheckHeader) + err := m.checkHeaderValidity(candidate) + span.End() + if err != nil { + return fmt.Errorf("error checking header validity: %w", err) + } - setupSpan.End() - checkAnsSpan, _ := m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendCheckAncestry) + span, _ = m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendGetExtendCtx) + extendCtx, err := m.getExtendCtx(candidate) + span.End() + if err != nil { + return fmt.Errorf("error gettting extend context data: %w", err) + } - // start with the extending block's parent - parentID := header.ParentID - for parentID != finalizedClusterBlock.ID() { + span, _ = m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendCheckAncestry) + err = m.checkConnectsToFinalizedState(extendCtx) + span.End() + if err != nil { + return fmt.Errorf("error checking connection to finalized state: %w", err) + } - // get the parent of current block - ancestor, err := m.headers.ByBlockID(parentID) - if err != nil { - return fmt.Errorf("could not get parent (%x): %w", block.Header.ParentID, err) - } + span, _ = m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendCheckReferenceBlock) + err = m.checkPayloadReferenceBlock(extendCtx) + span.End() + if err != nil { + return fmt.Errorf("error checking reference block: %w", err) + } - // if its height is below current boundary, the block does not connect - // to the finalized protocol state and would break database consistency - if ancestor.Height < finalizedClusterBlock.Height { - return state.NewOutdatedExtensionErrorf("block doesn't connect to finalized state. ancestor.Height (%d), final.Height (%d)", - ancestor.Height, finalizedClusterBlock.Height) - } + span, _ = m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendCheckTransactionsValid) + err = m.checkPayloadTransactions(extendCtx) + span.End() + if err != nil { + return fmt.Errorf("error checking payload transactions: %w", err) + } - parentID = ancestor.ParentID - } + span, _ = m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendDBInsert) + err = operation.RetryOnConflict(m.State.db.Update, procedure.InsertClusterBlock(candidate)) + span.End() + if err != nil { + return fmt.Errorf("could not insert cluster block: %w", err) + } + return nil +} + +// checkHeaderValidity validates that the candidate block has a header which is +// valid generally for inclusion in the cluster consensus, and w.r.t. its parent. +// Expected error returns: +// - state.InvalidExtensionError if the candidate header is invalid +func (m *MutableState) checkHeaderValidity(candidate *cluster.Block) error { + header := candidate.Header + + // check chain ID + if header.ChainID != m.State.clusterID { + return state.NewInvalidExtensionErrorf("new block chain ID (%s) does not match configured (%s)", header.ChainID, m.State.clusterID) + } + + // get the header of the parent of the new block + parent, err := m.headers.ByBlockID(header.ParentID) + if err != nil { + return irrecoverable.NewExceptionf("could not retrieve latest finalized header: %w", err) + } + + // extending block must have correct parent view + if header.ParentView != parent.View { + return state.NewInvalidExtensionErrorf("candidate build with inconsistent parent view (candidate: %d, parent %d)", + header.ParentView, parent.View) + } - checkAnsSpan.End() - checkTxsSpan, _ := m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendCheckTransactionsValid) - defer checkTxsSpan.End() + // the extending block must increase height by 1 from parent + if header.Height != parent.Height+1 { + return state.NewInvalidExtensionErrorf("extending block height (%d) must be parent height + 1 (%d)", + header.Height, parent.Height) + } + return nil +} - // a valid collection must reference a valid reference block - // NOTE: it is valid for a collection to be expired at this point, - // otherwise we would compromise liveness of the cluster. - refBlock, err := m.headers.ByBlockID(payload.ReferenceBlockID) +// checkConnectsToFinalizedState validates that the candidate block connects to +// the latest finalized state (ie. is not extending an orphaned fork). +// Expected error returns: +// - state.OutdatedExtensionError if the candidate extends an orphaned fork +func (m *MutableState) checkConnectsToFinalizedState(ctx extendContext) error { + header := ctx.candidate.Header + finalizedID := ctx.finalizedClusterBlock.ID() + finalizedHeight := ctx.finalizedClusterBlock.Height + + // start with the extending block's parent + parentID := header.ParentID + for parentID != finalizedID { + // get the parent of current block + ancestor, err := m.headers.ByBlockID(parentID) if err != nil { - if errors.Is(err, storage.ErrNotFound) { - return state.NewUnverifiableExtensionError("cluster block references unknown reference block (id=%x)", payload.ReferenceBlockID) - } - return fmt.Errorf("could not check reference block: %w", err) + return irrecoverable.NewExceptionf("could not get parent which must be known (%x): %w", header.ParentID, err) } - // no validation of transactions is necessary for empty collections - if payload.Collection.Len() == 0 { - return nil + // if its height is below current boundary, the block does not connect + // to the finalized protocol state and would break database consistency + if ancestor.Height < finalizedHeight { + return state.NewOutdatedExtensionErrorf( + "block doesn't connect to latest finalized block (height=%d, id=%x): orphaned ancestor (height=%d, id=%x)", + finalizedHeight, finalizedID, ancestor.Height, parentID) } + parentID = ancestor.ParentID + } + return nil +} - // check that all transactions within the collection are valid - // keep track of the min/max reference blocks - the collection must be non-empty - // at this point so these are guaranteed to be set correctly - minRefID := flow.ZeroID - minRefHeight := uint64(math.MaxUint64) - maxRefHeight := uint64(0) - for _, flowTx := range payload.Collection.Transactions { - refBlock, err := m.headers.ByBlockID(flowTx.ReferenceBlockID) - if errors.Is(err, storage.ErrNotFound) { - // unknown reference blocks are invalid - return state.NewUnverifiableExtensionError("collection contains tx (tx_id=%x) with unknown reference block (block_id=%x): %w", flowTx.ID(), flowTx.ReferenceBlockID, err) - } - if err != nil { - return fmt.Errorf("could not check reference block (id=%x): %w", flowTx.ReferenceBlockID, err) - } - - if refBlock.Height < minRefHeight { - minRefHeight = refBlock.Height - minRefID = flowTx.ReferenceBlockID - } - if refBlock.Height > maxRefHeight { - maxRefHeight = refBlock.Height - } +// checkPayloadReferenceBlock validates the reference block is valid. +// - it must be a known, finalized block on the main consensus chain +// - it must be within the cluster's operating epoch +// +// Expected error returns: +// - state.InvalidExtensionError if the reference block is invalid for use. +// - state.UnverifiableExtensionError if the reference block is unknown. +func (m *MutableState) checkPayloadReferenceBlock(ctx extendContext) error { + payload := ctx.candidate.Payload + + // 1 - the reference block must be known + refBlock, err := m.headers.ByBlockID(payload.ReferenceBlockID) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return state.NewUnverifiableExtensionError("cluster block references unknown reference block (id=%x)", payload.ReferenceBlockID) } + return fmt.Errorf("could not check reference block: %w", err) + } - // a valid collection must reference the oldest reference block among - // its constituent transactions - if minRefID != payload.ReferenceBlockID { - return state.NewInvalidExtensionErrorf( - "reference block (id=%x) must match oldest transaction's reference block (id=%x)", - payload.ReferenceBlockID, minRefID, - ) + // 2 - the reference block must be finalized + if refBlock.Height > ctx.finalizedConsensusHeight { + // a reference block which is above the finalized boundary can't be verified yet + return state.NewUnverifiableExtensionError("reference block is above finalized boundary (%d>%d)", refBlock.Height, ctx.finalizedConsensusHeight) + } else { + storedBlockIDForHeight, err := m.headers.BlockIDByHeight(refBlock.Height) + if err != nil { + return irrecoverable.NewExceptionf("could not look up block ID for finalized height: %w", err) } - // a valid collection must contain only transactions within its expiry window - if maxRefHeight-minRefHeight >= flow.DefaultTransactionExpiry { - return state.NewInvalidExtensionErrorf( - "collection contains reference height range [%d,%d] exceeding expiry window size: %d", - minRefHeight, maxRefHeight, flow.DefaultTransactionExpiry) + // a reference block with height at or below the finalized boundary must have been finalized + if storedBlockIDForHeight != payload.ReferenceBlockID { + return state.NewInvalidExtensionErrorf("cluster block references orphaned reference block (id=%x, height=%d), the block finalized at this height is %x", + payload.ReferenceBlockID, refBlock.Height, storedBlockIDForHeight) } + } - // TODO ensure the reference block is part of the main chain - _ = refBlock + // TODO ensure the reference block is part of the main chain https://github.com/onflow/flow-go/issues/4204 + _ = refBlock - // check for duplicate transactions in block's ancestry - txLookup := make(map[flow.Identifier]struct{}) - for _, tx := range block.Payload.Collection.Transactions { - txID := tx.ID() - if _, exists := txLookup[txID]; exists { - return state.NewInvalidExtensionErrorf("collection contains transaction (id=%x) more than once", txID) - } - txLookup[txID] = struct{}{} - } + // 3 - the reference block must be within the cluster's operating epoch + if refBlock.Height < ctx.epochFirstHeight { + return state.NewInvalidExtensionErrorf("invalid reference block is before operating epoch for cluster, height %d<%d", refBlock.Height, ctx.epochFirstHeight) + } + if ctx.epochHasEnded && refBlock.Height > ctx.epochLastHeight { + return state.NewInvalidExtensionErrorf("invalid reference block is after operating epoch for cluster, height %d>%d", refBlock.Height, ctx.epochLastHeight) + } + return nil +} - // first, check for duplicate transactions in the un-finalized ancestry - duplicateTxIDs, err := m.checkDupeTransactionsInUnfinalizedAncestry(block, txLookup, finalizedClusterBlock.Height) - if err != nil { - return fmt.Errorf("could not check for duplicate txs in un-finalized ancestry: %w", err) +// checkPayloadTransactions validates the transactions included int the candidate cluster block's payload. +// It enforces: +// - transactions are individually valid +// - no duplicate transaction exists along the fork being extended +// - the collection's reference block is equal to the oldest reference block among +// its constituent transactions +// +// Expected error returns: +// - state.InvalidExtensionError if the reference block is invalid for use. +// - state.UnverifiableExtensionError if the reference block is unknown. +func (m *MutableState) checkPayloadTransactions(ctx extendContext) error { + block := ctx.candidate + payload := block.Payload + + if payload.Collection.Len() == 0 { + return nil + } + + // check that all transactions within the collection are valid + // keep track of the min/max reference blocks - the collection must be non-empty + // at this point so these are guaranteed to be set correctly + minRefID := flow.ZeroID + minRefHeight := uint64(math.MaxUint64) + maxRefHeight := uint64(0) + for _, flowTx := range payload.Collection.Transactions { + refBlock, err := m.headers.ByBlockID(flowTx.ReferenceBlockID) + if errors.Is(err, storage.ErrNotFound) { + // unknown reference blocks are invalid + return state.NewUnverifiableExtensionError("collection contains tx (tx_id=%x) with unknown reference block (block_id=%x): %w", flowTx.ID(), flowTx.ReferenceBlockID, err) } - if len(duplicateTxIDs) > 0 { - return state.NewInvalidExtensionErrorf("payload includes duplicate transactions in un-finalized ancestry (duplicates: %s)", duplicateTxIDs) + if err != nil { + return fmt.Errorf("could not check reference block (id=%x): %w", flowTx.ReferenceBlockID, err) } - // second, check for duplicate transactions in the finalized ancestry - duplicateTxIDs, err = m.checkDupeTransactionsInFinalizedAncestry(txLookup, minRefHeight, maxRefHeight) - if err != nil { - return fmt.Errorf("could not check for duplicate txs in finalized ancestry: %w", err) + if refBlock.Height < minRefHeight { + minRefHeight = refBlock.Height + minRefID = flowTx.ReferenceBlockID } - if len(duplicateTxIDs) > 0 { - return state.NewInvalidExtensionErrorf("payload includes duplicate transactions in finalized ancestry (duplicates: %s)", duplicateTxIDs) + if refBlock.Height > maxRefHeight { + maxRefHeight = refBlock.Height } + } - return nil - }) - if err != nil { - return fmt.Errorf("could not validate extending block: %w", err) + // a valid collection must reference the oldest reference block among + // its constituent transactions + if minRefID != payload.ReferenceBlockID { + return state.NewInvalidExtensionErrorf( + "reference block (id=%x) must match oldest transaction's reference block (id=%x)", + payload.ReferenceBlockID, minRefID, + ) + } + // a valid collection must contain only transactions within its expiry window + if maxRefHeight-minRefHeight >= flow.DefaultTransactionExpiry { + return state.NewInvalidExtensionErrorf( + "collection contains reference height range [%d,%d] exceeding expiry window size: %d", + minRefHeight, maxRefHeight, flow.DefaultTransactionExpiry) } - insertDbSpan, _ := m.tracer.StartSpanFromContext(ctx, trace.COLClusterStateMutatorExtendDBInsert) - defer insertDbSpan.End() + // check for duplicate transactions in block's ancestry + txLookup := make(map[flow.Identifier]struct{}) + for _, tx := range block.Payload.Collection.Transactions { + txID := tx.ID() + if _, exists := txLookup[txID]; exists { + return state.NewInvalidExtensionErrorf("collection contains transaction (id=%x) more than once", txID) + } + txLookup[txID] = struct{}{} + } - // insert the new block - err = operation.RetryOnConflict(m.State.db.Update, procedure.InsertClusterBlock(block)) + // first, check for duplicate transactions in the un-finalized ancestry + duplicateTxIDs, err := m.checkDupeTransactionsInUnfinalizedAncestry(block, txLookup, ctx.finalizedClusterBlock.Height) if err != nil { - return fmt.Errorf("could not insert cluster block: %w", err) + return fmt.Errorf("could not check for duplicate txs in un-finalized ancestry: %w", err) } + if len(duplicateTxIDs) > 0 { + return state.NewInvalidExtensionErrorf("payload includes duplicate transactions in un-finalized ancestry (duplicates: %s)", duplicateTxIDs) + } + + // second, check for duplicate transactions in the finalized ancestry + duplicateTxIDs, err = m.checkDupeTransactionsInFinalizedAncestry(txLookup, minRefHeight, maxRefHeight) + if err != nil { + return fmt.Errorf("could not check for duplicate txs in finalized ancestry: %w", err) + } + if len(duplicateTxIDs) > 0 { + return state.NewInvalidExtensionErrorf("payload includes duplicate transactions in finalized ancestry (duplicates: %s)", duplicateTxIDs) + } + return nil } diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index a62da45140b..1897cf6a39a 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -7,7 +7,6 @@ import ( "math/rand" "os" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/rs/zerolog" @@ -38,8 +37,9 @@ type MutatorSuite struct { db *badger.DB dbdir string - genesis *model.Block - chainID flow.ChainID + genesis *model.Block + chainID flow.ChainID + epochCounter uint64 // protocol state for reference blocks for transactions protoState protocol.FollowerState @@ -52,9 +52,6 @@ type MutatorSuite struct { func (suite *MutatorSuite) SetupTest() { var err error - // seed the RNG - rand.Seed(time.Now().UnixNano()) - suite.genesis = model.Genesis() suite.chainID = suite.genesis.Header.ChainID @@ -67,40 +64,41 @@ func (suite *MutatorSuite) SetupTest() { all := util.StorageLayer(suite.T(), suite.db) colPayloads := storage.NewClusterPayloads(metrics, suite.db) - clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) - suite.NoError(err) - clusterState, err := Bootstrap(suite.db, clusterStateRoot) - suite.Assert().Nil(err) - suite.state, err = NewMutableState(clusterState, tracer, all.Headers, colPayloads) - suite.Assert().Nil(err) - consumer := events.NewNoop() - // just bootstrap with a genesis block, we'll use this as reference - participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) - genesis, result, seal := unittest.BootstrapFixture(participants) - qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(genesis.ID())) + genesis, result, seal := unittest.BootstrapFixture(unittest.IdentityListFixture(5, unittest.WithAllRoles())) // ensure we don't enter a new epoch for tests that build many blocks - result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = genesis.Header.View + 100000 + result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = genesis.Header.View + 100_000 seal.ResultID = result.ID() - + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(genesis.ID())) rootSnapshot, err := inmem.SnapshotFromBootstrapState(genesis, result, seal, qc) require.NoError(suite.T(), err) + suite.epochCounter = rootSnapshot.Encodable().Epochs.Current.Counter suite.protoGenesis = genesis.Header - - state, err := pbadger.Bootstrap(metrics, suite.db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) - require.NoError(suite.T(), err) - - suite.protoState, err = pbadger.NewFollowerState( - log, - tracer, - consumer, - state, - all.Index, - all.Payloads, - protocolutil.MockBlockTimer(), + state, err := pbadger.Bootstrap( + metrics, + suite.db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, ) require.NoError(suite.T(), err) + suite.protoState, err = pbadger.NewFollowerState(log, tracer, events.NewNoop(), state, all.Index, all.Payloads, protocolutil.MockBlockTimer()) + require.NoError(suite.T(), err) + + clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) + suite.NoError(err) + clusterState, err := Bootstrap(suite.db, clusterStateRoot) + suite.Assert().Nil(err) + suite.state, err = NewMutableState(clusterState, tracer, all.Headers, colPayloads) + suite.Assert().Nil(err) } // runs after each test finishes @@ -175,24 +173,24 @@ func TestMutator(t *testing.T) { suite.Run(t, new(MutatorSuite)) } -func (suite *MutatorSuite) TestBootstrap_InvalidNumber() { +func (suite *MutatorSuite) TestBootstrap_InvalidHeight() { suite.genesis.Header.Height = 1 - _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) suite.Assert().Error(err) } func (suite *MutatorSuite) TestBootstrap_InvalidParentHash() { suite.genesis.Header.ParentID = unittest.IdentifierFixture() - _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) suite.Assert().Error(err) } func (suite *MutatorSuite) TestBootstrap_InvalidPayloadHash() { suite.genesis.Header.PayloadHash = unittest.IdentifierFixture() - _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) suite.Assert().Error(err) } @@ -200,7 +198,7 @@ func (suite *MutatorSuite) TestBootstrap_InvalidPayload() { // this is invalid because genesis collection should be empty suite.genesis.Payload = unittest.ClusterPayloadFixture(2) - _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) suite.Assert().Error(err) } @@ -258,7 +256,7 @@ func (suite *MutatorSuite) TestExtend_InvalidChainID() { suite.Assert().True(state.IsInvalidExtensionError(err)) } -func (suite *MutatorSuite) TestExtend_InvalidBlockNumber() { +func (suite *MutatorSuite) TestExtend_InvalidBlockHeight() { block := suite.Block() // change the block height block.Header.Height = block.Header.Height - 1 @@ -396,6 +394,69 @@ func (suite *MutatorSuite) TestExtend_WithReferenceBlockFromClusterChain() { suite.Assert().Error(err) } +// TestExtend_WithReferenceBlockFromDifferentEpoch tests extending the cluster state +// using a reference block in a different epoch than the cluster's epoch. +func (suite *MutatorSuite) TestExtend_WithReferenceBlockFromDifferentEpoch() { + // build and complete the current epoch, then use a reference block from next epoch + eb := unittest.NewEpochBuilder(suite.T(), suite.protoState) + eb.BuildEpoch().CompleteEpoch() + heights, ok := eb.EpochHeights(1) + require.True(suite.T(), ok) + nextEpochHeader, err := suite.protoState.AtHeight(heights.FinalHeight() + 1).Head() + require.NoError(suite.T(), err) + + block := suite.Block() + block.SetPayload(model.EmptyPayload(nextEpochHeader.ID())) + err = suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + +// TestExtend_WithUnfinalizedReferenceBlock tests that extending the cluster state +// with a reference block which is un-finalized and above the finalized boundary +// should be considered an unverifiable extension. It's possible that this reference +// block has been finalized, we just haven't processed it yet. +func (suite *MutatorSuite) TestExtend_WithUnfinalizedReferenceBlock() { + unfinalized := unittest.BlockWithParentFixture(suite.protoGenesis) + unfinalized.Payload.Guarantees = nil + unfinalized.SetPayload(*unfinalized.Payload) + err := suite.protoState.ExtendCertified(context.Background(), unfinalized, unittest.CertifyBlock(unfinalized.Header)) + suite.Require().NoError(err) + + block := suite.Block() + block.SetPayload(model.EmptyPayload(unfinalized.ID())) + err = suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsUnverifiableExtensionError(err)) +} + +// TestExtend_WithOrphanedReferenceBlock tests that extending the cluster state +// with a un-finalized reference block below the finalized boundary +// (i.e. orphaned) should be considered an invalid extension. As the proposer is supposed +// to only use finalized blocks as reference, the proposer knowingly generated an invalid +func (suite *MutatorSuite) TestExtend_WithOrphanedReferenceBlock() { + // create a block extending genesis which is not finalized + orphaned := unittest.BlockWithParentFixture(suite.protoGenesis) + err := suite.protoState.ExtendCertified(context.Background(), orphaned, unittest.CertifyBlock(orphaned.Header)) + suite.Require().NoError(err) + + // create a block extending genesis (conflicting with previous) which is finalized + finalized := unittest.BlockWithParentFixture(suite.protoGenesis) + finalized.Payload.Guarantees = nil + finalized.SetPayload(*finalized.Payload) + err = suite.protoState.ExtendCertified(context.Background(), finalized, unittest.CertifyBlock(finalized.Header)) + suite.Require().NoError(err) + err = suite.protoState.Finalize(context.Background(), finalized.ID()) + suite.Require().NoError(err) + + // test referencing the orphaned block + block := suite.Block() + block.SetPayload(model.EmptyPayload(orphaned.ID())) + err = suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + func (suite *MutatorSuite) TestExtend_UnfinalizedBlockWithDupeTx() { tx1 := suite.Tx() diff --git a/state/cluster/badger/snapshot_test.go b/state/cluster/badger/snapshot_test.go index b17a24e8d6e..7dd81c0ed4d 100644 --- a/state/cluster/badger/snapshot_test.go +++ b/state/cluster/badger/snapshot_test.go @@ -2,14 +2,11 @@ package badger import ( "math" - "math/rand" "os" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" model "github.com/onflow/flow-go/model/cluster" @@ -31,8 +28,9 @@ type SnapshotSuite struct { db *badger.DB dbdir string - genesis *model.Block - chainID flow.ChainID + genesis *model.Block + chainID flow.ChainID + epochCounter uint64 protoState protocol.State @@ -43,9 +41,6 @@ type SnapshotSuite struct { func (suite *SnapshotSuite) SetupTest() { var err error - // seed the RNG - rand.Seed(time.Now().UnixNano()) - suite.genesis = model.Genesis() suite.chainID = suite.genesis.Header.ChainID @@ -58,20 +53,31 @@ func (suite *SnapshotSuite) SetupTest() { all := util.StorageLayer(suite.T(), suite.db) colPayloads := storage.NewClusterPayloads(metrics, suite.db) - clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) - suite.Assert().Nil(err) + root := unittest.RootSnapshotFixture(unittest.IdentityListFixture(5, unittest.WithAllRoles())) + suite.epochCounter = root.Encodable().Epochs.Current.Counter + + suite.protoState, err = pbadger.Bootstrap( + metrics, + suite.db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + root, + ) + suite.Require().NoError(err) + + clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture(), suite.epochCounter) + suite.Require().NoError(err) clusterState, err := Bootstrap(suite.db, clusterStateRoot) - suite.Assert().Nil(err) + suite.Require().NoError(err) suite.state, err = NewMutableState(clusterState, tracer, all.Headers, colPayloads) - suite.Assert().Nil(err) - - participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) - root := unittest.RootSnapshotFixture(participants) - - suite.protoState, err = pbadger.Bootstrap(metrics, suite.db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, root) - require.NoError(suite.T(), err) - - suite.Require().Nil(err) + suite.Require().NoError(err) } // runs after each test finishes diff --git a/state/cluster/badger/state.go b/state/cluster/badger/state.go index 33186a14b14..f088328823e 100644 --- a/state/cluster/badger/state.go +++ b/state/cluster/badger/state.go @@ -17,7 +17,8 @@ import ( type State struct { db *badger.DB - clusterID flow.ChainID + clusterID flow.ChainID // the chain ID for the cluster + epoch uint64 // the operating epoch for the cluster } // Bootstrap initializes the persistent cluster state with a genesis block. @@ -31,7 +32,7 @@ func Bootstrap(db *badger.DB, stateRoot *StateRoot) (*State, error) { if isBootstrapped { return nil, fmt.Errorf("expected empty cluster state for cluster ID %s", stateRoot.ClusterID()) } - state := newState(db, stateRoot.ClusterID()) + state := newState(db, stateRoot.ClusterID(), stateRoot.EpochCounter()) genesis := stateRoot.Block() rootQC := stateRoot.QC() @@ -84,7 +85,7 @@ func Bootstrap(db *badger.DB, stateRoot *StateRoot) (*State, error) { return state, nil } -func OpenState(db *badger.DB, tracer module.Tracer, headers storage.Headers, payloads storage.ClusterPayloads, clusterID flow.ChainID) (*State, error) { +func OpenState(db *badger.DB, _ module.Tracer, _ storage.Headers, _ storage.ClusterPayloads, clusterID flow.ChainID, epoch uint64) (*State, error) { isBootstrapped, err := IsBootstrapped(db, clusterID) if err != nil { return nil, fmt.Errorf("failed to determine whether database contains bootstrapped state: %w", err) @@ -92,14 +93,15 @@ func OpenState(db *badger.DB, tracer module.Tracer, headers storage.Headers, pay if !isBootstrapped { return nil, fmt.Errorf("expected database to contain bootstrapped state") } - state := newState(db, clusterID) + state := newState(db, clusterID, epoch) return state, nil } -func newState(db *badger.DB, clusterID flow.ChainID) *State { +func newState(db *badger.DB, clusterID flow.ChainID, epoch uint64) *State { state := &State{ db: db, clusterID: clusterID, + epoch: epoch, } return state } @@ -149,7 +151,7 @@ func (s *State) AtBlockID(blockID flow.Identifier) cluster.Snapshot { return snapshot } -// IsBootstrapped returns whether or not the database contains a bootstrapped state +// IsBootstrapped returns whether the database contains a bootstrapped state. func IsBootstrapped(db *badger.DB, clusterID flow.ChainID) (bool, error) { var finalized uint64 err := db.View(operation.RetrieveClusterFinalizedHeight(clusterID, &finalized)) diff --git a/state/cluster/badger/state_root.go b/state/cluster/badger/state_root.go index e592ebd4a3c..50f15d0a373 100644 --- a/state/cluster/badger/state_root.go +++ b/state/cluster/badger/state_root.go @@ -7,13 +7,14 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// StateRoot is the root information required to bootstrap the cluster state +// StateRoot is the root information required to bootstrap the cluster state. type StateRoot struct { - block *cluster.Block - qc *flow.QuorumCertificate + block *cluster.Block // root block for the cluster chain + qc *flow.QuorumCertificate // root QC for the cluster chain + epoch uint64 // operating epoch for the cluster chain } -func NewStateRoot(genesis *cluster.Block, qc *flow.QuorumCertificate) (*StateRoot, error) { +func NewStateRoot(genesis *cluster.Block, qc *flow.QuorumCertificate, epoch uint64) (*StateRoot, error) { err := validateClusterGenesis(genesis) if err != nil { return nil, fmt.Errorf("inconsistent state root: %w", err) @@ -21,6 +22,7 @@ func NewStateRoot(genesis *cluster.Block, qc *flow.QuorumCertificate) (*StateRoo return &StateRoot{ block: genesis, qc: qc, + epoch: epoch, }, nil } @@ -59,3 +61,7 @@ func (s StateRoot) Block() *cluster.Block { func (s StateRoot) QC() *flow.QuorumCertificate { return s.qc } + +func (s StateRoot) EpochCounter() uint64 { + return s.epoch +} diff --git a/state/cluster/params.go b/state/cluster/params.go index 78581809922..8bfc2be46bd 100644 --- a/state/cluster/params.go +++ b/state/cluster/params.go @@ -8,5 +8,6 @@ import ( type Params interface { // ChainID returns the chain ID for this cluster. + // No errors are expected during normal operation. ChainID() (flow.ChainID, error) } diff --git a/state/cluster/state.go b/state/cluster/state.go index 19b58a64425..ea01f7f908d 100644 --- a/state/cluster/state.go +++ b/state/cluster/state.go @@ -34,8 +34,10 @@ type MutableState interface { State // Extend introduces the given block into the cluster state as a pending // without modifying the current finalized state. + // The block's parent must have already been successfully inserted. // Expected errors during normal operations: // - state.OutdatedExtensionError if the candidate block is outdated (e.g. orphaned) + // - state.UnverifiableExtensionError if the reference block is _not_ a known finalized block // - state.InvalidExtensionError if the candidate block is invalid Extend(candidate *cluster.Block) error } diff --git a/state/protocol/badger/mutator.go b/state/protocol/badger/mutator.go index a84c8842395..dd2f2035656 100644 --- a/state/protocol/badger/mutator.go +++ b/state/protocol/badger/mutator.go @@ -537,7 +537,7 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, certi } } else { // trigger BlockProcessable for parent blocks above root height - if parent.Height > m.rootHeight { + if parent.Height > m.finalizedRootHeight { events = append(events, func() { m.consumer.BlockProcessable(parent, qc) }) @@ -630,11 +630,11 @@ func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) e // We also want to update the last sealed height. Retrieve the block // seal indexed for the block and retrieve the block that was sealed by it. - last, err := m.seals.HighestInFork(blockID) + lastSeal, err := m.seals.HighestInFork(blockID) if err != nil { return fmt.Errorf("could not look up sealed header: %w", err) } - sealed, err := m.headers.ByBlockID(last.BlockID) + sealed, err := m.headers.ByBlockID(lastSeal.BlockID) if err != nil { return fmt.Errorf("could not retrieve sealed header: %w", err) } @@ -694,6 +694,12 @@ func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) e } } + // Extract and validate version beacon events from the block seals. + versionBeacons, err := m.versionBeaconOnBlockFinalized(block) + if err != nil { + return fmt.Errorf("cannot process version beacon: %w", err) + } + // Persist updates in database // * Add this block to the height-indexed set of finalized blocks. // * Update the largest finalized height to this block's height. @@ -737,12 +743,27 @@ func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) e } } + if len(versionBeacons) > 0 { + // only index the last version beacon as that is the relevant one. + // TODO: The other version beacons can be used for validation. + err := operation.IndexVersionBeaconByHeight(versionBeacons[len(versionBeacons)-1])(tx) + if err != nil { + return fmt.Errorf("could not index version beacon or height (%d): %w", header.Height, err) + } + } + return nil }) if err != nil { return fmt.Errorf("could not persist finalization operations for block (%x): %w", blockID, err) } + // update the cache + m.State.cachedFinal.Store(&cachedHeader{blockID, header}) + if len(block.Payload.Seals) > 0 { + m.State.cachedSealed.Store(&cachedHeader{lastSeal.BlockID, sealed}) + } + // Emit protocol events after database transaction succeeds. Event delivery is guaranteed, // _except_ in case of a crash. Hence, when recovering from a crash, consumers need to deduce // from the state whether they have missed events and re-execute the respective actions. @@ -915,8 +936,10 @@ func (m *FollowerState) epochPhaseMetricsAndEventsOnBlockFinalized(block *flow.B return nil, nil, fmt.Errorf("could not retrieve setup event for next epoch: %w", err) } events = append(events, func() { m.metrics.CommittedEpochFinalView(nextEpochSetup.FinalView) }) + case *flow.VersionBeacon: + // do nothing for now default: - return nil, nil, fmt.Errorf("invalid service event type in payload (%T)", event) + return nil, nil, fmt.Errorf("invalid service event type in payload (%T)", ev) } } } @@ -979,6 +1002,68 @@ func (m *FollowerState) epochStatus(block *flow.Header, epochFallbackTriggered b } +// versionBeaconOnBlockFinalized extracts and returns the VersionBeacons from the +// finalized block's seals. +// This could return multiple VersionBeacons if the parent block contains multiple Seals. +// The version beacons will be returned in the ascending height order of the seals. +// Technically only the last VersionBeacon is relevant. +func (m *FollowerState) versionBeaconOnBlockFinalized( + finalized *flow.Block, +) ([]*flow.SealedVersionBeacon, error) { + var versionBeacons []*flow.SealedVersionBeacon + + seals, err := protocol.OrderedSeals(finalized.Payload, m.headers) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, fmt.Errorf( + "ordering seals: parent payload contains"+ + " seals for unknown block: %w", err) + } + return nil, fmt.Errorf("unexpected error ordering seals: %w", err) + } + + for _, seal := range seals { + result, err := m.results.ByID(seal.ResultID) + if err != nil { + return nil, fmt.Errorf( + "could not retrieve result (id=%x) for seal (id=%x): %w", + seal.ResultID, + seal.ID(), + err) + } + for _, event := range result.ServiceEvents { + + ev, ok := event.Event.(*flow.VersionBeacon) + + if !ok { + // skip other service event types. + // validation if this is a known service event type is done elsewhere. + continue + } + + err := ev.Validate() + if err != nil { + m.logger.Warn(). + Err(err). + Str("block_id", finalized.ID().String()). + Interface("event", ev). + Msg("invalid VersionBeacon service event") + continue + } + + // The version beacon only becomes actionable/valid/active once the block + // containing the version beacon has been sealed. That is why we set the + // Seal height to the current block height. + versionBeacons = append(versionBeacons, &flow.SealedVersionBeacon{ + VersionBeacon: ev, + SealHeight: finalized.Header.Height, + }) + } + } + + return versionBeacons, nil +} + // handleEpochServiceEvents handles applying state changes which occur as a result // of service events being included in a block payload: // - inserting incorporated service events @@ -1112,7 +1197,8 @@ func (m *FollowerState) handleEpochServiceEvents(candidate *flow.Block) (dbUpdat // we'll insert the commit event when we insert the block dbUpdates = append(dbUpdates, m.epoch.commits.StoreTx(ev)) - + case *flow.VersionBeacon: + // do nothing for now default: return nil, fmt.Errorf("invalid service event type (type_name=%s, go_type=%T)", event.Type, ev) } diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 685e79d5931..53ecd1a6e79 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -40,10 +40,6 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - var participants = unittest.IdentityListFixture(5, unittest.WithAllRoles()) func TestBootstrapValid(t *testing.T) { @@ -103,7 +99,20 @@ func TestExtendValid(t *testing.T) { rootSnapshot, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) require.NoError(t, err) - state, err := protocol.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := protocol.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) fullState, err := protocol.NewFullConsensusState( @@ -261,6 +270,173 @@ func TestSealedIndex(t *testing.T) { } +func TestVersionBeaconIndex(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { + rootHeader, err := rootSnapshot.Head() + require.NoError(t, err) + + // build a chain: + // G <- B1 <- B2 (resultB1(vb1)) <- B3 <- B4 (resultB2(vb2), resultB3(vb3)) <- B5 (sealB1) <- B6 (sealB2, sealB3) + // up until and including finalization of B5 there should be no VBs indexed + // when B5 is finalized, index VB1 + // when B6 is finalized, we can index VB2 and VB3, but (only) the last one should be indexed by seal height + + // block 1 + b1 := unittest.BlockWithParentFixture(rootHeader) + b1.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), b1) + require.NoError(t, err) + + vb1 := unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + BlockHeight: rootHeader.Height, + Version: "0.21.37", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 100, + Version: "0.21.38", + }, + ), + ) + vb2 := unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + BlockHeight: rootHeader.Height, + Version: "0.21.37", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 101, + Version: "0.21.38", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 201, + Version: "0.21.39", + }, + ), + ) + vb3 := unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + BlockHeight: rootHeader.Height, + Version: "0.21.37", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 99, + Version: "0.21.38", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 199, + Version: "0.21.39", + }, + flow.VersionBoundary{ + BlockHeight: rootHeader.Height + 299, + Version: "0.21.40", + }, + ), + ) + + b1Receipt := unittest.ReceiptForBlockFixture(b1) + b1Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{vb1.ServiceEvent()} + b2 := unittest.BlockWithParentFixture(b1.Header) + b2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(b1Receipt))) + err = state.Extend(context.Background(), b2) + require.NoError(t, err) + + // block 3 + b3 := unittest.BlockWithParentFixture(b2.Header) + b3.SetPayload(flow.EmptyPayload()) + err = state.Extend(context.Background(), b3) + require.NoError(t, err) + + // block 4 (resultB2, resultB3) + b2Receipt := unittest.ReceiptForBlockFixture(b2) + b2Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{vb2.ServiceEvent()} + + b3Receipt := unittest.ReceiptForBlockFixture(b3) + b3Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{vb3.ServiceEvent()} + + b4 := unittest.BlockWithParentFixture(b3.Header) + b4.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{b2Receipt.Meta(), b3Receipt.Meta()}, + Results: []*flow.ExecutionResult{&b2Receipt.ExecutionResult, &b3Receipt.ExecutionResult}, + }) + err = state.Extend(context.Background(), b4) + require.NoError(t, err) + + // block 5 (sealB1) + b1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b1Receipt.ExecutionResult)) + b5 := unittest.BlockWithParentFixture(b4.Header) + b5.SetPayload(flow.Payload{ + Seals: []*flow.Seal{b1Seal}, + }) + err = state.Extend(context.Background(), b5) + require.NoError(t, err) + + // block 6 (sealB2, sealB3) + b2Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b2Receipt.ExecutionResult)) + b3Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b3Receipt.ExecutionResult)) + b6 := unittest.BlockWithParentFixture(b5.Header) + b6.SetPayload(flow.Payload{ + Seals: []*flow.Seal{b2Seal, b3Seal}, + }) + err = state.Extend(context.Background(), b6) + require.NoError(t, err) + + versionBeacons := bstorage.NewVersionBeacons(db) + + // No VB can be found before finalizing anything + vb, err := versionBeacons.Highest(b6.Header.Height) + require.NoError(t, err) + require.Nil(t, vb) + + // finalizing b1 - b5 + err = state.Finalize(context.Background(), b1.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), b2.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), b3.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), b4.ID()) + require.NoError(t, err) + + // No VB can be found after finalizing B4 + vb, err = versionBeacons.Highest(b6.Header.Height) + require.NoError(t, err) + require.Nil(t, vb) + + // once B5 is finalized, B1 and VB1 are sealed, hence index should now find it + err = state.Finalize(context.Background(), b5.ID()) + require.NoError(t, err) + + versionBeacon, err := versionBeacons.Highest(b6.Header.Height) + require.NoError(t, err) + require.Equal(t, + &flow.SealedVersionBeacon{ + VersionBeacon: vb1, + SealHeight: b5.Header.Height, + }, + versionBeacon, + ) + + // finalizing B6 should index events sealed by B6, so VB2 and VB3 + // while we don't expect multiple VBs in one block, we index newest, so last one emitted - VB3 + err = state.Finalize(context.Background(), b6.ID()) + require.NoError(t, err) + + versionBeacon, err = versionBeacons.Highest(b6.Header.Height) + require.NoError(t, err) + require.Equal(t, + &flow.SealedVersionBeacon{ + VersionBeacon: vb3, + SealHeight: b6.Header.Height, + }, + versionBeacon, + ) + }) +} + func TestExtendSealedBoundary(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) { @@ -639,7 +815,20 @@ func TestExtendEpochTransitionValid(t *testing.T) { tracer := trace.NewNoopTracer() log := zerolog.Nop() all := storeutil.StorageLayer(t, db) - protoState, err := protocol.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + protoState, err := protocol.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) receiptValidator := util.MockReceiptValidator() sealValidator := util.MockSealValidator(all.Seals) @@ -1732,7 +1921,20 @@ func TestExtendInvalidSealsInBlock(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) - state, err := protocol.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := protocol.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) head, err := rootSnapshot.Head() @@ -2249,7 +2451,20 @@ func TestHeaderInvalidTimestamp(t *testing.T) { rootSnapshot, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) require.NoError(t, err) - state, err := protocol.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := protocol.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) blockTimer := &mockprotocol.BlockTimer{} diff --git a/state/protocol/badger/params.go b/state/protocol/badger/params.go index 7f19d26234f..52a447f7351 100644 --- a/state/protocol/badger/params.go +++ b/state/protocol/badger/params.go @@ -17,7 +17,7 @@ var _ protocol.Params = (*Params)(nil) func (p Params) ChainID() (flow.ChainID, error) { // retrieve root header - root, err := p.Root() + root, err := p.FinalizedRoot() if err != nil { return "", fmt.Errorf("could not get root: %w", err) } @@ -76,11 +76,29 @@ func (p Params) EpochFallbackTriggered() (bool, error) { return triggered, nil } -func (p Params) Root() (*flow.Header, error) { +func (p Params) FinalizedRoot() (*flow.Header, error) { // look up root block ID var rootID flow.Identifier - err := p.state.db.View(operation.LookupBlockHeight(p.state.rootHeight, &rootID)) + err := p.state.db.View(operation.LookupBlockHeight(p.state.finalizedRootHeight, &rootID)) + if err != nil { + return nil, fmt.Errorf("could not look up root header: %w", err) + } + + // retrieve root header + header, err := p.state.headers.ByBlockID(rootID) + if err != nil { + return nil, fmt.Errorf("could not retrieve root header: %w", err) + } + + return header, nil +} + +func (p Params) SealedRoot() (*flow.Header, error) { + // look up root block ID + var rootID flow.Identifier + err := p.state.db.View(operation.LookupBlockHeight(p.state.sealedRootHeight, &rootID)) + if err != nil { return nil, fmt.Errorf("could not look up root header: %w", err) } @@ -98,7 +116,7 @@ func (p Params) Seal() (*flow.Seal, error) { // look up root header var rootID flow.Identifier - err := p.state.db.View(operation.LookupBlockHeight(p.state.rootHeight, &rootID)) + err := p.state.db.View(operation.LookupBlockHeight(p.state.finalizedRootHeight, &rootID)) if err != nil { return nil, fmt.Errorf("could not look up root header: %w", err) } diff --git a/state/protocol/badger/snapshot.go b/state/protocol/badger/snapshot.go index 03d89d9bbdc..1a121e81748 100644 --- a/state/protocol/badger/snapshot.go +++ b/state/protocol/badger/snapshot.go @@ -8,6 +8,7 @@ import ( "github.com/dgraph-io/badger/v2" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/flow/mapfunc" @@ -16,7 +17,6 @@ import ( "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/inmem" "github.com/onflow/flow-go/state/protocol/invalid" - "github.com/onflow/flow-go/state/protocol/seed" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/operation" "github.com/onflow/flow-go/storage/badger/procedure" @@ -33,7 +33,15 @@ type Snapshot struct { blockID flow.Identifier // reference block for this snapshot } +// FinalizedSnapshot represents a read-only immutable snapshot of the protocol state +// at a finalized block. It is guaranteed to have a header available. +type FinalizedSnapshot struct { + Snapshot + header *flow.Header +} + var _ protocol.Snapshot = (*Snapshot)(nil) +var _ protocol.Snapshot = (*FinalizedSnapshot)(nil) // newSnapshotWithIncorporatedReferenceBlock creates a new state snapshot with the given reference block. // CAUTION: The caller is responsible for ensuring that the reference block has been incorporated. @@ -44,6 +52,22 @@ func newSnapshotWithIncorporatedReferenceBlock(state *State, blockID flow.Identi } } +// NewFinalizedSnapshot instantiates a `FinalizedSnapshot`. +// CAUTION: the header's ID _must_ match `blockID` (not checked) +func NewFinalizedSnapshot(state *State, blockID flow.Identifier, header *flow.Header) *FinalizedSnapshot { + return &FinalizedSnapshot{ + Snapshot: Snapshot{ + state: state, + blockID: blockID, + }, + header: header, + } +} + +func (s *FinalizedSnapshot) Head() (*flow.Header, error) { + return s.header, nil +} + func (s *Snapshot) Head() (*flow.Header, error) { head, err := s.state.headers.ByBlockID(s.blockID) return head, err @@ -215,7 +239,7 @@ func (s *Snapshot) SealingSegment() (*flow.SealingSegment, error) { if err != nil { return nil, fmt.Errorf("could not get snapshot's reference block: %w", err) } - if head.Header.Height < s.state.rootHeight { + if head.Header.Height < s.state.finalizedRootHeight { return nil, protocol.ErrSealingSegmentBelowRootBlock } @@ -353,7 +377,7 @@ func (s *Snapshot) descendants(blockID flow.Identifier) ([]flow.Identifier, erro return descendantIDs, nil } -// RandomSource returns the seed for the current block snapshot. +// RandomSource returns the seed for the current block's snapshot. // Expected error returns: // * storage.ErrNotFound is returned if the QC is unknown. func (s *Snapshot) RandomSource() ([]byte, error) { @@ -361,7 +385,7 @@ func (s *Snapshot) RandomSource() ([]byte, error) { if err != nil { return nil, err } - randomSource, err := seed.FromParentQCSignature(qc.SigData) + randomSource, err := model.BeaconSignature(qc) if err != nil { return nil, fmt.Errorf("could not create seed from QC's signature: %w", err) } @@ -378,6 +402,15 @@ func (s *Snapshot) Params() protocol.GlobalParams { return s.state.Params() } +func (s *Snapshot) VersionBeacon() (*flow.SealedVersionBeacon, error) { + head, err := s.state.headers.ByBlockID(s.blockID) + if err != nil { + return nil, err + } + + return s.state.versionBeacons.Highest(head.Height) +} + // EpochQuery encapsulates querying epochs w.r.t. a snapshot. type EpochQuery struct { snap *Snapshot diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index 93c72cbeb9e..03e98d6f067 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -7,7 +7,6 @@ import ( "errors" "math/rand" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -21,16 +20,12 @@ import ( "github.com/onflow/flow-go/state/protocol" bprotocol "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/inmem" - "github.com/onflow/flow-go/state/protocol/seed" + "github.com/onflow/flow-go/state/protocol/prg" "github.com/onflow/flow-go/state/protocol/util" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/unittest" ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - // TestUnknownReferenceBlock tests queries for snapshots which should be unknown. // We use this fixture: // - Root height: 100 @@ -195,16 +190,18 @@ func TestIdentities(t *testing.T) { }) t.Run("single identity", func(t *testing.T) { - expected := identities.Sample(1)[0] + expected := identities[rand.Intn(len(identities))] actual, err := state.Final().Identity(expected.NodeID) require.NoError(t, err) assert.Equal(t, expected, actual) }) t.Run("filtered", func(t *testing.T) { + sample, err := identities.SamplePct(0.1) + require.NoError(t, err) filters := []flow.IdentityFilter{ filter.HasRole(flow.RoleCollection), - filter.HasNodeID(identities.SamplePct(0.1).NodeIDs()...), + filter.HasNodeID(sample.NodeIDs()...), filter.HasWeight(true), } @@ -693,7 +690,7 @@ func TestSealingSegment_FailureCases(t *testing.T) { // Thereby, the state should have b3 as its local root block. In addition, the blocks contained in the sealing // segment, such as b2 should be stored in the state. util.RunWithFollowerProtocolState(t, multipleBlockSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { - localStateRootBlock, err := state.Params().Root() + localStateRootBlock, err := state.Params().FinalizedRoot() require.NoError(t, err) assert.Equal(t, b3.ID(), localStateRootBlock.ID()) @@ -940,7 +937,7 @@ func TestQuorumCertificate(t *testing.T) { assert.NoError(t, err) randomSeed, err := state.AtBlockID(head.ID()).RandomSource() assert.NoError(t, err) - assert.Equal(t, len(randomSeed), seed.RandomSourceLength) + assert.Equal(t, len(randomSeed), prg.RandomSourceLength) }) }) @@ -1246,7 +1243,7 @@ func TestSnapshot_CrossEpochIdentities(t *testing.T) { // 1 identity added at epoch 2 that was not present in epoch 1 addedAtEpoch2 := unittest.IdentityFixture() // 1 identity removed in epoch 2 that was present in epoch 1 - removedAtEpoch2 := epoch1Identities.Sample(1)[0] + removedAtEpoch2 := epoch1Identities[rand.Intn(len(epoch1Identities))] // epoch 2 has partial overlap with epoch 1 epoch2Identities := append( epoch1Identities.Filter(filter.Not(filter.HasNodeID(removedAtEpoch2.NodeID))), @@ -1276,7 +1273,7 @@ func TestSnapshot_CrossEpochIdentities(t *testing.T) { require.True(t, ok) t.Run("should be able to query at root block", func(t *testing.T) { - root, err := state.Params().Root() + root, err := state.Params().FinalizedRoot() require.NoError(t, err) snapshot := state.AtHeight(root.Height) identities, err := snapshot.Identities(filter.Any) diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index 3ec2e39ec16..40973dc05f2 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -5,6 +5,7 @@ package badger import ( "errors" "fmt" + "sync/atomic" "github.com/dgraph-io/badger/v2" @@ -19,6 +20,12 @@ import ( "github.com/onflow/flow-go/storage/badger/transaction" ) +// cachedHeader caches a block header and its ID. +type cachedHeader struct { + id flow.Identifier + header *flow.Header +} + type State struct { metrics module.ComplianceMetrics db *badger.DB @@ -32,17 +39,31 @@ type State struct { commits storage.EpochCommits statuses storage.EpochStatuses } - // cache the root height because it cannot change over the lifecycle of a protocol state instance - rootHeight uint64 - // cache the spork root block height because it cannot change over the lifecycle of a protocol state instance + versionBeacons storage.VersionBeacons + + // rootHeight marks the cutoff of the history this node knows about. We cache it in the state + // because it cannot change over the lifecycle of a protocol state instance. It is frequently + // larger than the height of the root block of the spork, (also cached below as + // `sporkRootBlockHeight`), for instance if the node joined in an epoch after the last spork. + finalizedRootHeight uint64 + // sealedRootHeight returns the root block that is sealed. + sealedRootHeight uint64 + // sporkRootBlockHeight is the height of the root block in the current spork. We cache it in + // the state, because it cannot change over the lifecycle of a protocol state instance. + // Caution: A node that joined in a later epoch past the spork, the node will likely _not_ + // know the spork's root block in full (though it will always know the height). sporkRootBlockHeight uint64 + // cache the latest finalized and sealed block headers as these are common queries. + // It can be cached because the protocol state is solely responsible for updating these values. + cachedFinal *atomic.Pointer[cachedHeader] + cachedSealed *atomic.Pointer[cachedHeader] } var _ protocol.State = (*State)(nil) type BootstrapConfig struct { - // SkipNetworkAddressValidation flags allows skipping all the network address related validations not needed for - // an unstaked node + // SkipNetworkAddressValidation flags allows skipping all the network address related + // validations not needed for an unstaked node SkipNetworkAddressValidation bool } @@ -69,6 +90,7 @@ func Bootstrap( setups storage.EpochSetups, commits storage.EpochCommits, statuses storage.EpochStatuses, + versionBeacons storage.VersionBeacons, root protocol.Snapshot, options ...BootstrapConfigOptions, ) (*State, error) { @@ -86,7 +108,19 @@ func Bootstrap( return nil, fmt.Errorf("expected empty database") } - state := newState(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses) + state := newState( + metrics, + db, + headers, + seals, + results, + blocks, + qcs, + setups, + commits, + statuses, + versionBeacons, + ) if err := IsValidRootSnapshot(root, !config.SkipNetworkAddressValidation); err != nil { return nil, fmt.Errorf("cannot bootstrap invalid root snapshot: %w", err) @@ -97,15 +131,22 @@ func Bootstrap( return nil, fmt.Errorf("could not get sealing segment: %w", err) } + _, rootSeal, err := root.SealedResult() + if err != nil { + return nil, fmt.Errorf("could not get sealed result for sealing segment: %w", err) + } + err = operation.RetryOnConflictTx(db, transaction.Update, func(tx *transaction.Tx) error { // sealing segment is in ascending height order, so the tail is the // oldest ancestor and head is the newest child in the segment // TAIL <- ... <- HEAD - highest := segment.Highest() // reference block of the snapshot - lowest := segment.Sealed() // last sealed block + lastFinalized := segment.Finalized() // the highest block in sealing segment is the last finalized block + lastSealed := segment.Sealed() // the lowest block in sealing segment is the last sealed block // 1) bootstrap the sealing segment - err = state.bootstrapSealingSegment(segment, highest)(tx) + // creating sealed root block with the rootResult + // creating finalized root block with lastFinalized + err = state.bootstrapSealingSegment(segment, lastFinalized, rootSeal)(tx) if err != nil { return fmt.Errorf("could not bootstrap sealing chain segment blocks: %w", err) } @@ -143,13 +184,19 @@ func Bootstrap( if err != nil { return fmt.Errorf("could not update epoch metrics: %w", err) } - state.metrics.BlockSealed(lowest) - state.metrics.SealedHeight(lowest.Header.Height) - state.metrics.FinalizedHeight(highest.Header.Height) + state.metrics.BlockSealed(lastSealed) + state.metrics.SealedHeight(lastSealed.Header.Height) + state.metrics.FinalizedHeight(lastFinalized.Header.Height) for _, block := range segment.Blocks { state.metrics.BlockFinalized(block) } + // 7) initialize version beacon + err = transaction.WithTx(state.boostrapVersionBeacon(root))(tx) + if err != nil { + return fmt.Errorf("could not bootstrap version beacon: %w", err) + } + return nil }) if err != nil { @@ -167,7 +214,7 @@ func Bootstrap( // bootstrapSealingSegment inserts all blocks and associated metadata for the // protocol state root snapshot to disk. -func (state *State) bootstrapSealingSegment(segment *flow.SealingSegment, head *flow.Block) func(tx *transaction.Tx) error { +func (state *State) bootstrapSealingSegment(segment *flow.SealingSegment, head *flow.Block, rootSeal *flow.Seal) func(tx *transaction.Tx) error { return func(tx *transaction.Tx) error { for _, result := range segment.ExecutionResults { @@ -189,6 +236,15 @@ func (state *State) bootstrapSealingSegment(segment *flow.SealingSegment, head * } } + // root seal contains the result ID for the sealed root block. If the sealed root block is + // different from the finalized root block, then it means the node dynamically bootstrapped. + // In that case, we should index the result of the sealed root block so that the EN is able + // to execute the next block. + err := transaction.WithTx(operation.SkipDuplicates(operation.IndexExecutionResult(rootSeal.BlockID, rootSeal.ResultID)))(tx) + if err != nil { + return fmt.Errorf("could not index root result: %w", err) + } + for _, block := range segment.ExtraBlocks { blockID := block.ID() height := block.Header.Height @@ -249,7 +305,7 @@ func (state *State) bootstrapSealingSegment(segment *flow.SealingSegment, head * } // insert an empty child index for the final block in the segment - err := transaction.WithTx(operation.InsertBlockChildren(head.ID(), nil))(tx) + err = transaction.WithTx(operation.InsertBlockChildren(head.ID(), nil))(tx) if err != nil { return fmt.Errorf("could not insert child index for head block (id=%x): %w", head.ID(), err) } @@ -266,7 +322,7 @@ func (state *State) bootstrapStatePointers(root protocol.Snapshot) func(*badger. if err != nil { return fmt.Errorf("could not get sealing segment: %w", err) } - highest := segment.Highest() + highest := segment.Finalized() lowest := segment.Sealed() // find the finalized seal that seals the lowest block, meaning seal.BlockID == lowest.ID() seal, err := segment.FinalizedSeal() @@ -318,7 +374,12 @@ func (state *State) bootstrapStatePointers(root protocol.Snapshot) func(*badger. // insert height pointers err = operation.InsertRootHeight(highest.Header.Height)(tx) if err != nil { - return fmt.Errorf("could not insert root height: %w", err) + return fmt.Errorf("could not insert finalized root height: %w", err) + } + // the sealed root height is the lowest block in sealing segment + err = operation.InsertSealedRootHeight(lowest.Header.Height)(tx) + if err != nil { + return fmt.Errorf("could not insert sealed root height: %w", err) } err = operation.InsertFinalizedHeight(highest.Header.Height)(tx) if err != nil { @@ -555,6 +616,7 @@ func OpenState( setups storage.EpochSetups, commits storage.EpochCommits, statuses storage.EpochStatuses, + versionBeacons storage.VersionBeacons, ) (*State, error) { isBootstrapped, err := IsBootstrapped(db) if err != nil { @@ -563,7 +625,23 @@ func OpenState( if !isBootstrapped { return nil, fmt.Errorf("expected database to contain bootstrapped state") } - state := newState(metrics, db, headers, seals, results, blocks, qcs, setups, commits, statuses) + state := newState( + metrics, + db, + headers, + seals, + results, + blocks, + qcs, + setups, + commits, + statuses, + versionBeacons, + ) // populate the protocol state cache + err = state.populateCache() + if err != nil { + return nil, fmt.Errorf("failed to populate cache: %w", err) + } // report last finalized and sealed block height finalSnapshot := state.Final() @@ -584,11 +662,6 @@ func OpenState( if err != nil { return nil, fmt.Errorf("failed to update epoch metrics: %w", err) } - // populate the protocol state cache - err = state.populateCache() - if err != nil { - return nil, fmt.Errorf("failed to populate cache: %w", err) - } return state, nil } @@ -600,27 +673,21 @@ func (state *State) Params() protocol.Params { // Sealed returns a snapshot for the latest sealed block. A latest sealed block // must always exist, so this function always returns a valid snapshot. func (state *State) Sealed() protocol.Snapshot { - // retrieve the latest sealed height - var sealed uint64 - err := state.db.View(operation.RetrieveSealedHeight(&sealed)) - if err != nil { - // sealed height must always be set, all errors here are critical - return invalid.NewSnapshotf("could not retrieve sealed height: %w", err) + cached := state.cachedSealed.Load() + if cached == nil { + return invalid.NewSnapshotf("internal inconsistency: no cached sealed header") } - return state.AtHeight(sealed) + return NewFinalizedSnapshot(state, cached.id, cached.header) } // Final returns a snapshot for the latest finalized block. A latest finalized // block must always exist, so this function always returns a valid snapshot. func (state *State) Final() protocol.Snapshot { - // retrieve the latest finalized height - var finalized uint64 - err := state.db.View(operation.RetrieveFinalizedHeight(&finalized)) - if err != nil { - // finalized height must always be set, so all errors here are critical - return invalid.NewSnapshot(fmt.Errorf("could not retrieve finalized height: %w", err)) + cached := state.cachedFinal.Load() + if cached == nil { + return invalid.NewSnapshotf("internal inconsistency: no cached final header") } - return state.AtHeight(finalized) + return NewFinalizedSnapshot(state, cached.id, cached.header) } // AtHeight returns a snapshot for the finalized block at the given height. @@ -675,6 +742,7 @@ func newState( setups storage.EpochSetups, commits storage.EpochCommits, statuses storage.EpochStatuses, + versionBeacons storage.VersionBeacons, ) *State { return &State{ metrics: metrics, @@ -693,6 +761,9 @@ func newState( commits: commits, statuses: statuses, }, + versionBeacons: versionBeacons, + cachedFinal: new(atomic.Pointer[cachedHeader]), + cachedSealed: new(atomic.Pointer[cachedHeader]), } } @@ -761,20 +832,85 @@ func (state *State) updateEpochMetrics(snap protocol.Snapshot) error { return nil } +// boostrapVersionBeacon bootstraps version beacon, by adding the latest beacon +// to an index, if present. +func (state *State) boostrapVersionBeacon( + snapshot protocol.Snapshot, +) func(*badger.Txn) error { + return func(txn *badger.Txn) error { + versionBeacon, err := snapshot.VersionBeacon() + if err != nil { + return err + } + + if versionBeacon == nil { + return nil + } + + return operation.IndexVersionBeaconByHeight(versionBeacon)(txn) + } +} + // populateCache is used after opening or bootstrapping the state to populate the cache. +// The cache must be populated before the State receives any queries. +// No errors expected during normal operations. func (state *State) populateCache() error { - var rootHeight uint64 - err := state.db.View(operation.RetrieveRootHeight(&rootHeight)) - if err != nil { - return fmt.Errorf("could not read root block to populate cache: %w", err) - } - state.rootHeight = rootHeight - sporkRootBlockHeight, err := state.Params().SporkRootBlockHeight() + // cache the initial value for finalized block + err := state.db.View(func(tx *badger.Txn) error { + // root height + err := state.db.View(operation.RetrieveRootHeight(&state.finalizedRootHeight)) + if err != nil { + return fmt.Errorf("could not read root block to populate cache: %w", err) + } + // sealed root height + err = state.db.View(operation.RetrieveSealedRootHeight(&state.sealedRootHeight)) + if err != nil { + return fmt.Errorf("could not read sealed root block to populate cache: %w", err) + } + // spork root block height + err = state.db.View(operation.RetrieveSporkRootBlockHeight(&state.sporkRootBlockHeight)) + if err != nil { + return fmt.Errorf("could not get spork root block height: %w", err) + } + // finalized header + var finalizedHeight uint64 + err = operation.RetrieveFinalizedHeight(&finalizedHeight)(tx) + if err != nil { + return fmt.Errorf("could not lookup finalized height: %w", err) + } + var cachedFinalHeader cachedHeader + err = operation.LookupBlockHeight(finalizedHeight, &cachedFinalHeader.id)(tx) + if err != nil { + return fmt.Errorf("could not lookup finalized id (height=%d): %w", finalizedHeight, err) + } + cachedFinalHeader.header, err = state.headers.ByBlockID(cachedFinalHeader.id) + if err != nil { + return fmt.Errorf("could not get finalized block (id=%x): %w", cachedFinalHeader.id, err) + } + state.cachedFinal.Store(&cachedFinalHeader) + // sealed header + var sealedHeight uint64 + err = operation.RetrieveSealedHeight(&sealedHeight)(tx) + if err != nil { + return fmt.Errorf("could not lookup sealed height: %w", err) + } + var cachedSealedHeader cachedHeader + err = operation.LookupBlockHeight(sealedHeight, &cachedSealedHeader.id)(tx) + if err != nil { + return fmt.Errorf("could not lookup sealed id (height=%d): %w", sealedHeight, err) + } + cachedSealedHeader.header, err = state.headers.ByBlockID(cachedSealedHeader.id) + if err != nil { + return fmt.Errorf("could not get sealed block (id=%x): %w", cachedSealedHeader.id, err) + } + state.cachedSealed.Store(&cachedSealedHeader) + return nil + }) if err != nil { - return fmt.Errorf("could not read spork root block height: %w", err) + return fmt.Errorf("could not cache finalized header: %w", err) } - state.sporkRootBlockHeight = sporkRootBlockHeight + return nil } diff --git a/state/protocol/badger/state_test.go b/state/protocol/badger/state_test.go index 66de7d3033f..c6bcc59854f 100644 --- a/state/protocol/badger/state_test.go +++ b/state/protocol/badger/state_test.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -74,12 +73,17 @@ func TestBootstrapAndOpen(t *testing.T) { all.Setups, all.EpochCommits, all.Statuses, + all.VersionBeacons, ) require.NoError(t, err) complianceMetrics.AssertExpectations(t) unittest.AssertSnapshotsEqual(t, rootSnapshot, state.Final()) + + vb, err := state.Final().VersionBeacon() + require.NoError(t, err) + require.Nil(t, vb) }) } @@ -154,6 +158,7 @@ func TestBootstrapAndOpen_EpochCommitted(t *testing.T) { all.Setups, all.EpochCommits, all.Statuses, + all.VersionBeacons, ) require.NoError(t, err) @@ -420,7 +425,9 @@ func TestBootstrap_InvalidIdentities(t *testing.T) { root := unittest.RootSnapshotFixture(participants) // randomly shuffle the identities so they are not canonically ordered encodable := root.Encodable() - encodable.Identities = participants.DeterministicShuffle(time.Now().UnixNano()) + var err error + encodable.Identities, err = participants.Shuffle() + require.NoError(t, err) root = inmem.SnapshotFromEncodable(encodable) bootstrap(t, root, func(state *bprotocol.State, err error) { assert.Error(t, err) @@ -524,7 +531,20 @@ func bootstrap(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotocol.S db := unittest.BadgerDB(t, dir) defer db.Close() all := storutil.StorageLayer(t, db) - state, err := bprotocol.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := bprotocol.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) f(state, err) } @@ -565,7 +585,7 @@ func assertSealingSegmentBlocksQueryableAfterBootstrap(t *testing.T, snapshot pr segment, err := state.Final().SealingSegment() require.NoError(t, err) - rootBlock, err := state.Params().Root() + rootBlock, err := state.Params().FinalizedRoot() require.NoError(t, err) // root block should be the highest block from the sealing segment diff --git a/state/protocol/badger/validity.go b/state/protocol/badger/validity.go index 04379abbc29..264831512ec 100644 --- a/state/protocol/badger/validity.go +++ b/state/protocol/badger/validity.go @@ -265,6 +265,11 @@ func IsValidRootSnapshot(snap protocol.Snapshot, verifyResultID bool) error { return fmt.Errorf("final view of epoch less than first block view") } + err = validateVersionBeacon(snap) + if err != nil { + return err + } + return nil } @@ -343,6 +348,40 @@ func validateClusterQC(cluster protocol.Cluster) error { return nil } +// validateVersionBeacon returns an InvalidServiceEventError if the snapshot +// version beacon is invalid +func validateVersionBeacon(snap protocol.Snapshot) error { + errf := func(msg string, args ...any) error { + return protocol.NewInvalidServiceEventErrorf(msg, args) + } + + versionBeacon, err := snap.VersionBeacon() + if err != nil { + return errf("could not get version beacon: %w", err) + } + + if versionBeacon == nil { + return nil + } + + head, err := snap.Head() + if err != nil { + return errf("could not get snapshot head: %w", err) + } + + // version beacon must be included in a past block to be effective + if versionBeacon.SealHeight > head.Height { + return errf("version table height higher than highest height") + } + + err = versionBeacon.Validate() + if err != nil { + return errf("version beacon is invalid: %w", err) + } + + return nil +} + // ValidRootSnapshotContainsEntityExpiryRange performs a sanity check to make sure the // root snapshot has enough history to encompass at least one full entity expiry window. // Entities (in particular transactions and collections) may reference a block within diff --git a/state/protocol/badger/validity_test.go b/state/protocol/badger/validity_test.go index 2c0e3372e4b..30ee94c40d6 100644 --- a/state/protocol/badger/validity_test.go +++ b/state/protocol/badger/validity_test.go @@ -2,13 +2,14 @@ package badger import ( "testing" - "time" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -29,9 +30,10 @@ func TestEpochSetupValidity(t *testing.T) { _, result, _ := unittest.BootstrapFixture(participants) setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) // randomly shuffle the identities so they are not canonically ordered - setup.Participants = setup.Participants.DeterministicShuffle(time.Now().UnixNano()) - - err := verifyEpochSetup(setup, true) + var err error + setup.Participants, err = setup.Participants.Shuffle() + require.NoError(t, err) + err = verifyEpochSetup(setup, true) require.Error(t, err) }) @@ -145,3 +147,88 @@ func TestEntityExpirySnapshotValidation(t *testing.T) { require.NoError(t, err) }) } + +func TestValidateVersionBeacon(t *testing.T) { + t.Run("no version beacon is ok", func(t *testing.T) { + snap := new(mock.Snapshot) + + snap.On("VersionBeacon").Return(nil, nil) + + err := validateVersionBeacon(snap) + require.NoError(t, err) + }) + t.Run("valid version beacon is ok", func(t *testing.T) { + snap := new(mock.Snapshot) + block := unittest.BlockFixture() + block.Header.Height = 100 + + vb := &flow.SealedVersionBeacon{ + VersionBeacon: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + { + BlockHeight: 1000, + Version: "1.0.0", + }, + }, + Sequence: 50, + }, + SealHeight: uint64(37), + } + + snap.On("Head").Return(block.Header, nil) + snap.On("VersionBeacon").Return(vb, nil) + + err := validateVersionBeacon(snap) + require.NoError(t, err) + }) + t.Run("height must be below highest block", func(t *testing.T) { + snap := new(mock.Snapshot) + block := unittest.BlockFixture() + block.Header.Height = 12 + + vb := &flow.SealedVersionBeacon{ + VersionBeacon: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + { + BlockHeight: 1000, + Version: "1.0.0", + }, + }, + Sequence: 50, + }, + SealHeight: uint64(37), + } + + snap.On("Head").Return(block.Header, nil) + snap.On("VersionBeacon").Return(vb, nil) + + err := validateVersionBeacon(snap) + require.Error(t, err) + require.True(t, protocol.IsInvalidServiceEventError(err)) + }) + t.Run("version beacon must be valid", func(t *testing.T) { + snap := new(mock.Snapshot) + block := unittest.BlockFixture() + block.Header.Height = 12 + + vb := &flow.SealedVersionBeacon{ + VersionBeacon: &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + { + BlockHeight: 0, + Version: "asdf", // invalid semver - hence will be considered invalid + }, + }, + Sequence: 50, + }, + SealHeight: uint64(1), + } + + snap.On("Head").Return(block.Header, nil) + snap.On("VersionBeacon").Return(vb, nil) + + err := validateVersionBeacon(snap) + require.Error(t, err) + require.True(t, protocol.IsInvalidServiceEventError(err)) + }) +} diff --git a/state/protocol/events.go b/state/protocol/events.go index 08608d0ffd3..e97c4f7c84c 100644 --- a/state/protocol/events.go +++ b/state/protocol/events.go @@ -29,7 +29,6 @@ import ( // NOTE: the epoch-related callbacks are only called once the fork containing // the relevant event has been finalized. type Consumer interface { - // BlockFinalized is called when a block is finalized. // Formally, this callback is informationally idempotent. I.e. the consumer // of this callback must handle repeated calls for the same block. diff --git a/state/protocol/events/noop.go b/state/protocol/events/noop.go index 1925a5e4776..4eb46885787 100644 --- a/state/protocol/events/noop.go +++ b/state/protocol/events/noop.go @@ -14,14 +14,16 @@ func NewNoop() *Noop { return &Noop{} } -func (n Noop) BlockFinalized(block *flow.Header) {} +func (n Noop) BlockFinalized(*flow.Header) {} -func (n Noop) BlockProcessable(block *flow.Header, certifyingQC *flow.QuorumCertificate) {} +func (n Noop) BlockProcessable(*flow.Header, *flow.QuorumCertificate) {} -func (n Noop) EpochTransition(newEpoch uint64, first *flow.Header) {} +func (n Noop) EpochTransition(uint64, *flow.Header) {} -func (n Noop) EpochSetupPhaseStarted(epoch uint64, first *flow.Header) {} +func (n Noop) EpochSetupPhaseStarted(uint64, *flow.Header) {} -func (n Noop) EpochCommittedPhaseStarted(epoch uint64, first *flow.Header) {} +func (n Noop) EpochCommittedPhaseStarted(uint64, *flow.Header) {} func (n Noop) EpochEmergencyFallbackTriggered() {} + +func (n Noop) ActiveClustersChanged(flow.ChainIDList) {} diff --git a/state/protocol/inmem/convert.go b/state/protocol/inmem/convert.go index 411f6aae7df..5a1150c2992 100644 --- a/state/protocol/inmem/convert.go +++ b/state/protocol/inmem/convert.go @@ -82,6 +82,14 @@ func FromSnapshot(from protocol.Snapshot) (*Snapshot, error) { } snap.Params = params.enc + // convert version beacon + versionBeacon, err := from.VersionBeacon() + if err != nil { + return nil, fmt.Errorf("could not get version beacon: %w", err) + } + + snap.SealedVersionBeacon = versionBeacon + return &Snapshot{snap}, nil } @@ -330,10 +338,11 @@ func SnapshotFromBootstrapStateWithParams( FirstSeal: seal, ExtraBlocks: make([]*flow.Block, 0), }, - QuorumCertificate: qc, - Phase: flow.EpochPhaseStaking, - Epochs: epochs, - Params: params, + QuorumCertificate: qc, + Phase: flow.EpochPhaseStaking, + Epochs: epochs, + Params: params, + SealedVersionBeacon: nil, }) return snap, nil } diff --git a/state/protocol/inmem/convert_test.go b/state/protocol/inmem/convert_test.go index 72047ac2efc..6da32088947 100644 --- a/state/protocol/inmem/convert_test.go +++ b/state/protocol/inmem/convert_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" bprotocol "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/inmem" @@ -40,9 +41,9 @@ func TestFromSnapshot(t *testing.T) { epoch2, ok := epochBuilder.EpochHeights(2) require.True(t, ok) - // test that we are able retrieve an in-memory version of root snapshot + // test that we are able to retrieve an in-memory version of root snapshot t.Run("root snapshot", func(t *testing.T) { - root, err := state.Params().Root() + root, err := state.Params().FinalizedRoot() require.NoError(t, err) expected := state.AtHeight(root.Height) actual, err := inmem.FromSnapshot(expected) @@ -100,6 +101,36 @@ func TestFromSnapshot(t *testing.T) { testEncodeDecode(t, actual) }) }) + + // ensure last version beacon is included + t.Run("version beacon", func(t *testing.T) { + + expectedVB := &flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + BlockHeight: 1012, + Version: "1.2.3", + }), + ), + } + unittest.AddVersionBeacon(t, expectedVB.VersionBeacon, state) + + expected := state.Final() + head, err := expected.Head() + require.NoError(t, err) + + expectedVB.SealHeight = head.Height + + actual, err := inmem.FromSnapshot(expected) + require.NoError(t, err) + assertSnapshotsEqual(t, expected, actual) + testEncodeDecode(t, actual) + + actualVB, err := actual.VersionBeacon() + require.NoError(t, err) + require.Equal(t, expectedVB, actualVB) + }) }) } diff --git a/state/protocol/inmem/encodable.go b/state/protocol/inmem/encodable.go index 4601ec36578..4ab60a6aefe 100644 --- a/state/protocol/inmem/encodable.go +++ b/state/protocol/inmem/encodable.go @@ -8,15 +8,16 @@ import ( // EncodableSnapshot is the encoding format for protocol.Snapshot type EncodableSnapshot struct { - Head *flow.Header - Identities flow.IdentityList - LatestSeal *flow.Seal - LatestResult *flow.ExecutionResult - SealingSegment *flow.SealingSegment - QuorumCertificate *flow.QuorumCertificate - Phase flow.EpochPhase - Epochs EncodableEpochs - Params EncodableParams + Head *flow.Header + Identities flow.IdentityList + LatestSeal *flow.Seal + LatestResult *flow.ExecutionResult + SealingSegment *flow.SealingSegment + QuorumCertificate *flow.QuorumCertificate + Phase flow.EpochPhase + Epochs EncodableEpochs + Params EncodableParams + SealedVersionBeacon *flow.SealedVersionBeacon } // EncodableEpochs is the encoding format for protocol.EpochQuery diff --git a/state/protocol/inmem/snapshot.go b/state/protocol/inmem/snapshot.go index 228c319aa91..a30c1b0fcad 100644 --- a/state/protocol/inmem/snapshot.go +++ b/state/protocol/inmem/snapshot.go @@ -1,9 +1,9 @@ package inmem import ( + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/state/protocol/seed" ) // Snapshot is a memory-backed implementation of protocol.Snapshot. The snapshot @@ -57,7 +57,7 @@ func (s Snapshot) Phase() (flow.EpochPhase, error) { } func (s Snapshot) RandomSource() ([]byte, error) { - return seed.FromParentQCSignature(s.enc.QuorumCertificate.SigData) + return model.BeaconSignature(s.enc.QuorumCertificate) } func (s Snapshot) Epochs() protocol.EpochQuery { @@ -72,6 +72,10 @@ func (s Snapshot) Encodable() EncodableSnapshot { return s.enc } +func (s Snapshot) VersionBeacon() (*flow.SealedVersionBeacon, error) { + return s.enc.SealedVersionBeacon, nil +} + func SnapshotFromEncodable(enc EncodableSnapshot) *Snapshot { return &Snapshot{ enc: enc, diff --git a/state/protocol/invalid/snapshot.go b/state/protocol/invalid/snapshot.go index ab54103c191..78ee386ebcb 100644 --- a/state/protocol/invalid/snapshot.go +++ b/state/protocol/invalid/snapshot.go @@ -75,3 +75,7 @@ func (u *Snapshot) RandomSource() ([]byte, error) { func (u *Snapshot) Params() protocol.GlobalParams { return Params{u.err} } + +func (u *Snapshot) VersionBeacon() (*flow.SealedVersionBeacon, error) { + return nil, u.err +} diff --git a/state/protocol/mock/cluster_events.go b/state/protocol/mock/cluster_events.go new file mode 100644 index 00000000000..a17e4db4a9a --- /dev/null +++ b/state/protocol/mock/cluster_events.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// ClusterEvents is an autogenerated mock type for the ClusterEvents type +type ClusterEvents struct { + mock.Mock +} + +// ActiveClustersChanged provides a mock function with given fields: _a0 +func (_m *ClusterEvents) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewClusterEvents interface { + mock.TestingT + Cleanup(func()) +} + +// NewClusterEvents creates a new instance of ClusterEvents. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewClusterEvents(t mockConstructorTestingTNewClusterEvents) *ClusterEvents { + mock := &ClusterEvents{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/state/protocol/mock/cluster_id_update_consumer.go b/state/protocol/mock/cluster_id_update_consumer.go new file mode 100644 index 00000000000..3594d504c0f --- /dev/null +++ b/state/protocol/mock/cluster_id_update_consumer.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// ClusterIDUpdateConsumer is an autogenerated mock type for the ClusterIDUpdateConsumer type +type ClusterIDUpdateConsumer struct { + mock.Mock +} + +// ClusterIdsUpdated provides a mock function with given fields: _a0 +func (_m *ClusterIDUpdateConsumer) ActiveClustersChanged(_a0 flow.ChainIDList) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewClusterIDUpdateConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewClusterIDUpdateConsumer creates a new instance of ClusterIDUpdateConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewClusterIDUpdateConsumer(t mockConstructorTestingTNewClusterIDUpdateConsumer) *ClusterIDUpdateConsumer { + mock := &ClusterIDUpdateConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/state/protocol/mock/instance_params.go b/state/protocol/mock/instance_params.go index fb428410d19..4398e7fa5b4 100644 --- a/state/protocol/mock/instance_params.go +++ b/state/protocol/mock/instance_params.go @@ -36,8 +36,8 @@ func (_m *InstanceParams) EpochFallbackTriggered() (bool, error) { return r0, r1 } -// Root provides a mock function with given fields: -func (_m *InstanceParams) Root() (*flow.Header, error) { +// FinalizedRoot provides a mock function with given fields: +func (_m *InstanceParams) FinalizedRoot() (*flow.Header, error) { ret := _m.Called() var r0 *flow.Header @@ -88,6 +88,32 @@ func (_m *InstanceParams) Seal() (*flow.Seal, error) { return r0, r1 } +// SealedRoot provides a mock function with given fields: +func (_m *InstanceParams) SealedRoot() (*flow.Header, error) { + ret := _m.Called() + + var r0 *flow.Header + var r1 error + if rf, ok := ret.Get(0).(func() (*flow.Header, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *flow.Header); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Header) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + type mockConstructorTestingTNewInstanceParams interface { mock.TestingT Cleanup(func()) diff --git a/state/protocol/mock/params.go b/state/protocol/mock/params.go index 6940960ba4b..a6000f165e5 100644 --- a/state/protocol/mock/params.go +++ b/state/protocol/mock/params.go @@ -84,6 +84,32 @@ func (_m *Params) EpochFallbackTriggered() (bool, error) { return r0, r1 } +// FinalizedRoot provides a mock function with given fields: +func (_m *Params) FinalizedRoot() (*flow.Header, error) { + ret := _m.Called() + + var r0 *flow.Header + var r1 error + if rf, ok := ret.Get(0).(func() (*flow.Header, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *flow.Header); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Header) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ProtocolVersion provides a mock function with given fields: func (_m *Params) ProtocolVersion() (uint, error) { ret := _m.Called() @@ -108,20 +134,20 @@ func (_m *Params) ProtocolVersion() (uint, error) { return r0, r1 } -// Root provides a mock function with given fields: -func (_m *Params) Root() (*flow.Header, error) { +// Seal provides a mock function with given fields: +func (_m *Params) Seal() (*flow.Seal, error) { ret := _m.Called() - var r0 *flow.Header + var r0 *flow.Seal var r1 error - if rf, ok := ret.Get(0).(func() (*flow.Header, error)); ok { + if rf, ok := ret.Get(0).(func() (*flow.Seal, error)); ok { return rf() } - if rf, ok := ret.Get(0).(func() *flow.Header); ok { + if rf, ok := ret.Get(0).(func() *flow.Seal); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.Header) + r0 = ret.Get(0).(*flow.Seal) } } @@ -134,20 +160,20 @@ func (_m *Params) Root() (*flow.Header, error) { return r0, r1 } -// Seal provides a mock function with given fields: -func (_m *Params) Seal() (*flow.Seal, error) { +// SealedRoot provides a mock function with given fields: +func (_m *Params) SealedRoot() (*flow.Header, error) { ret := _m.Called() - var r0 *flow.Seal + var r0 *flow.Header var r1 error - if rf, ok := ret.Get(0).(func() (*flow.Seal, error)); ok { + if rf, ok := ret.Get(0).(func() (*flow.Header, error)); ok { return rf() } - if rf, ok := ret.Get(0).(func() *flow.Seal); ok { + if rf, ok := ret.Get(0).(func() *flow.Header); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.Seal) + r0 = ret.Get(0).(*flow.Header) } } diff --git a/state/protocol/mock/snapshot.go b/state/protocol/mock/snapshot.go index 0cce1c96112..95c22c64fb4 100644 --- a/state/protocol/mock/snapshot.go +++ b/state/protocol/mock/snapshot.go @@ -313,6 +313,32 @@ func (_m *Snapshot) SealingSegment() (*flow.SealingSegment, error) { return r0, r1 } +// VersionBeacon provides a mock function with given fields: +func (_m *Snapshot) VersionBeacon() (*flow.SealedVersionBeacon, error) { + ret := _m.Called() + + var r0 *flow.SealedVersionBeacon + var r1 error + if rf, ok := ret.Get(0).(func() (*flow.SealedVersionBeacon, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *flow.SealedVersionBeacon); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.SealedVersionBeacon) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + type mockConstructorTestingTNewSnapshot interface { mock.TestingT Cleanup(func()) diff --git a/state/protocol/params.go b/state/protocol/params.go index 2c65ae73690..be308d30145 100644 --- a/state/protocol/params.go +++ b/state/protocol/params.go @@ -17,11 +17,16 @@ type Params interface { // different instance params. type InstanceParams interface { - // Root returns the root header of the current protocol state. This will be + // FinalizedRoot returns the finalized root header of the current protocol state. This will be // the head of the protocol state snapshot used to bootstrap this state and // may differ from node to node for the same protocol state. // No errors are expected during normal operation. - Root() (*flow.Header, error) + FinalizedRoot() (*flow.Header, error) + + // SealedRoot returns the sealed root block. If it's different from FinalizedRoot() block, + // it means the node is bootstrapped from mid-spork. + // No errors are expected during normal operation. + SealedRoot() (*flow.Header, error) // Seal returns the root block seal of the current protocol state. This will be // the seal for the root block used to bootstrap this state and may differ from diff --git a/state/protocol/prg/customizers.go b/state/protocol/prg/customizers.go new file mode 100644 index 00000000000..70012f65e83 --- /dev/null +++ b/state/protocol/prg/customizers.go @@ -0,0 +1,51 @@ +package prg + +import ( + "encoding/binary" + "fmt" + "math" +) + +// List of customizers used for different sub-protocol PRGs. +// These customizers help instantiate different PRGs from the +// same source of randomness. +// +// Customizers used by the Flow protocol should not be equal or +// prefixing each other to guarantee independent PRGs. This +// is enforced by test `TestProtocolConstants` in `./prg_test.go` + +var ( + // ConsensusLeaderSelection is the customizer for consensus leader selection + ConsensusLeaderSelection = customizerFromIndices(0, 1, 1) + // VerificationChunkAssignment is the customizer for verification chunk assignment + VerificationChunkAssignment = customizerFromIndices(0, 2, 0) + // ExecutionEnvironment is the customizer for Flow's transaction execution environment + ExecutionEnvironment = customizerFromIndices(1) + // + // clusterLeaderSelectionPrefix is the prefix used for CollectorClusterLeaderSelection + clusterLeaderSelectionPrefix = []uint16{0, 0} +) + +// CollectorClusterLeaderSelection returns the indices for the leader selection for the i-th collector cluster +func CollectorClusterLeaderSelection(clusterIndex uint) []byte { + if uint(math.MaxUint16) < clusterIndex { + // sanity check to guarantee no overflows during type conversion -- this should never happen + panic(fmt.Sprintf("input cluster index (%d) exceeds max uint16 value %d", clusterIndex, math.MaxUint16)) + } + indices := append(clusterLeaderSelectionPrefix, uint16(clusterIndex)) + return customizerFromIndices(indices...) +} + +// customizerFromIndices converts the input indices into a slice of bytes. +// The function has to be injective (no different indices map to the same customizer) +// +// The output is built as a concatenation of indices, each index is encoded over 2 bytes. +func customizerFromIndices(indices ...uint16) []byte { + customizerLen := 2 * len(indices) + customizer := make([]byte, customizerLen) + // concatenate the indices + for i, index := range indices { + binary.LittleEndian.PutUint16(customizer[2*i:2*i+2], index) + } + return customizer +} diff --git a/state/protocol/prg/prg.go b/state/protocol/prg/prg.go new file mode 100644 index 00000000000..8dec841c05b --- /dev/null +++ b/state/protocol/prg/prg.go @@ -0,0 +1,68 @@ +package prg + +import ( + "fmt" + + "golang.org/x/crypto/sha3" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/crypto/random" +) + +const RandomSourceLength = crypto.SignatureLenBLSBLS12381 + +// New returns a PRG seeded by the input source of randomness [SoR]. +// The customizer is used to generate a task-specific PRG. A customizer can be any slice +// of 12 bytes or less. +// The diversifier is used to further diversify the PRGs beyond the customizer. A diversifier +// can be a slice of any length. If no diversification is needed, `diversifier` can be `nil`. +// +// The function uses an extendable-output function (xof) to extract and expand the the input source, +// so that any source with enough entropy (at least 128 bits) can be used (no need to pre-hash). +// Current implementation generates a ChaCha20-based CSPRG. +// +// How to use the function in Flow protocol: any sub-protocol that requires deterministic and +// distributed randomness should rely on the Flow native randomness provided by the Random Beacon. +// The beacon SoR for block B is part of the QC certifying B and can be extracted using the +// function `consensus/hotstuff/model.BeaconSignature(*flow.QuorumCertificate)`. It can also be +// extracted using the `state/protocol/snapshot.RandomSource()` function. +// +// While the output is a distributed source of randomness, it should _not_ be used as random +// numbers itself. Instead, please use the function `New` to instantiate a PRG, +// for deterministic generation of random numbers or permutations (check the random.Rand interface). +// +// Every Flow sub-protocol should use its own customizer to create an independent PRG. Use the list in +// "customizers.go" to add new values. The same sub-protocol can further create independent PRGs +// by using `diversifier`. +func New(source []byte, customizer []byte, diversifier []byte) (random.Rand, error) { + seed, err := xof(source, diversifier, random.Chacha20SeedLen) + if err != nil { + return nil, fmt.Errorf("extendable output function failed: %w", err) + } + + // create random number generator from the seed and customizer + rng, err := random.NewChacha20PRG(seed, customizer) + if err != nil { + return nil, fmt.Errorf("could not create ChaCha20 PRG: %w", err) + } + return rng, nil +} + +// xof (extendable output function) extracts and expands the input's entropy into +// an output byte-slice of length `outLen`. +// It also takes a `diversifier` slice as an input to create independent outputs. +// +// Purpose of this function: it abstracts the extraction and expansion of +// entropy from the rest of PRG logic. The source doesn't necessarily have a uniformly +// distributed entropy (for instance a cryptographic signature), and hashing doesn't necessarily +// output the number of bytes required by the PRG (the code currently relies on ChaCha20 but this +// choice could evolve). +func xof(source []byte, diversifier []byte, outLen int) ([]byte, error) { + // CShake is used in this case but any other primitive that acts as a xof + // and accepts a diversifier can be used. + shake := sha3.NewCShake128(nil, diversifier) + _, _ = shake.Write(source) // cshake Write doesn't error + out := make([]byte, outLen) + _, _ = shake.Read(out) // cshake Read doesn't error + return out, nil +} diff --git a/state/protocol/prg/prg_test.go b/state/protocol/prg/prg_test.go new file mode 100644 index 00000000000..4c44539a24e --- /dev/null +++ b/state/protocol/prg/prg_test.go @@ -0,0 +1,78 @@ +package prg + +import ( + "bytes" + "crypto/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func getRandomSource(t *testing.T) []byte { + seed := make([]byte, RandomSourceLength) + _, err := rand.Read(seed) // checking err is enough + require.NoError(t, err) + t.Logf("seed is %#x", seed) + return seed +} + +func getRandoms(t *testing.T, seed, customizer, diversifier []byte, N int) []byte { + prg, err := New(seed, customizer, diversifier) + require.NoError(t, err) + rand := make([]byte, N) + prg.Read(rand) + return rand +} + +// check PRGs created from the same source give the same outputs +func TestDeterministic(t *testing.T) { + seed := getRandomSource(t) + customizer := []byte("cust test") + diversifier := []byte("div test") + + rand1 := getRandoms(t, seed, customizer, diversifier, 100) + rand2 := getRandoms(t, seed, customizer, diversifier, 100) + assert.Equal(t, rand1, rand2) +} + +// check different customizers lead to different randoms +func TestDifferentInstances(t *testing.T) { + seed := getRandomSource(t) + customizer1 := []byte("cust test1") + customizer2 := []byte("cust test2") + diversifer1 := []byte("div test1") + diversifer2 := []byte("div test2") + // different customizers + rand1 := getRandoms(t, seed, customizer1, diversifer1, 2) + rand2 := getRandoms(t, seed, customizer2, diversifer1, 2) + assert.NotEqual(t, rand1, rand2) + // different customizers + rand1 = getRandoms(t, seed, customizer1, diversifer1, 2) + rand2 = getRandoms(t, seed, customizer1, diversifer2, 2) + assert.NotEqual(t, rand1, rand2) + // test no error is returned with empty customizer and diversifier + _ = getRandoms(t, seed, nil, nil, 2) // error is checked inside the call +} + +// Sanity check that all customizers used by the Flow protocol +// are different and are not prefixes of each other +func TestProtocolConstants(t *testing.T) { + // include all sub-protocol customizers + customizers := [][]byte{ + ConsensusLeaderSelection, + VerificationChunkAssignment, + ExecutionEnvironment, + customizerFromIndices(clusterLeaderSelectionPrefix...), + } + + // go through all couples + for i, c := range customizers { + for j, other := range customizers { + if i == j { + continue + } + assert.False(t, bytes.HasPrefix(c, other)) + } + } +} diff --git a/state/protocol/seed/customizers.go b/state/protocol/seed/customizers.go deleted file mode 100644 index 8b65564b412..00000000000 --- a/state/protocol/seed/customizers.go +++ /dev/null @@ -1,46 +0,0 @@ -package seed - -import "encoding/binary" - -// list of customizers used for different sub-protocol PRNGs. -// These customizers help instantiate different PRNGs from the -// same source of randomness. - -var ( - // ProtocolConsensusLeaderSelection is the customizer for consensus leader selection - ProtocolConsensusLeaderSelection = customizerFromIndices([]uint16{0, 1, 1}) - // ProtocolVerificationChunkAssignment is the customizer for verification nodes determines chunk assignment - ProtocolVerificationChunkAssignment = customizerFromIndices([]uint16{0, 2, 0}) - // collectorClusterLeaderSelectionPrefix is the prefix of the customizer for the leader selection of collector clusters - collectorClusterLeaderSelectionPrefix = []uint16{0, 0} - // executionChunkPrefix is the prefix of the customizer for executing chunks - executionChunkPrefix = []uint16{1} -) - -// ProtocolCollectorClusterLeaderSelection returns the indices for the leader selection for the i-th collector cluster -func ProtocolCollectorClusterLeaderSelection(clusterIndex uint) []byte { - indices := append(collectorClusterLeaderSelectionPrefix, uint16(clusterIndex)) - return customizerFromIndices(indices) -} - -// ExecutionChunk returns the indices for i-th chunk -func ExecutionChunk(chunkIndex uint16) []byte { - indices := append(executionChunkPrefix, chunkIndex) - return customizerFromIndices(indices) -} - -// customizerFromIndices maps the input indices into a slice of bytes. -// The implementation ensures there are no collisions of mapping of different indices. -// -// The output is built as a concatenation of indices, each index encoded over 2 bytes. -// (the implementation could be updated to map the indices differently depending on the -// constraints over the output length) -func customizerFromIndices(indices []uint16) []byte { - customizerLen := 2 * len(indices) - customizer := make([]byte, customizerLen) - // concatenate the indices - for i, index := range indices { - binary.LittleEndian.PutUint16(customizer[2*i:2*i+2], index) - } - return customizer -} diff --git a/state/protocol/seed/prg_test.go b/state/protocol/seed/prg_test.go deleted file mode 100644 index 5111fa50aa6..00000000000 --- a/state/protocol/seed/prg_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package seed - -import ( - "math/rand" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func getRandomSource(t *testing.T) []byte { - r := time.Now().UnixNano() - rand.Seed(r) - t.Logf("math rand seed is %d", r) - seed := make([]byte, RandomSourceLength) - rand.Read(seed) - return seed -} - -// check PRGs created from the same source give the same outputs -func TestDeterministic(t *testing.T) { - seed := getRandomSource(t) - customizer := []byte("test") - prg1, err := PRGFromRandomSource(seed, customizer) - require.NoError(t, err) - prg2, err := PRGFromRandomSource(seed, customizer) - require.NoError(t, err) - - rand1 := make([]byte, 100) - prg1.Read(rand1) - rand2 := make([]byte, 100) - prg2.Read(rand2) - - assert.Equal(t, rand1, rand2) -} - -func TestCustomizer(t *testing.T) { - seed := getRandomSource(t) - customizer1 := []byte("test1") - prg1, err := PRGFromRandomSource(seed, customizer1) - require.NoError(t, err) - customizer2 := []byte("test2") - prg2, err := PRGFromRandomSource(seed, customizer2) - require.NoError(t, err) - - rand1 := make([]byte, 100) - prg1.Read(rand1) - rand2 := make([]byte, 100) - prg2.Read(rand2) - - assert.NotEqual(t, rand1, rand2) -} diff --git a/state/protocol/seed/seed.go b/state/protocol/seed/seed.go deleted file mode 100644 index f8160e1c334..00000000000 --- a/state/protocol/seed/seed.go +++ /dev/null @@ -1,43 +0,0 @@ -package seed - -import ( - "fmt" - - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/crypto" - "github.com/onflow/flow-go/crypto/hash" - "github.com/onflow/flow-go/crypto/random" -) - -// PRGFromRandomSource returns a PRG seeded by the source of randomness of the protocol. -// The customizer is used to generate a task-specific PRG (customizer in this implementation -// is up to 12-bytes long). -// -// The function hashes the input random source to obtain the PRG seed. -// Hashing is required to uniformize the entropy over the output. -func PRGFromRandomSource(randomSource []byte, customizer []byte) (random.Rand, error) { - // hash the source of randomness (signature) to uniformize the entropy - var seed [hash.HashLenSHA3_256]byte - hash.ComputeSHA3_256(&seed, randomSource) - - // create random number generator from the seed and customizer - rng, err := random.NewChacha20PRG(seed[:], customizer) - if err != nil { - return nil, fmt.Errorf("could not create ChaCha20 PRG: %w", err) - } - return rng, nil -} - -const RandomSourceLength = crypto.SignatureLenBLSBLS12381 - -// FromParentQCSignature extracts the source of randomness from the given QC sigData. -// The sigData is an RLP encoded structure that is part of QuorumCertificate. -func FromParentQCSignature(sigData []byte) ([]byte, error) { - // unpack sig data to extract random beacon sig - randomBeaconSig, err := model.UnpackRandomBeaconSig(sigData) - if err != nil { - return nil, fmt.Errorf("could not unpack block signature: %w", err) - } - - return randomBeaconSig, nil -} diff --git a/state/protocol/snapshot.go b/state/protocol/snapshot.go index 73b3acf8930..f557bc59fbc 100644 --- a/state/protocol/snapshot.go +++ b/state/protocol/snapshot.go @@ -136,4 +136,12 @@ type Snapshot interface { // Params returns global parameters of the state this snapshot is taken from. // Returns invalid.Params with state.ErrUnknownSnapshotReference if snapshot reference block is unknown. Params() GlobalParams + + // VersionBeacon returns the latest sealed version beacon. + // If no version beacon has been sealed so far during the current spork, returns nil. + // The latest VersionBeacon is only updated for finalized blocks. This means that, when + // querying an un-finalized fork, `VersionBeacon` will have the same value as querying + // the snapshot for the latest finalized block, even if a newer version beacon is included + // in a seal along the un-finalized fork. + VersionBeacon() (*flow.SealedVersionBeacon, error) } diff --git a/state/protocol/util/testing.go b/state/protocol/util/testing.go index 9b31e00fb9c..24eb8016f6f 100644 --- a/state/protocol/util/testing.go +++ b/state/protocol/util/testing.go @@ -67,7 +67,20 @@ func RunWithBootstrapState(t testing.TB, rootSnapshot protocol.Snapshot, f func( unittest.RunWithBadgerDB(t, func(db *badger.DB) { metrics := metrics.NewNoopCollector() all := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := pbadger.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) f(db, state) }) @@ -80,7 +93,20 @@ func RunWithFullProtocolState(t testing.TB, rootSnapshot protocol.Snapshot, f fu log := zerolog.Nop() consumer := events.NewNoop() all := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := pbadger.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) receiptValidator := MockReceiptValidator() sealValidator := MockSealValidator(all.Seals) @@ -97,7 +123,20 @@ func RunWithFullProtocolStateAndMetrics(t testing.TB, rootSnapshot protocol.Snap log := zerolog.Nop() consumer := events.NewNoop() all := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := pbadger.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) receiptValidator := MockReceiptValidator() sealValidator := MockSealValidator(all.Seals) @@ -115,7 +154,20 @@ func RunWithFullProtocolStateAndValidator(t testing.TB, rootSnapshot protocol.Sn log := zerolog.Nop() consumer := events.NewNoop() all := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := pbadger.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) sealValidator := MockSealValidator(all.Seals) mockTimer := MockBlockTimer() @@ -132,7 +184,20 @@ func RunWithFollowerProtocolState(t testing.TB, rootSnapshot protocol.Snapshot, log := zerolog.Nop() consumer := events.NewNoop() all := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := pbadger.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) mockTimer := MockBlockTimer() followerState, err := pbadger.NewFollowerState(log, tracer, consumer, state, all.Index, all.Payloads, mockTimer) @@ -147,7 +212,20 @@ func RunWithFullProtocolStateAndConsumer(t testing.TB, rootSnapshot protocol.Sna tracer := trace.NewNoopTracer() log := zerolog.Nop() all := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := pbadger.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) receiptValidator := MockReceiptValidator() sealValidator := MockSealValidator(all.Seals) @@ -163,7 +241,20 @@ func RunWithFullProtocolStateAndMetricsAndConsumer(t testing.TB, rootSnapshot pr tracer := trace.NewNoopTracer() log := zerolog.Nop() all := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := pbadger.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) receiptValidator := MockReceiptValidator() sealValidator := MockSealValidator(all.Seals) @@ -181,7 +272,20 @@ func RunWithFollowerProtocolStateAndHeaders(t testing.TB, rootSnapshot protocol. log := zerolog.Nop() consumer := events.NewNoop() all := util.StorageLayer(t, db) - state, err := pbadger.Bootstrap(metrics, db, all.Headers, all.Seals, all.Results, all.Blocks, all.QuorumCertificates, all.Setups, all.EpochCommits, all.Statuses, rootSnapshot) + state, err := pbadger.Bootstrap( + metrics, + db, + all.Headers, + all.Seals, + all.Results, + all.Blocks, + all.QuorumCertificates, + all.Setups, + all.EpochCommits, + all.Statuses, + all.VersionBeacons, + rootSnapshot, + ) require.NoError(t, err) mockTimer := MockBlockTimer() followerState, err := pbadger.NewFollowerState(log, tracer, consumer, state, all.Index, all.Payloads, mockTimer) diff --git a/storage/all.go b/storage/all.go index bc6fc22e7c2..eb2c9eb0328 100644 --- a/storage/all.go +++ b/storage/all.go @@ -20,4 +20,5 @@ type All struct { TransactionResults TransactionResults Collections Collections Events Events + VersionBeacons VersionBeacons } diff --git a/storage/badger/all.go b/storage/badger/all.go index 52795591262..58bc45e6848 100644 --- a/storage/badger/all.go +++ b/storage/badger/all.go @@ -20,6 +20,7 @@ func InitAll(metrics module.CacheMetrics, db *badger.DB) *storage.All { setups := NewEpochSetups(metrics, db) epochCommits := NewEpochCommits(metrics, db) statuses := NewEpochStatuses(metrics, db) + versionBeacons := NewVersionBeacons(db) commits := NewCommits(metrics, db) transactions := NewTransactions(metrics, db) @@ -39,6 +40,7 @@ func InitAll(metrics module.CacheMetrics, db *badger.DB) *storage.All { Setups: setups, EpochCommits: epochCommits, Statuses: statuses, + VersionBeacons: versionBeacons, Results: results, Receipts: receipts, ChunkDataPacks: chunkDataPacks, diff --git a/storage/badger/approvals.go b/storage/badger/approvals.go index 5c33ef123f3..eb3cf4ae820 100644 --- a/storage/badger/approvals.go +++ b/storage/badger/approvals.go @@ -17,20 +17,18 @@ import ( // ResultApprovals implements persistent storage for result approvals. type ResultApprovals struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.ResultApproval] } func NewResultApprovals(collector module.CacheMetrics, db *badger.DB) *ResultApprovals { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - approval := val.(*flow.ResultApproval) - return transaction.WithTx(operation.SkipDuplicates(operation.InsertResultApproval(approval))) + store := func(key flow.Identifier, val *flow.ResultApproval) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertResultApproval(val))) } - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - approvalID := key.(flow.Identifier) + retrieve := func(approvalID flow.Identifier) func(tx *badger.Txn) (*flow.ResultApproval, error) { var approval flow.ResultApproval - return func(tx *badger.Txn) (interface{}, error) { + return func(tx *badger.Txn) (*flow.ResultApproval, error) { err := operation.RetrieveResultApproval(approvalID, &approval)(tx) return &approval, err } @@ -38,10 +36,10 @@ func NewResultApprovals(collector module.CacheMetrics, db *badger.DB) *ResultApp res := &ResultApprovals{ db: db, - cache: newCache(collector, metrics.ResourceResultApprovals, - withLimit(flow.DefaultTransactionExpiry+100), - withStore(store), - withRetrieve(retrieve)), + cache: newCache[flow.Identifier, *flow.ResultApproval](collector, metrics.ResourceResultApprovals, + withLimit[flow.Identifier, *flow.ResultApproval](flow.DefaultTransactionExpiry+100), + withStore[flow.Identifier, *flow.ResultApproval](store), + withRetrieve[flow.Identifier, *flow.ResultApproval](retrieve)), } return res @@ -57,7 +55,7 @@ func (r *ResultApprovals) byID(approvalID flow.Identifier) func(*badger.Txn) (*f if err != nil { return nil, err } - return val.(*flow.ResultApproval), nil + return val, nil } } diff --git a/storage/badger/cache.go b/storage/badger/cache.go index 5af5d23f8b1..67bc0b7f055 100644 --- a/storage/badger/cache.go +++ b/storage/badger/cache.go @@ -5,92 +5,92 @@ import ( "fmt" "github.com/dgraph-io/badger/v2" - lru "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru/v2" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/transaction" ) -func withLimit(limit uint) func(*Cache) { - return func(c *Cache) { +func withLimit[K comparable, V any](limit uint) func(*Cache[K, V]) { + return func(c *Cache[K, V]) { c.limit = limit } } -type storeFunc func(key interface{}, val interface{}) func(*transaction.Tx) error +type storeFunc[K comparable, V any] func(key K, val V) func(*transaction.Tx) error const DefaultCacheSize = uint(1000) -func withStore(store storeFunc) func(*Cache) { - return func(c *Cache) { +func withStore[K comparable, V any](store storeFunc[K, V]) func(*Cache[K, V]) { + return func(c *Cache[K, V]) { c.store = store } } -func noStore(key interface{}, val interface{}) func(*transaction.Tx) error { +func noStore[K comparable, V any](_ K, _ V) func(*transaction.Tx) error { return func(tx *transaction.Tx) error { return fmt.Errorf("no store function for cache put available") } } -func noopStore(key interface{}, val interface{}) func(*transaction.Tx) error { +func noopStore[K comparable, V any](_ K, _ V) func(*transaction.Tx) error { return func(tx *transaction.Tx) error { return nil } } -type retrieveFunc func(key interface{}) func(*badger.Txn) (interface{}, error) +type retrieveFunc[K comparable, V any] func(key K) func(*badger.Txn) (V, error) -func withRetrieve(retrieve retrieveFunc) func(*Cache) { - return func(c *Cache) { +func withRetrieve[K comparable, V any](retrieve retrieveFunc[K, V]) func(*Cache[K, V]) { + return func(c *Cache[K, V]) { c.retrieve = retrieve } } -func noRetrieve(key interface{}) func(*badger.Txn) (interface{}, error) { - return func(tx *badger.Txn) (interface{}, error) { - return nil, fmt.Errorf("no retrieve function for cache get available") +func noRetrieve[K comparable, V any](_ K) func(*badger.Txn) (V, error) { + return func(tx *badger.Txn) (V, error) { + var nullV V + return nullV, fmt.Errorf("no retrieve function for cache get available") } } -type Cache struct { +type Cache[K comparable, V any] struct { metrics module.CacheMetrics limit uint - store storeFunc - retrieve retrieveFunc + store storeFunc[K, V] + retrieve retrieveFunc[K, V] resource string - cache *lru.Cache + cache *lru.Cache[K, V] } -func newCache(collector module.CacheMetrics, resourceName string, options ...func(*Cache)) *Cache { - c := Cache{ +func newCache[K comparable, V any](collector module.CacheMetrics, resourceName string, options ...func(*Cache[K, V])) *Cache[K, V] { + c := Cache[K, V]{ metrics: collector, limit: 1000, - store: noStore, - retrieve: noRetrieve, + store: noStore[K, V], + retrieve: noRetrieve[K, V], resource: resourceName, } for _, option := range options { option(&c) } - c.cache, _ = lru.New(int(c.limit)) + c.cache, _ = lru.New[K, V](int(c.limit)) c.metrics.CacheEntries(c.resource, uint(c.cache.Len())) return &c } // IsCached returns true if the key exists in the cache. // It DOES NOT check whether the key exists in the underlying data store. -func (c *Cache) IsCached(key any) bool { - exists := c.cache.Contains(key) - return exists +func (c *Cache[K, V]) IsCached(key K) bool { + return c.cache.Contains(key) } // Get will try to retrieve the resource from cache first, and then from the // injected. During normal operations, the following error returns are expected: // - `storage.ErrNotFound` if key is unknown. -func (c *Cache) Get(key interface{}) func(*badger.Txn) (interface{}, error) { - return func(tx *badger.Txn) (interface{}, error) { +func (c *Cache[K, V]) Get(key K) func(*badger.Txn) (V, error) { + return func(tx *badger.Txn) (V, error) { // check if we have it in the cache resource, cached := c.cache.Get(key) @@ -105,7 +105,8 @@ func (c *Cache) Get(key interface{}) func(*badger.Txn) (interface{}, error) { if errors.Is(err, storage.ErrNotFound) { c.metrics.CacheNotFound(c.resource) } - return nil, fmt.Errorf("could not retrieve resource: %w", err) + var nullV V + return nullV, fmt.Errorf("could not retrieve resource: %w", err) } c.metrics.CacheMiss(c.resource) @@ -120,12 +121,12 @@ func (c *Cache) Get(key interface{}) func(*badger.Txn) (interface{}, error) { } } -func (c *Cache) Remove(key interface{}) { +func (c *Cache[K, V]) Remove(key K) { c.cache.Remove(key) } // Insert will add a resource directly to the cache with the given ID -func (c *Cache) Insert(key interface{}, resource interface{}) { +func (c *Cache[K, V]) Insert(key K, resource V) { // cache the resource and eject least recently used one if we reached limit evicted := c.cache.Add(key, resource) if !evicted { @@ -134,7 +135,7 @@ func (c *Cache) Insert(key interface{}, resource interface{}) { } // PutTx will return tx which adds a resource to the cache with the given ID. -func (c *Cache) PutTx(key interface{}, resource interface{}) func(*transaction.Tx) error { +func (c *Cache[K, V]) PutTx(key K, resource V) func(*transaction.Tx) error { storeOps := c.store(key, resource) // assemble DB operations to store resource (no execution) return func(tx *transaction.Tx) error { diff --git a/storage/badger/cache_test.go b/storage/badger/cache_test.go index fdc0e73dc51..76ea7ce18bc 100644 --- a/storage/badger/cache_test.go +++ b/storage/badger/cache_test.go @@ -5,13 +5,14 @@ import ( "github.com/stretchr/testify/assert" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/utils/unittest" ) // TestCache_Exists tests existence checking items in the cache. func TestCache_Exists(t *testing.T) { - cache := newCache(metrics.NewNoopCollector(), "test") + cache := newCache[flow.Identifier, any](metrics.NewNoopCollector(), "test") t.Run("non-existent", func(t *testing.T) { key := unittest.IdentifierFixture() diff --git a/storage/badger/chunkDataPacks.go b/storage/badger/chunkDataPacks.go index c54a95c1c80..2a5c733f60e 100644 --- a/storage/badger/chunkDataPacks.go +++ b/storage/badger/chunkDataPacks.go @@ -17,28 +17,25 @@ import ( type ChunkDataPacks struct { db *badger.DB collections storage.Collections - byChunkIDCache *Cache + byChunkIDCache *Cache[flow.Identifier, *badgermodel.StoredChunkDataPack] } func NewChunkDataPacks(collector module.CacheMetrics, db *badger.DB, collections storage.Collections, byChunkIDCacheSize uint) *ChunkDataPacks { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - chdp := val.(*badgermodel.StoredChunkDataPack) - return transaction.WithTx(operation.SkipDuplicates(operation.InsertChunkDataPack(chdp))) + store := func(key flow.Identifier, val *badgermodel.StoredChunkDataPack) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertChunkDataPack(val))) } - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - chunkID := key.(flow.Identifier) - - var c badgermodel.StoredChunkDataPack - return func(tx *badger.Txn) (interface{}, error) { - err := operation.RetrieveChunkDataPack(chunkID, &c)(tx) + retrieve := func(key flow.Identifier) func(tx *badger.Txn) (*badgermodel.StoredChunkDataPack, error) { + return func(tx *badger.Txn) (*badgermodel.StoredChunkDataPack, error) { + var c badgermodel.StoredChunkDataPack + err := operation.RetrieveChunkDataPack(key, &c)(tx) return &c, err } } - cache := newCache(collector, metrics.ResourceChunkDataPack, - withLimit(byChunkIDCacheSize), + cache := newCache[flow.Identifier, *badgermodel.StoredChunkDataPack](collector, metrics.ResourceChunkDataPack, + withLimit[flow.Identifier, *badgermodel.StoredChunkDataPack](byChunkIDCacheSize), withStore(store), withRetrieve(retrieve), ) @@ -135,7 +132,7 @@ func (ch *ChunkDataPacks) retrieveCHDP(chunkID flow.Identifier) func(*badger.Txn if err != nil { return nil, err } - return val.(*badgermodel.StoredChunkDataPack), nil + return val, nil } } diff --git a/storage/badger/cleaner.go b/storage/badger/cleaner.go index 025b8d141f8..d9cd07997e7 100644 --- a/storage/badger/cleaner.go +++ b/storage/badger/cleaner.go @@ -3,7 +3,6 @@ package badger import ( - "math/rand" "time" "github.com/dgraph-io/badger/v2" @@ -12,6 +11,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/utils/rand" ) // Cleaner uses component.ComponentManager to implement module.Startable and module.ReadyDoneAware @@ -82,7 +82,17 @@ func (c *Cleaner) gcWorkerRoutine(ctx irrecoverable.SignalerContext, ready compo // We add 20% jitter into the interval, so that we don't risk nodes syncing their GC calls over time. // Therefore GC is run every X seconds, where X is uniformly sampled from [interval, interval*1.2] func (c *Cleaner) nextWaitDuration() time.Duration { - return time.Duration(c.interval.Milliseconds() + rand.Int63n(c.interval.Milliseconds()/5)) + jitter, err := rand.Uint64n(uint64(c.interval.Nanoseconds() / 5)) + if err != nil { + // if randomness fails, do not use a jitter for this instance. + // TODO: address the error properly and not swallow it. + // In this specific case, `utils/rand` only errors if the system randomness fails + // which is a symptom of a wider failure. Many other node components would catch such + // a failure. + c.log.Warn().Msg("jitter is zero beacuse system randomness has failed") + jitter = 0 + } + return time.Duration(c.interval.Nanoseconds() + int64(jitter)) } // runGC runs garbage collection for badger DB, handles sentinel errors and reports metrics. diff --git a/storage/badger/cluster_blocks_test.go b/storage/badger/cluster_blocks_test.go new file mode 100644 index 00000000000..64def9fec6b --- /dev/null +++ b/storage/badger/cluster_blocks_test.go @@ -0,0 +1,50 @@ +package badger + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger/operation" + "github.com/onflow/flow-go/storage/badger/procedure" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestClusterBlocksByHeight(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + chain := unittest.ClusterBlockChainFixture(5) + parent, blocks := chain[0], chain[1:] + + // add parent as boundary + err := db.Update(operation.IndexClusterBlockHeight(parent.Header.ChainID, parent.Header.Height, parent.ID())) + require.NoError(t, err) + + err = db.Update(operation.InsertClusterFinalizedHeight(parent.Header.ChainID, parent.Header.Height)) + require.NoError(t, err) + + // store a chain of blocks + for _, block := range blocks { + err := db.Update(procedure.InsertClusterBlock(&block)) + require.NoError(t, err) + + err = db.Update(procedure.FinalizeClusterBlock(block.Header.ID())) + require.NoError(t, err) + } + + clusterBlocks := NewClusterBlocks( + db, + blocks[0].Header.ChainID, + NewHeaders(metrics.NewNoopCollector(), db), + NewClusterPayloads(metrics.NewNoopCollector(), db), + ) + + // check if the block can be retrieved by height + for _, block := range blocks { + retrievedBlock, err := clusterBlocks.ByHeight(block.Header.Height) + require.NoError(t, err) + require.Equal(t, block.ID(), retrievedBlock.ID()) + } + }) +} diff --git a/storage/badger/cluster_payloads.go b/storage/badger/cluster_payloads.go index 84e260b9a75..0fc3ba3ee28 100644 --- a/storage/badger/cluster_payloads.go +++ b/storage/badger/cluster_payloads.go @@ -16,21 +16,18 @@ import ( // cluster consensus. type ClusterPayloads struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *cluster.Payload] } func NewClusterPayloads(cacheMetrics module.CacheMetrics, db *badger.DB) *ClusterPayloads { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - blockID := key.(flow.Identifier) - payload := val.(*cluster.Payload) + store := func(blockID flow.Identifier, payload *cluster.Payload) func(*transaction.Tx) error { return transaction.WithTx(procedure.InsertClusterPayload(blockID, payload)) } - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - blockID := key.(flow.Identifier) + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (*cluster.Payload, error) { var payload cluster.Payload - return func(tx *badger.Txn) (interface{}, error) { + return func(tx *badger.Txn) (*cluster.Payload, error) { err := procedure.RetrieveClusterPayload(blockID, &payload)(tx) return &payload, err } @@ -38,8 +35,8 @@ func NewClusterPayloads(cacheMetrics module.CacheMetrics, db *badger.DB) *Cluste cp := &ClusterPayloads{ db: db, - cache: newCache(cacheMetrics, metrics.ResourceClusterPayload, - withLimit(flow.DefaultTransactionExpiry*4), + cache: newCache[flow.Identifier, *cluster.Payload](cacheMetrics, metrics.ResourceClusterPayload, + withLimit[flow.Identifier, *cluster.Payload](flow.DefaultTransactionExpiry*4), withStore(store), withRetrieve(retrieve)), } @@ -56,7 +53,7 @@ func (cp *ClusterPayloads) retrieveTx(blockID flow.Identifier) func(*badger.Txn) if err != nil { return nil, err } - return val.(*cluster.Payload), nil + return val, nil } } diff --git a/storage/badger/commits.go b/storage/badger/commits.go index af60946d043..11a4e4aa8e2 100644 --- a/storage/badger/commits.go +++ b/storage/badger/commits.go @@ -13,21 +13,18 @@ import ( type Commits struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, flow.StateCommitment] } func NewCommits(collector module.CacheMetrics, db *badger.DB) *Commits { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - blockID := key.(flow.Identifier) - commit := val.(flow.StateCommitment) + store := func(blockID flow.Identifier, commit flow.StateCommitment) func(*transaction.Tx) error { return transaction.WithTx(operation.SkipDuplicates(operation.IndexStateCommitment(blockID, commit))) } - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - blockID := key.(flow.Identifier) - var commit flow.StateCommitment - return func(tx *badger.Txn) (interface{}, error) { + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (flow.StateCommitment, error) { + return func(tx *badger.Txn) (flow.StateCommitment, error) { + var commit flow.StateCommitment err := operation.LookupStateCommitment(blockID, &commit)(tx) return commit, err } @@ -35,8 +32,8 @@ func NewCommits(collector module.CacheMetrics, db *badger.DB) *Commits { c := &Commits{ db: db, - cache: newCache(collector, metrics.ResourceCommit, - withLimit(1000), + cache: newCache[flow.Identifier, flow.StateCommitment](collector, metrics.ResourceCommit, + withLimit[flow.Identifier, flow.StateCommitment](1000), withStore(store), withRetrieve(retrieve), ), @@ -55,7 +52,7 @@ func (c *Commits) retrieveTx(blockID flow.Identifier) func(tx *badger.Txn) (flow if err != nil { return flow.DummyStateCommitment, err } - return val.(flow.StateCommitment), nil + return val, nil } } diff --git a/storage/badger/computation_result_test.go b/storage/badger/computation_result_test.go index e0be65017f3..6575611632c 100644 --- a/storage/badger/computation_result_test.go +++ b/storage/badger/computation_result_test.go @@ -10,18 +10,14 @@ import ( "github.com/dgraph-io/badger/v2" "github.com/onflow/flow-go/engine/execution" - "github.com/onflow/flow-go/ledger" - "github.com/onflow/flow-go/ledger/common/pathfinder" - "github.com/onflow/flow-go/ledger/complete" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/engine/execution/testutil" bstorage "github.com/onflow/flow-go/storage/badger" "github.com/onflow/flow-go/utils/unittest" ) func TestUpsertAndRetrieveComputationResult(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { - expected := generateComputationResult(t) + expected := testutil.ComputationResultFixture(t) crStorage := bstorage.NewComputationResultUploadStatus(db) crId := expected.ExecutableBlock.ID() @@ -50,7 +46,7 @@ func TestUpsertAndRetrieveComputationResult(t *testing.T) { func TestRemoveComputationResults(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { t.Run("Remove ComputationResult", func(t *testing.T) { - expected := generateComputationResult(t) + expected := testutil.ComputationResultFixture(t) crId := expected.ExecutableBlock.ID() crStorage := bstorage.NewComputationResultUploadStatus(db) @@ -74,8 +70,8 @@ func TestListComputationResults(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { t.Run("List all ComputationResult with given status", func(t *testing.T) { expected := [...]*execution.ComputationResult{ - generateComputationResult(t), - generateComputationResult(t), + testutil.ComputationResultFixture(t), + testutil.ComputationResultFixture(t), } crStorage := bstorage.NewComputationResultUploadStatus(db) @@ -89,8 +85,8 @@ func TestListComputationResults(t *testing.T) { } // Add in entries with non-targeted status unexpected := [...]*execution.ComputationResult{ - generateComputationResult(t), - generateComputationResult(t), + testutil.ComputationResultFixture(t), + testutil.ComputationResultFixture(t), } for _, cr := range unexpected { crId := cr.ExecutableBlock.ID() @@ -111,135 +107,3 @@ func TestListComputationResults(t *testing.T) { }) }) } - -// Generate ComputationResult for testing purposes -func generateComputationResult(t *testing.T) *execution.ComputationResult { - - update1, err := ledger.NewUpdate( - ledger.State(unittest.StateCommitmentFixture()), - []ledger.Key{ - ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(3, []byte{33})}), - ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(1, []byte{11})}), - ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(2, []byte{1, 1}), ledger.NewKeyPart(3, []byte{2, 5})}), - }, - []ledger.Value{ - []byte{21, 37}, - nil, - []byte{3, 3, 3, 3, 3}, - }, - ) - require.NoError(t, err) - - trieUpdate1, err := pathfinder.UpdateToTrieUpdate(update1, complete.DefaultPathFinderVersion) - require.NoError(t, err) - - update2, err := ledger.NewUpdate( - ledger.State(unittest.StateCommitmentFixture()), - []ledger.Key{}, - []ledger.Value{}, - ) - require.NoError(t, err) - - trieUpdate2, err := pathfinder.UpdateToTrieUpdate(update2, complete.DefaultPathFinderVersion) - require.NoError(t, err) - - update3, err := ledger.NewUpdate( - ledger.State(unittest.StateCommitmentFixture()), - []ledger.Key{ - ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(9, []byte{6})}), - }, - []ledger.Value{ - []byte{21, 37}, - }, - ) - require.NoError(t, err) - - trieUpdate3, err := pathfinder.UpdateToTrieUpdate(update3, complete.DefaultPathFinderVersion) - require.NoError(t, err) - - update4, err := ledger.NewUpdate( - ledger.State(unittest.StateCommitmentFixture()), - []ledger.Key{ - ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(9, []byte{6})}), - }, - []ledger.Value{ - []byte{21, 37}, - }, - ) - require.NoError(t, err) - - trieUpdate4, err := pathfinder.UpdateToTrieUpdate(update4, complete.DefaultPathFinderVersion) - require.NoError(t, err) - - return &execution.ComputationResult{ - ExecutableBlock: unittest.ExecutableBlockFixture([][]flow.Identifier{ - {unittest.IdentifierFixture()}, - {unittest.IdentifierFixture()}, - {unittest.IdentifierFixture()}, - }), - StateSnapshots: nil, - Events: []flow.EventsList{ - { - unittest.EventFixture("what", 0, 0, unittest.IdentifierFixture(), 2), - unittest.EventFixture("ever", 0, 1, unittest.IdentifierFixture(), 22), - }, - {}, - { - unittest.EventFixture("what", 2, 0, unittest.IdentifierFixture(), 2), - unittest.EventFixture("ever", 2, 1, unittest.IdentifierFixture(), 22), - unittest.EventFixture("ever", 2, 2, unittest.IdentifierFixture(), 2), - unittest.EventFixture("ever", 2, 3, unittest.IdentifierFixture(), 22), - }, - {}, // system chunk events - }, - EventsHashes: nil, - ServiceEvents: nil, - TransactionResults: []flow.TransactionResult{ - { - TransactionID: unittest.IdentifierFixture(), - ErrorMessage: "", - ComputationUsed: 23, - }, - { - TransactionID: unittest.IdentifierFixture(), - ErrorMessage: "fail", - ComputationUsed: 1, - }, - }, - TransactionResultIndex: []int{1, 1, 2, 2}, - BlockExecutionData: &execution_data.BlockExecutionData{ - ChunkExecutionDatas: []*execution_data.ChunkExecutionData{ - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate1, - }, - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate2, - }, - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate3, - }, - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate4, - }, - }, - }, - ExecutionReceipt: &flow.ExecutionReceipt{ - ExecutionResult: flow.ExecutionResult{ - Chunks: flow.ChunkList{ - { - EndState: unittest.StateCommitmentFixture(), - }, - { - EndState: unittest.StateCommitmentFixture(), - }, - { - EndState: unittest.StateCommitmentFixture(), - }, - { - EndState: unittest.StateCommitmentFixture(), - }, - }, - }, - }, - } -} diff --git a/storage/badger/dkg_state.go b/storage/badger/dkg_state.go index 63beb4c23a2..73e2b3e8133 100644 --- a/storage/badger/dkg_state.go +++ b/storage/badger/dkg_state.go @@ -18,7 +18,7 @@ import ( // computed keys. Must be instantiated using secrets database. type DKGState struct { db *badger.DB - keyCache *Cache + keyCache *Cache[uint64, *encodable.RandomBeaconPrivKey] } // NewDKGState returns the DKGState implementation backed by Badger DB. @@ -28,23 +28,20 @@ func NewDKGState(collector module.CacheMetrics, db *badger.DB) (*DKGState, error return nil, fmt.Errorf("cannot instantiate dkg state storage in non-secret db: %w", err) } - storeKey := func(key interface{}, val interface{}) func(*transaction.Tx) error { - epochCounter := key.(uint64) - info := val.(*encodable.RandomBeaconPrivKey) + storeKey := func(epochCounter uint64, info *encodable.RandomBeaconPrivKey) func(*transaction.Tx) error { return transaction.WithTx(operation.InsertMyBeaconPrivateKey(epochCounter, info)) } - retrieveKey := func(key interface{}) func(*badger.Txn) (interface{}, error) { - epochCounter := key.(uint64) - var info encodable.RandomBeaconPrivKey - return func(tx *badger.Txn) (interface{}, error) { + retrieveKey := func(epochCounter uint64) func(*badger.Txn) (*encodable.RandomBeaconPrivKey, error) { + return func(tx *badger.Txn) (*encodable.RandomBeaconPrivKey, error) { + var info encodable.RandomBeaconPrivKey err := operation.RetrieveMyBeaconPrivateKey(epochCounter, &info)(tx) return &info, err } } - cache := newCache(collector, metrics.ResourceBeaconKey, - withLimit(10), + cache := newCache[uint64, *encodable.RandomBeaconPrivKey](collector, metrics.ResourceBeaconKey, + withLimit[uint64, *encodable.RandomBeaconPrivKey](10), withStore(storeKey), withRetrieve(retrieveKey), ) @@ -67,7 +64,7 @@ func (ds *DKGState) retrieveKeyTx(epochCounter uint64) func(tx *badger.Txn) (*en if err != nil { return nil, err } - return val.(*encodable.RandomBeaconPrivKey), nil + return val, nil } } diff --git a/storage/badger/dkg_state_test.go b/storage/badger/dkg_state_test.go index 3c9a6653b49..5643b064d22 100644 --- a/storage/badger/dkg_state_test.go +++ b/storage/badger/dkg_state_test.go @@ -4,7 +4,6 @@ import ( "errors" "math/rand" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -53,7 +52,6 @@ func TestDKGState_BeaconKeys(t *testing.T) { store, err := bstorage.NewDKGState(metrics, db) require.NoError(t, err) - rand.Seed(time.Now().UnixNano()) epochCounter := rand.Uint64() // attempt to get a non-existent key @@ -96,7 +94,6 @@ func TestDKGState_EndState(t *testing.T) { store, err := bstorage.NewDKGState(metrics, db) require.NoError(t, err) - rand.Seed(time.Now().UnixNano()) epochCounter := rand.Uint64() endState := flow.DKGEndStateNoKey diff --git a/storage/badger/epoch_commits.go b/storage/badger/epoch_commits.go index 8f9022e7f09..20dadaccdba 100644 --- a/storage/badger/epoch_commits.go +++ b/storage/badger/epoch_commits.go @@ -12,21 +12,18 @@ import ( type EpochCommits struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.EpochCommit] } func NewEpochCommits(collector module.CacheMetrics, db *badger.DB) *EpochCommits { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - id := key.(flow.Identifier) - commit := val.(*flow.EpochCommit) + store := func(id flow.Identifier, commit *flow.EpochCommit) func(*transaction.Tx) error { return transaction.WithTx(operation.SkipDuplicates(operation.InsertEpochCommit(id, commit))) } - retrieve := func(key interface{}) func(*badger.Txn) (interface{}, error) { - id := key.(flow.Identifier) - var commit flow.EpochCommit - return func(tx *badger.Txn) (interface{}, error) { + retrieve := func(id flow.Identifier) func(*badger.Txn) (*flow.EpochCommit, error) { + return func(tx *badger.Txn) (*flow.EpochCommit, error) { + var commit flow.EpochCommit err := operation.RetrieveEpochCommit(id, &commit)(tx) return &commit, err } @@ -34,8 +31,8 @@ func NewEpochCommits(collector module.CacheMetrics, db *badger.DB) *EpochCommits ec := &EpochCommits{ db: db, - cache: newCache(collector, metrics.ResourceEpochCommit, - withLimit(4*flow.DefaultTransactionExpiry), + cache: newCache[flow.Identifier, *flow.EpochCommit](collector, metrics.ResourceEpochCommit, + withLimit[flow.Identifier, *flow.EpochCommit](4*flow.DefaultTransactionExpiry), withStore(store), withRetrieve(retrieve)), } @@ -53,7 +50,7 @@ func (ec *EpochCommits) retrieveTx(commitID flow.Identifier) func(tx *badger.Txn if err != nil { return nil, err } - return val.(*flow.EpochCommit), nil + return val, nil } } diff --git a/storage/badger/epoch_setups.go b/storage/badger/epoch_setups.go index 4b31db4b867..24757067f8f 100644 --- a/storage/badger/epoch_setups.go +++ b/storage/badger/epoch_setups.go @@ -12,22 +12,19 @@ import ( type EpochSetups struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.EpochSetup] } // NewEpochSetups instantiates a new EpochSetups storage. func NewEpochSetups(collector module.CacheMetrics, db *badger.DB) *EpochSetups { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - id := key.(flow.Identifier) - setup := val.(*flow.EpochSetup) + store := func(id flow.Identifier, setup *flow.EpochSetup) func(*transaction.Tx) error { return transaction.WithTx(operation.SkipDuplicates(operation.InsertEpochSetup(id, setup))) } - retrieve := func(key interface{}) func(*badger.Txn) (interface{}, error) { - id := key.(flow.Identifier) - var setup flow.EpochSetup - return func(tx *badger.Txn) (interface{}, error) { + retrieve := func(id flow.Identifier) func(*badger.Txn) (*flow.EpochSetup, error) { + return func(tx *badger.Txn) (*flow.EpochSetup, error) { + var setup flow.EpochSetup err := operation.RetrieveEpochSetup(id, &setup)(tx) return &setup, err } @@ -35,8 +32,8 @@ func NewEpochSetups(collector module.CacheMetrics, db *badger.DB) *EpochSetups { es := &EpochSetups{ db: db, - cache: newCache(collector, metrics.ResourceEpochSetup, - withLimit(4*flow.DefaultTransactionExpiry), + cache: newCache[flow.Identifier, *flow.EpochSetup](collector, metrics.ResourceEpochSetup, + withLimit[flow.Identifier, *flow.EpochSetup](4*flow.DefaultTransactionExpiry), withStore(store), withRetrieve(retrieve)), } @@ -54,7 +51,7 @@ func (es *EpochSetups) retrieveTx(setupID flow.Identifier) func(tx *badger.Txn) if err != nil { return nil, err } - return val.(*flow.EpochSetup), nil + return val, nil } } diff --git a/storage/badger/epoch_statuses.go b/storage/badger/epoch_statuses.go index e5be69ab080..2d64fcfea8f 100644 --- a/storage/badger/epoch_statuses.go +++ b/storage/badger/epoch_statuses.go @@ -12,22 +12,19 @@ import ( type EpochStatuses struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.EpochStatus] } // NewEpochStatuses ... func NewEpochStatuses(collector module.CacheMetrics, db *badger.DB) *EpochStatuses { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - blockID := key.(flow.Identifier) - status := val.(*flow.EpochStatus) + store := func(blockID flow.Identifier, status *flow.EpochStatus) func(*transaction.Tx) error { return transaction.WithTx(operation.InsertEpochStatus(blockID, status)) } - retrieve := func(key interface{}) func(*badger.Txn) (interface{}, error) { - blockID := key.(flow.Identifier) - var status flow.EpochStatus - return func(tx *badger.Txn) (interface{}, error) { + retrieve := func(blockID flow.Identifier) func(*badger.Txn) (*flow.EpochStatus, error) { + return func(tx *badger.Txn) (*flow.EpochStatus, error) { + var status flow.EpochStatus err := operation.RetrieveEpochStatus(blockID, &status)(tx) return &status, err } @@ -35,8 +32,8 @@ func NewEpochStatuses(collector module.CacheMetrics, db *badger.DB) *EpochStatus es := &EpochStatuses{ db: db, - cache: newCache(collector, metrics.ResourceEpochStatus, - withLimit(4*flow.DefaultTransactionExpiry), + cache: newCache[flow.Identifier, *flow.EpochStatus](collector, metrics.ResourceEpochStatus, + withLimit[flow.Identifier, *flow.EpochStatus](4*flow.DefaultTransactionExpiry), withStore(store), withRetrieve(retrieve)), } @@ -54,7 +51,7 @@ func (es *EpochStatuses) retrieveTx(blockID flow.Identifier) func(tx *badger.Txn if err != nil { return nil, err } - return val.(*flow.EpochStatus), nil + return val, nil } } diff --git a/storage/badger/events.go b/storage/badger/events.go index 59112f01e36..f2d0242c9e7 100644 --- a/storage/badger/events.go +++ b/storage/badger/events.go @@ -14,14 +14,13 @@ import ( type Events struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, []flow.Event] } func NewEvents(collector module.CacheMetrics, db *badger.DB) *Events { - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - blockID := key.(flow.Identifier) + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) ([]flow.Event, error) { var events []flow.Event - return func(tx *badger.Txn) (interface{}, error) { + return func(tx *badger.Txn) ([]flow.Event, error) { err := operation.LookupEventsByBlockID(blockID, &events)(tx) return events, handleError(err, flow.Event{}) } @@ -29,8 +28,8 @@ func NewEvents(collector module.CacheMetrics, db *badger.DB) *Events { return &Events{ db: db, - cache: newCache(collector, metrics.ResourceEvents, - withStore(noopStore), + cache: newCache[flow.Identifier, []flow.Event](collector, metrics.ResourceEvents, + withStore(noopStore[flow.Identifier, []flow.Event]), withRetrieve(retrieve)), } } @@ -77,7 +76,7 @@ func (e *Events) ByBlockID(blockID flow.Identifier) ([]flow.Event, error) { if err != nil { return nil, err } - return val.([]flow.Event), nil + return val, nil } // ByBlockIDTransactionID returns the events for the given block ID and transaction ID @@ -142,14 +141,13 @@ func (e *Events) BatchRemoveByBlockID(blockID flow.Identifier, batch storage.Bat type ServiceEvents struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, []flow.Event] } func NewServiceEvents(collector module.CacheMetrics, db *badger.DB) *ServiceEvents { - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - blockID := key.(flow.Identifier) + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) ([]flow.Event, error) { var events []flow.Event - return func(tx *badger.Txn) (interface{}, error) { + return func(tx *badger.Txn) ([]flow.Event, error) { err := operation.LookupServiceEventsByBlockID(blockID, &events)(tx) return events, handleError(err, flow.Event{}) } @@ -157,8 +155,8 @@ func NewServiceEvents(collector module.CacheMetrics, db *badger.DB) *ServiceEven return &ServiceEvents{ db: db, - cache: newCache(collector, metrics.ResourceEvents, - withStore(noopStore), + cache: newCache[flow.Identifier, []flow.Event](collector, metrics.ResourceEvents, + withStore(noopStore[flow.Identifier, []flow.Event]), withRetrieve(retrieve)), } } @@ -190,7 +188,7 @@ func (e *ServiceEvents) ByBlockID(blockID flow.Identifier) ([]flow.Event, error) if err != nil { return nil, err } - return val.([]flow.Event), nil + return val, nil } // RemoveByBlockID removes service events by block ID diff --git a/storage/badger/guarantees.go b/storage/badger/guarantees.go index 23bc929db9c..b7befd342b6 100644 --- a/storage/badger/guarantees.go +++ b/storage/badger/guarantees.go @@ -13,21 +13,18 @@ import ( // Guarantees implements persistent storage for collection guarantees. type Guarantees struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.CollectionGuarantee] } func NewGuarantees(collector module.CacheMetrics, db *badger.DB, cacheSize uint) *Guarantees { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - collID := key.(flow.Identifier) - guarantee := val.(*flow.CollectionGuarantee) + store := func(collID flow.Identifier, guarantee *flow.CollectionGuarantee) func(*transaction.Tx) error { return transaction.WithTx(operation.SkipDuplicates(operation.InsertGuarantee(collID, guarantee))) } - retrieve := func(key interface{}) func(*badger.Txn) (interface{}, error) { - collID := key.(flow.Identifier) + retrieve := func(collID flow.Identifier) func(*badger.Txn) (*flow.CollectionGuarantee, error) { var guarantee flow.CollectionGuarantee - return func(tx *badger.Txn) (interface{}, error) { + return func(tx *badger.Txn) (*flow.CollectionGuarantee, error) { err := operation.RetrieveGuarantee(collID, &guarantee)(tx) return &guarantee, err } @@ -35,8 +32,8 @@ func NewGuarantees(collector module.CacheMetrics, db *badger.DB, cacheSize uint) g := &Guarantees{ db: db, - cache: newCache(collector, metrics.ResourceGuarantee, - withLimit(cacheSize), + cache: newCache[flow.Identifier, *flow.CollectionGuarantee](collector, metrics.ResourceGuarantee, + withLimit[flow.Identifier, *flow.CollectionGuarantee](cacheSize), withStore(store), withRetrieve(retrieve)), } @@ -54,7 +51,7 @@ func (g *Guarantees) retrieveTx(collID flow.Identifier) func(*badger.Txn) (*flow if err != nil { return nil, err } - return val.(*flow.CollectionGuarantee), nil + return val, nil } } diff --git a/storage/badger/headers.go b/storage/badger/headers.go index 90725af1c10..bfdaaa320df 100644 --- a/storage/badger/headers.go +++ b/storage/badger/headers.go @@ -10,7 +10,6 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/operation" "github.com/onflow/flow-go/storage/badger/procedure" "github.com/onflow/flow-go/storage/badger/transaction" @@ -18,76 +17,50 @@ import ( // Headers implements a simple read-only header storage around a badger DB. type Headers struct { - db *badger.DB - cache *Cache - heightCache *Cache - chunkIDCache *Cache + db *badger.DB + cache *Cache[flow.Identifier, *flow.Header] + heightCache *Cache[uint64, flow.Identifier] } func NewHeaders(collector module.CacheMetrics, db *badger.DB) *Headers { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - blockID := key.(flow.Identifier) - header := val.(*flow.Header) + store := func(blockID flow.Identifier, header *flow.Header) func(*transaction.Tx) error { return transaction.WithTx(operation.InsertHeader(blockID, header)) } // CAUTION: should only be used to index FINALIZED blocks by their // respective height - storeHeight := func(key interface{}, val interface{}) func(*transaction.Tx) error { - height := key.(uint64) - id := val.(flow.Identifier) + storeHeight := func(height uint64, id flow.Identifier) func(*transaction.Tx) error { return transaction.WithTx(operation.IndexBlockHeight(height, id)) } - storeChunkID := func(key interface{}, val interface{}) func(*transaction.Tx) error { - chunkID := key.(flow.Identifier) - blockID := val.(flow.Identifier) - return transaction.WithTx(operation.IndexBlockIDByChunkID(chunkID, blockID)) - } - - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - blockID := key.(flow.Identifier) + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (*flow.Header, error) { var header flow.Header - return func(tx *badger.Txn) (interface{}, error) { + return func(tx *badger.Txn) (*flow.Header, error) { err := operation.RetrieveHeader(blockID, &header)(tx) return &header, err } } - retrieveHeight := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - height := key.(uint64) - var id flow.Identifier - return func(tx *badger.Txn) (interface{}, error) { + retrieveHeight := func(height uint64) func(tx *badger.Txn) (flow.Identifier, error) { + return func(tx *badger.Txn) (flow.Identifier, error) { + var id flow.Identifier err := operation.LookupBlockHeight(height, &id)(tx) return id, err } } - retrieveChunkID := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - chunkID := key.(flow.Identifier) - var blockID flow.Identifier - return func(tx *badger.Txn) (interface{}, error) { - err := operation.LookupBlockIDByChunkID(chunkID, &blockID)(tx) - return blockID, err - } - } - h := &Headers{ db: db, - cache: newCache(collector, metrics.ResourceHeader, - withLimit(4*flow.DefaultTransactionExpiry), + cache: newCache[flow.Identifier, *flow.Header](collector, metrics.ResourceHeader, + withLimit[flow.Identifier, *flow.Header](4*flow.DefaultTransactionExpiry), withStore(store), withRetrieve(retrieve)), - heightCache: newCache(collector, metrics.ResourceFinalizedHeight, - withLimit(4*flow.DefaultTransactionExpiry), + heightCache: newCache[uint64, flow.Identifier](collector, metrics.ResourceFinalizedHeight, + withLimit[uint64, flow.Identifier](4*flow.DefaultTransactionExpiry), withStore(storeHeight), withRetrieve(retrieveHeight)), - chunkIDCache: newCache(collector, metrics.ResourceFinalizedHeight, - withLimit(4*flow.DefaultTransactionExpiry), - withStore(storeChunkID), - withRetrieve(retrieveChunkID)), } return h @@ -103,7 +76,7 @@ func (h *Headers) retrieveTx(blockID flow.Identifier) func(*badger.Txn) (*flow.H if err != nil { return nil, err } - return val.(*flow.Header), nil + return val, nil } } @@ -114,7 +87,7 @@ func (h *Headers) retrieveIdByHeightTx(height uint64) func(*badger.Txn) (flow.Id if err != nil { return flow.ZeroID, fmt.Errorf("failed to retrieve block ID for height %d: %w", height, err) } - return blockID.(flow.Identifier), nil + return blockID, nil } } @@ -155,8 +128,8 @@ func (h *Headers) Exists(blockID flow.Identifier) (bool, error) { return exists, nil } -// BlockIDByHeight the block ID that is finalized at the given height. It is an optimized version -// of `ByHeight` that skips retrieving the block. Expected errors during normal operations: +// BlockIDByHeight returns the block ID that is finalized at the given height. It is an optimized +// version of `ByHeight` that skips retrieving the block. Expected errors during normal operations: // - `storage.ErrNotFound` if no finalized block is known at given height. func (h *Headers) BlockIDByHeight(height uint64) (flow.Identifier, error) { tx := h.db.NewTransaction(false) @@ -192,38 +165,6 @@ func (h *Headers) FindHeaders(filter func(header *flow.Header) bool) ([]flow.Hea return blocks, err } -func (h *Headers) IDByChunkID(chunkID flow.Identifier) (flow.Identifier, error) { - tx := h.db.NewTransaction(false) - defer tx.Discard() - - bID, err := h.chunkIDCache.Get(chunkID)(tx) - if err != nil { - return flow.Identifier{}, fmt.Errorf("could not look up by chunk id: %w", err) - } - return bID.(flow.Identifier), nil -} - -func (h *Headers) IndexByChunkID(headerID, chunkID flow.Identifier) error { - return operation.RetryOnConflictTx(h.db, transaction.Update, h.chunkIDCache.PutTx(chunkID, headerID)) -} - -func (h *Headers) BatchIndexByChunkID(headerID, chunkID flow.Identifier, batch storage.BatchStorage) error { - writeBatch := batch.GetWriter() - return operation.BatchIndexBlockByChunkID(headerID, chunkID)(writeBatch) -} - -func (h *Headers) RemoveChunkBlockIndexByChunkID(chunkID flow.Identifier) error { - return h.db.Update(operation.RemoveBlockIDByChunkID(chunkID)) -} - -// BatchRemoveChunkBlockIndexByChunkID removes block to chunk index entry keyed by a blockID in a provided batch -// No errors are expected during normal operation, even if no entries are matched. -// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. -func (h *Headers) BatchRemoveChunkBlockIndexByChunkID(chunkID flow.Identifier, batch storage.BatchStorage) error { - writeBatch := batch.GetWriter() - return operation.BatchRemoveBlockIDByChunkID(chunkID)(writeBatch) -} - // RollbackExecutedBlock update the executed block header to the given header. // only useful for execution node to roll back executed block height func (h *Headers) RollbackExecutedBlock(header *flow.Header) error { diff --git a/storage/badger/index.go b/storage/badger/index.go index 4a5b4ba32b6..49d87b928da 100644 --- a/storage/badger/index.go +++ b/storage/badger/index.go @@ -16,21 +16,18 @@ import ( // Index implements a simple read-only payload storage around a badger DB. type Index struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.Index] } func NewIndex(collector module.CacheMetrics, db *badger.DB) *Index { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - blockID := key.(flow.Identifier) - index := val.(*flow.Index) + store := func(blockID flow.Identifier, index *flow.Index) func(*transaction.Tx) error { return transaction.WithTx(procedure.InsertIndex(blockID, index)) } - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - blockID := key.(flow.Identifier) + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (*flow.Index, error) { var index flow.Index - return func(tx *badger.Txn) (interface{}, error) { + return func(tx *badger.Txn) (*flow.Index, error) { err := procedure.RetrieveIndex(blockID, &index)(tx) return &index, err } @@ -38,8 +35,8 @@ func NewIndex(collector module.CacheMetrics, db *badger.DB) *Index { p := &Index{ db: db, - cache: newCache(collector, metrics.ResourceIndex, - withLimit(flow.DefaultTransactionExpiry+100), + cache: newCache[flow.Identifier, *flow.Index](collector, metrics.ResourceIndex, + withLimit[flow.Identifier, *flow.Index](flow.DefaultTransactionExpiry+100), withStore(store), withRetrieve(retrieve)), } @@ -57,7 +54,7 @@ func (i *Index) retrieveTx(blockID flow.Identifier) func(*badger.Txn) (*flow.Ind if err != nil { return nil, err } - return val.(*flow.Index), nil + return val, nil } } diff --git a/storage/badger/my_receipts.go b/storage/badger/my_receipts.go index 37054a35145..ff1584f44d6 100644 --- a/storage/badger/my_receipts.go +++ b/storage/badger/my_receipts.go @@ -21,14 +21,13 @@ import ( type MyExecutionReceipts struct { genericReceipts *ExecutionReceipts db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.ExecutionReceipt] } // NewMyExecutionReceipts creates instance of MyExecutionReceipts which is a wrapper wrapper around badger.ExecutionReceipts // It's useful for execution nodes to keep track of produced execution receipts. func NewMyExecutionReceipts(collector module.CacheMetrics, db *badger.DB, receipts *ExecutionReceipts) *MyExecutionReceipts { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - receipt := val.(*flow.ExecutionReceipt) + store := func(key flow.Identifier, receipt *flow.ExecutionReceipt) func(*transaction.Tx) error { // assemble DB operations to store receipt (no execution) storeReceiptOps := receipts.storeTx(receipt) // assemble DB operations to index receipt as one of my own (no execution) @@ -68,10 +67,8 @@ func NewMyExecutionReceipts(collector module.CacheMetrics, db *badger.DB, receip } } - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - blockID := key.(flow.Identifier) - - return func(tx *badger.Txn) (interface{}, error) { + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (*flow.ExecutionReceipt, error) { + return func(tx *badger.Txn) (*flow.ExecutionReceipt, error) { var receiptID flow.Identifier err := operation.LookupOwnExecutionReceipt(blockID, &receiptID)(tx) if err != nil { @@ -88,8 +85,8 @@ func NewMyExecutionReceipts(collector module.CacheMetrics, db *badger.DB, receip return &MyExecutionReceipts{ genericReceipts: receipts, db: db, - cache: newCache(collector, metrics.ResourceMyReceipt, - withLimit(flow.DefaultTransactionExpiry+100), + cache: newCache[flow.Identifier, *flow.ExecutionReceipt](collector, metrics.ResourceMyReceipt, + withLimit[flow.Identifier, *flow.ExecutionReceipt](flow.DefaultTransactionExpiry+100), withStore(store), withRetrieve(retrieve)), } @@ -108,7 +105,7 @@ func (m *MyExecutionReceipts) myReceipt(blockID flow.Identifier) func(*badger.Tx if err != nil { return nil, err } - return val.(*flow.ExecutionReceipt), nil + return val, nil } } diff --git a/storage/badger/operation/cluster.go b/storage/badger/operation/cluster.go index fdf80d30db2..8163285c62f 100644 --- a/storage/badger/operation/cluster.go +++ b/storage/badger/operation/cluster.go @@ -66,10 +66,11 @@ func IndexClusterBlockByReferenceHeight(refHeight uint64, clusterBlockID flow.Id func LookupClusterBlocksByReferenceHeightRange(start, end uint64, clusterBlockIDs *[]flow.Identifier) func(*badger.Txn) error { startPrefix := makePrefix(codeRefHeightToClusterBlock, start) endPrefix := makePrefix(codeRefHeightToClusterBlock, end) + prefixLen := len(startPrefix) return iterate(startPrefix, endPrefix, func() (checkFunc, createFunc, handleFunc) { check := func(key []byte) bool { - clusterBlockIDBytes := key[9:] + clusterBlockIDBytes := key[prefixLen:] var clusterBlockID flow.Identifier copy(clusterBlockID[:], clusterBlockIDBytes) *clusterBlockIDs = append(*clusterBlockIDs, clusterBlockID) diff --git a/storage/badger/operation/common.go b/storage/badger/operation/common.go index 97dddb91d12..6dbe96224b4 100644 --- a/storage/badger/operation/common.go +++ b/storage/badger/operation/common.go @@ -521,6 +521,43 @@ func traverse(prefix []byte, iteration iterationFunc) func(*badger.Txn) error { } } +// findHighestAtOrBelow searches for the highest key with the given prefix and a height +// at or below the target height, and retrieves and decodes the value associated with the +// key into the given entity. +// If no key is found, the function returns storage.ErrNotFound. +func findHighestAtOrBelow( + prefix []byte, + height uint64, + entity interface{}, +) func(*badger.Txn) error { + return func(tx *badger.Txn) error { + if len(prefix) == 0 { + return fmt.Errorf("prefix must not be empty") + } + + opts := badger.DefaultIteratorOptions + opts.Prefix = prefix + opts.Reverse = true + + it := tx.NewIterator(opts) + defer it.Close() + + it.Seek(append(prefix, b(height)...)) + + if !it.Valid() { + return storage.ErrNotFound + } + + return it.Item().Value(func(val []byte) error { + err := msgpack.Unmarshal(val, entity) + if err != nil { + return fmt.Errorf("could not decode entity: %w", err) + } + return nil + }) + } +} + // Fail returns a DB operation function that always fails with the given error. func Fail(err error) func(*badger.Txn) error { return func(_ *badger.Txn) error { diff --git a/storage/badger/operation/common_test.go b/storage/badger/operation/common_test.go index ebef5aef45d..65f64fbd5cb 100644 --- a/storage/badger/operation/common_test.go +++ b/storage/badger/operation/common_test.go @@ -5,10 +5,8 @@ package operation import ( "bytes" "fmt" - "math/rand" "reflect" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -20,10 +18,6 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - type Entity struct { ID uint64 } @@ -614,3 +608,97 @@ func TestIterateBoundaries(t *testing.T) { assert.ElementsMatch(t, keysInRange, found, "backward iteration should go over correct keys") }) } + +func TestFindHighestAtOrBelow(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + prefix := []byte("test_prefix") + + type Entity struct { + Value uint64 + } + + entity1 := Entity{Value: 41} + entity2 := Entity{Value: 42} + entity3 := Entity{Value: 43} + + err := db.Update(func(tx *badger.Txn) error { + key := append(prefix, b(uint64(15))...) + val, err := msgpack.Marshal(entity3) + if err != nil { + return err + } + err = tx.Set(key, val) + if err != nil { + return err + } + + key = append(prefix, b(uint64(5))...) + val, err = msgpack.Marshal(entity1) + if err != nil { + return err + } + err = tx.Set(key, val) + if err != nil { + return err + } + + key = append(prefix, b(uint64(10))...) + val, err = msgpack.Marshal(entity2) + if err != nil { + return err + } + err = tx.Set(key, val) + if err != nil { + return err + } + return nil + }) + require.NoError(t, err) + + var entity Entity + + t.Run("target height exists", func(t *testing.T) { + err = findHighestAtOrBelow( + prefix, + 10, + &entity)(db.NewTransaction(false)) + require.NoError(t, err) + require.Equal(t, uint64(42), entity.Value) + }) + + t.Run("target height above", func(t *testing.T) { + err = findHighestAtOrBelow( + prefix, + 11, + &entity)(db.NewTransaction(false)) + require.NoError(t, err) + require.Equal(t, uint64(42), entity.Value) + }) + + t.Run("target height above highest", func(t *testing.T) { + err = findHighestAtOrBelow( + prefix, + 20, + &entity)(db.NewTransaction(false)) + require.NoError(t, err) + require.Equal(t, uint64(43), entity.Value) + }) + + t.Run("target height below lowest", func(t *testing.T) { + err = findHighestAtOrBelow( + prefix, + 4, + &entity)(db.NewTransaction(false)) + require.ErrorIs(t, err, storage.ErrNotFound) + }) + + t.Run("empty prefix", func(t *testing.T) { + err = findHighestAtOrBelow( + []byte{}, + 5, + &entity)(db.NewTransaction(false)) + require.Error(t, err) + require.Contains(t, err.Error(), "prefix must not be empty") + }) + }) +} diff --git a/storage/badger/operation/computation_result_test.go b/storage/badger/operation/computation_result_test.go index e8d8d8e027f..79336a87964 100644 --- a/storage/badger/operation/computation_result_test.go +++ b/storage/badger/operation/computation_result_test.go @@ -9,18 +9,15 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/engine/execution" - "github.com/onflow/flow-go/ledger" - "github.com/onflow/flow-go/ledger/common/pathfinder" - "github.com/onflow/flow-go/ledger/complete" + "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/unittest" ) func TestInsertAndUpdateAndRetrieveComputationResultUpdateStatus(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { - expected := generateComputationResult(t) + expected := testutil.ComputationResultFixture(t) expectedId := expected.ExecutableBlock.ID() t.Run("Update existing ComputationResult", func(t *testing.T) { @@ -60,7 +57,7 @@ func TestInsertAndUpdateAndRetrieveComputationResultUpdateStatus(t *testing.T) { func TestUpsertAndRetrieveComputationResultUpdateStatus(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { - expected := generateComputationResult(t) + expected := testutil.ComputationResultFixture(t) expectedId := expected.ExecutableBlock.ID() t.Run("Upsert ComputationResult", func(t *testing.T) { @@ -92,7 +89,7 @@ func TestUpsertAndRetrieveComputationResultUpdateStatus(t *testing.T) { func TestRemoveComputationResultUploadStatus(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { - expected := generateComputationResult(t) + expected := testutil.ComputationResultFixture(t) expectedId := expected.ExecutableBlock.ID() t.Run("Remove ComputationResult", func(t *testing.T) { @@ -119,8 +116,8 @@ func TestRemoveComputationResultUploadStatus(t *testing.T) { func TestListComputationResults(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { expected := [...]*execution.ComputationResult{ - generateComputationResult(t), - generateComputationResult(t), + testutil.ComputationResultFixture(t), + testutil.ComputationResultFixture(t), } t.Run("List all ComputationResult with status True", func(t *testing.T) { expectedIDs := make(map[string]bool, 0) @@ -145,137 +142,3 @@ func TestListComputationResults(t *testing.T) { }) }) } - -// Generate ComputationResult for testing purposes -func generateComputationResult(t *testing.T) *execution.ComputationResult { - - update1, err := ledger.NewUpdate( - ledger.State(unittest.StateCommitmentFixture()), - []ledger.Key{ - ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(3, []byte{33})}), - ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(1, []byte{11})}), - ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(2, []byte{1, 1}), ledger.NewKeyPart(3, []byte{2, 5})}), - }, - []ledger.Value{ - []byte{21, 37}, - nil, - []byte{3, 3, 3, 3, 3}, - }, - ) - require.NoError(t, err) - - trieUpdate1, err := pathfinder.UpdateToTrieUpdate(update1, complete.DefaultPathFinderVersion) - require.NoError(t, err) - - update2, err := ledger.NewUpdate( - ledger.State(unittest.StateCommitmentFixture()), - []ledger.Key{}, - []ledger.Value{}, - ) - require.NoError(t, err) - - trieUpdate2, err := pathfinder.UpdateToTrieUpdate(update2, complete.DefaultPathFinderVersion) - require.NoError(t, err) - - update3, err := ledger.NewUpdate( - ledger.State(unittest.StateCommitmentFixture()), - []ledger.Key{ - ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(9, []byte{6})}), - }, - []ledger.Value{ - []byte{21, 37}, - }, - ) - require.NoError(t, err) - - trieUpdate3, err := pathfinder.UpdateToTrieUpdate(update3, complete.DefaultPathFinderVersion) - require.NoError(t, err) - - update4, err := ledger.NewUpdate( - ledger.State(unittest.StateCommitmentFixture()), - []ledger.Key{ - ledger.NewKey([]ledger.KeyPart{ledger.NewKeyPart(9, []byte{6})}), - }, - []ledger.Value{ - []byte{21, 37}, - }, - ) - require.NoError(t, err) - - trieUpdate4, err := pathfinder.UpdateToTrieUpdate(update4, complete.DefaultPathFinderVersion) - require.NoError(t, err) - - return &execution.ComputationResult{ - ExecutableBlock: unittest.ExecutableBlockFixture([][]flow.Identifier{ - {unittest.IdentifierFixture()}, - {unittest.IdentifierFixture()}, - {unittest.IdentifierFixture()}, - }), - StateSnapshots: nil, - Events: []flow.EventsList{ - { - unittest.EventFixture("what", 0, 0, unittest.IdentifierFixture(), 2), - unittest.EventFixture("ever", 0, 1, unittest.IdentifierFixture(), 22), - }, - {}, - { - unittest.EventFixture("what", 2, 0, unittest.IdentifierFixture(), 2), - unittest.EventFixture("ever", 2, 1, unittest.IdentifierFixture(), 22), - unittest.EventFixture("ever", 2, 2, unittest.IdentifierFixture(), 2), - unittest.EventFixture("ever", 2, 3, unittest.IdentifierFixture(), 22), - }, - {}, // system chunk events - }, - EventsHashes: nil, - ServiceEvents: nil, - TransactionResults: []flow.TransactionResult{ - { - TransactionID: unittest.IdentifierFixture(), - ErrorMessage: "", - ComputationUsed: 23, - MemoryUsed: 101, - }, - { - TransactionID: unittest.IdentifierFixture(), - ErrorMessage: "fail", - ComputationUsed: 1, - MemoryUsed: 22, - }, - }, - TransactionResultIndex: []int{1, 1, 2, 2}, - BlockExecutionData: &execution_data.BlockExecutionData{ - ChunkExecutionDatas: []*execution_data.ChunkExecutionData{ - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate1, - }, - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate2, - }, - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate3, - }, - &execution_data.ChunkExecutionData{ - TrieUpdate: trieUpdate4, - }, - }, - }, - ExecutionReceipt: &flow.ExecutionReceipt{ - ExecutionResult: flow.ExecutionResult{ - Chunks: flow.ChunkList{ - { - EndState: unittest.StateCommitmentFixture(), - }, - { - EndState: unittest.StateCommitmentFixture(), - }, - { - EndState: unittest.StateCommitmentFixture(), - }, - { - EndState: unittest.StateCommitmentFixture(), - }, - }, - }, - }, - } -} diff --git a/storage/badger/operation/headers.go b/storage/badger/operation/headers.go index 78af538801a..bd1c377cc16 100644 --- a/storage/badger/operation/headers.go +++ b/storage/badger/operation/headers.go @@ -50,37 +50,11 @@ func IndexCollectionBlock(collID flow.Identifier, blockID flow.Identifier) func( return insert(makePrefix(codeCollectionBlock, collID), blockID) } -func IndexBlockIDByChunkID(chunkID, blockID flow.Identifier) func(*badger.Txn) error { - return insert(makePrefix(codeIndexBlockByChunkID, chunkID), blockID) -} - -// BatchIndexBlockByChunkID indexes blockID by chunkID into a batch -func BatchIndexBlockByChunkID(blockID, chunkID flow.Identifier) func(batch *badger.WriteBatch) error { - return batchWrite(makePrefix(codeIndexBlockByChunkID, chunkID), blockID) -} - // LookupCollectionBlock looks up a block by a collection within that block. func LookupCollectionBlock(collID flow.Identifier, blockID *flow.Identifier) func(*badger.Txn) error { return retrieve(makePrefix(codeCollectionBlock, collID), blockID) } -// LookupBlockIDByChunkID looks up a block by a collection within that block. -func LookupBlockIDByChunkID(chunkID flow.Identifier, blockID *flow.Identifier) func(*badger.Txn) error { - return retrieve(makePrefix(codeIndexBlockByChunkID, chunkID), blockID) -} - -// RemoveBlockIDByChunkID removes chunkID-blockID index by chunkID -func RemoveBlockIDByChunkID(chunkID flow.Identifier) func(*badger.Txn) error { - return remove(makePrefix(codeIndexBlockByChunkID, chunkID)) -} - -// BatchRemoveBlockIDByChunkID removes chunkID-to-blockID index entries keyed by a chunkID in a provided batch. -// No errors are expected during normal operation, even if no entries are matched. -// If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. -func BatchRemoveBlockIDByChunkID(chunkID flow.Identifier) func(batch *badger.WriteBatch) error { - return batchRemove(makePrefix(codeIndexBlockByChunkID, chunkID)) -} - // FindHeaders iterates through all headers, calling `filter` on each, and adding // them to the `found` slice if `filter` returned true func FindHeaders(filter func(header *flow.Header) bool, found *[]flow.Header) func(*badger.Txn) error { diff --git a/storage/badger/operation/heights.go b/storage/badger/operation/heights.go index 5741b03fa6b..0c6573ab24c 100644 --- a/storage/badger/operation/heights.go +++ b/storage/badger/operation/heights.go @@ -7,11 +7,19 @@ import ( ) func InsertRootHeight(height uint64) func(*badger.Txn) error { - return insert(makePrefix(codeRootHeight), height) + return insert(makePrefix(codeFinalizedRootHeight), height) } func RetrieveRootHeight(height *uint64) func(*badger.Txn) error { - return retrieve(makePrefix(codeRootHeight), height) + return retrieve(makePrefix(codeFinalizedRootHeight), height) +} + +func InsertSealedRootHeight(height uint64) func(*badger.Txn) error { + return insert(makePrefix(codeSealedRootHeight), height) +} + +func RetrieveSealedRootHeight(height *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeSealedRootHeight), height) } func InsertFinalizedHeight(height uint64) func(*badger.Txn) error { @@ -52,6 +60,20 @@ func RetrieveEpochFirstHeight(epoch uint64, height *uint64) func(*badger.Txn) er return retrieve(makePrefix(codeEpochFirstHeight, epoch), height) } +// RetrieveEpochLastHeight retrieves the height of the last block in the given epoch. +// It's a more readable, but equivalent query to RetrieveEpochFirstHeight when interested in the last height of an epoch. +// Returns storage.ErrNotFound if the first block of the epoch has not yet been finalized. +func RetrieveEpochLastHeight(epoch uint64, height *uint64) func(*badger.Txn) error { + var nextEpochFirstHeight uint64 + return func(tx *badger.Txn) error { + if err := retrieve(makePrefix(codeEpochFirstHeight, epoch+1), &nextEpochFirstHeight)(tx); err != nil { + return err + } + *height = nextEpochFirstHeight - 1 + return nil + } +} + // InsertLastCompleteBlockHeightIfNotExists inserts the last full block height if it is not already set. // Calling this function multiple times is a no-op and returns no expected errors. func InsertLastCompleteBlockHeightIfNotExists(height uint64) func(*badger.Txn) error { diff --git a/storage/badger/operation/interactions.go b/storage/badger/operation/interactions.go index 671c822e51b..952b2f7a188 100644 --- a/storage/badger/operation/interactions.go +++ b/storage/badger/operation/interactions.go @@ -1,7 +1,7 @@ package operation import ( - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/dgraph-io/badger/v2" @@ -9,7 +9,7 @@ import ( func InsertExecutionStateInteractions( blockID flow.Identifier, - executionSnapshots []*state.ExecutionSnapshot, + executionSnapshots []*snapshot.ExecutionSnapshot, ) func(*badger.Txn) error { return insert( makePrefix(codeExecutionStateInteractions, blockID), @@ -18,7 +18,7 @@ func InsertExecutionStateInteractions( func RetrieveExecutionStateInteractions( blockID flow.Identifier, - executionSnapshots *[]*state.ExecutionSnapshot, + executionSnapshots *[]*snapshot.ExecutionSnapshot, ) func(*badger.Txn) error { return retrieve( makePrefix(codeExecutionStateInteractions, blockID), executionSnapshots) diff --git a/storage/badger/operation/interactions_test.go b/storage/badger/operation/interactions_test.go index c8b808a6fc2..fd334c3a6b8 100644 --- a/storage/badger/operation/interactions_test.go +++ b/storage/badger/operation/interactions_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -23,7 +23,7 @@ func TestStateInteractionsInsertCheckRetrieve(t *testing.T) { id2 := flow.NewRegisterID(string([]byte{2}), "") id3 := flow.NewRegisterID(string([]byte{3}), "") - snapshot := &state.ExecutionSnapshot{ + executionSnapshot := &snapshot.ExecutionSnapshot{ ReadSet: map[flow.RegisterID]struct{}{ id2: struct{}{}, id3: struct{}{}, @@ -34,9 +34,9 @@ func TestStateInteractionsInsertCheckRetrieve(t *testing.T) { }, } - interactions := []*state.ExecutionSnapshot{ - snapshot, - &state.ExecutionSnapshot{}, + interactions := []*snapshot.ExecutionSnapshot{ + executionSnapshot, + &snapshot.ExecutionSnapshot{}, } blockID := unittest.IdentifierFixture() @@ -44,13 +44,19 @@ func TestStateInteractionsInsertCheckRetrieve(t *testing.T) { err := db.Update(InsertExecutionStateInteractions(blockID, interactions)) require.Nil(t, err) - var readInteractions []*state.ExecutionSnapshot + var readInteractions []*snapshot.ExecutionSnapshot err = db.View(RetrieveExecutionStateInteractions(blockID, &readInteractions)) require.NoError(t, err) assert.Equal(t, interactions, readInteractions) - assert.Equal(t, snapshot.WriteSet, readInteractions[0].WriteSet) - assert.Equal(t, snapshot.ReadSet, readInteractions[0].ReadSet) + assert.Equal( + t, + executionSnapshot.WriteSet, + readInteractions[0].WriteSet) + assert.Equal( + t, + executionSnapshot.ReadSet, + readInteractions[0].ReadSet) }) } diff --git a/storage/badger/operation/prefix.go b/storage/badger/operation/prefix.go index e2b5752fc39..e75497257ca 100644 --- a/storage/badger/operation/prefix.go +++ b/storage/badger/operation/prefix.go @@ -30,9 +30,10 @@ const ( codeSealedHeight = 21 // latest sealed block height codeClusterHeight = 22 // latest finalized height on cluster codeExecutedBlock = 23 // latest executed block with max height - codeRootHeight = 24 // the height of the highest block contained in the root snapshot + codeFinalizedRootHeight = 24 // the height of the highest finalized block contained in the root snapshot codeLastCompleteBlockHeight = 25 // the height of the last block for which all collections were received codeEpochFirstHeight = 26 // the height of the first block in a given epoch + codeSealedRootHeight = 27 // the height of the highest sealed block contained in the root snapshot // codes for single entity storage // 31 was used for identities before epochs @@ -56,23 +57,23 @@ const ( // codes for indexing multiple identifiers by identifier // NOTE: 51 was used for identity indexes before epochs - codeBlockChildren = 50 // index mapping block ID to children blocks - codePayloadGuarantees = 52 // index mapping block ID to payload guarantees - codePayloadSeals = 53 // index mapping block ID to payload seals - codeCollectionBlock = 54 // index mapping collection ID to block ID - codeOwnBlockReceipt = 55 // index mapping block ID to execution receipt ID for execution nodes - codeBlockEpochStatus = 56 // index mapping block ID to epoch status - codePayloadReceipts = 57 // index mapping block ID to payload receipts - codePayloadResults = 58 // index mapping block ID to payload results - codeAllBlockReceipts = 59 // index mapping of blockID to multiple receipts - codeIndexBlockByChunkID = 60 // index mapping chunk ID to block ID - - // codes related to epoch information + codeBlockChildren = 50 // index mapping block ID to children blocks + codePayloadGuarantees = 52 // index mapping block ID to payload guarantees + codePayloadSeals = 53 // index mapping block ID to payload seals + codeCollectionBlock = 54 // index mapping collection ID to block ID + codeOwnBlockReceipt = 55 // index mapping block ID to execution receipt ID for execution nodes + codeBlockEpochStatus = 56 // index mapping block ID to epoch status + codePayloadReceipts = 57 // index mapping block ID to payload receipts + codePayloadResults = 58 // index mapping block ID to payload results + codeAllBlockReceipts = 59 // index mapping of blockID to multiple receipts + + // codes related to protocol level information codeEpochSetup = 61 // EpochSetup service event, keyed by ID codeEpochCommit = 62 // EpochCommit service event, keyed by ID codeBeaconPrivateKey = 63 // BeaconPrivateKey, keyed by epoch counter codeDKGStarted = 64 // flag that the DKG for an epoch has been started codeDKGEnded = 65 // flag that the DKG for an epoch has ended (stores end state) + codeVersionBeacon = 67 // flag for storing version beacons // code for ComputationResult upload status storage // NOTE: for now only GCP uploader is supported. When other uploader (AWS e.g.) needs to diff --git a/storage/badger/operation/version_beacon.go b/storage/badger/operation/version_beacon.go new file mode 100644 index 00000000000..a90ae58e4fb --- /dev/null +++ b/storage/badger/operation/version_beacon.go @@ -0,0 +1,31 @@ +package operation + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" +) + +// IndexVersionBeaconByHeight stores a sealed version beacon indexed by +// flow.SealedVersionBeacon.SealHeight. +// +// No errors are expected during normal operation. +func IndexVersionBeaconByHeight( + beacon *flow.SealedVersionBeacon, +) func(*badger.Txn) error { + return upsert(makePrefix(codeVersionBeacon, beacon.SealHeight), beacon) +} + +// LookupLastVersionBeaconByHeight finds the highest flow.VersionBeacon but no higher +// than maxHeight. Returns storage.ErrNotFound if no version beacon exists at or below +// the given height. +func LookupLastVersionBeaconByHeight( + maxHeight uint64, + versionBeacon *flow.SealedVersionBeacon, +) func(*badger.Txn) error { + return findHighestAtOrBelow( + makePrefix(codeVersionBeacon), + maxHeight, + versionBeacon, + ) +} diff --git a/storage/badger/operation/version_beacon_test.go b/storage/badger/operation/version_beacon_test.go new file mode 100644 index 00000000000..d46ed334f93 --- /dev/null +++ b/storage/badger/operation/version_beacon_test.go @@ -0,0 +1,106 @@ +package operation + +import ( + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestResults_IndexByServiceEvents(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + height1 := uint64(21) + height2 := uint64(37) + height3 := uint64(55) + vb1 := flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + Version: "1.0.0", + BlockHeight: height1 + 5, + }, + ), + ), + SealHeight: height1, + } + vb2 := flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + Version: "1.1.0", + BlockHeight: height2 + 5, + }, + ), + ), + SealHeight: height2, + } + vb3 := flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries( + flow.VersionBoundary{ + Version: "2.0.0", + BlockHeight: height3 + 5, + }, + ), + ), + SealHeight: height3, + } + + // indexing 3 version beacons at different heights + err := db.Update(IndexVersionBeaconByHeight(&vb1)) + require.NoError(t, err) + + err = db.Update(IndexVersionBeaconByHeight(&vb2)) + require.NoError(t, err) + + err = db.Update(IndexVersionBeaconByHeight(&vb3)) + require.NoError(t, err) + + // index version beacon 2 again to make sure we tolerate duplicates + // it is possible for two or more events of the same type to be from the same height + err = db.Update(IndexVersionBeaconByHeight(&vb2)) + require.NoError(t, err) + + t.Run("retrieve exact height match", func(t *testing.T) { + var actualVB flow.SealedVersionBeacon + err := db.View(LookupLastVersionBeaconByHeight(height1, &actualVB)) + require.NoError(t, err) + require.Equal(t, vb1, actualVB) + + err = db.View(LookupLastVersionBeaconByHeight(height2, &actualVB)) + require.NoError(t, err) + require.Equal(t, vb2, actualVB) + + err = db.View(LookupLastVersionBeaconByHeight(height3, &actualVB)) + require.NoError(t, err) + require.Equal(t, vb3, actualVB) + }) + + t.Run("finds highest but not higher than given", func(t *testing.T) { + var actualVB flow.SealedVersionBeacon + + err := db.View(LookupLastVersionBeaconByHeight(height3-1, &actualVB)) + require.NoError(t, err) + require.Equal(t, vb2, actualVB) + }) + + t.Run("finds highest", func(t *testing.T) { + var actualVB flow.SealedVersionBeacon + + err := db.View(LookupLastVersionBeaconByHeight(height3+1, &actualVB)) + require.NoError(t, err) + require.Equal(t, vb3, actualVB) + }) + + t.Run("height below lowest entry returns nothing", func(t *testing.T) { + var actualVB flow.SealedVersionBeacon + + err := db.View(LookupLastVersionBeaconByHeight(height1-1, &actualVB)) + require.ErrorIs(t, err, storage.ErrNotFound) + }) + }) +} diff --git a/storage/badger/qcs.go b/storage/badger/qcs.go index 432a0f8dfd2..856595184d4 100644 --- a/storage/badger/qcs.go +++ b/storage/badger/qcs.go @@ -14,7 +14,7 @@ import ( // QuorumCertificates implements persistent storage for quorum certificates. type QuorumCertificates struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.QuorumCertificate] } var _ storage.QuorumCertificates = (*QuorumCertificates)(nil) @@ -22,15 +22,13 @@ var _ storage.QuorumCertificates = (*QuorumCertificates)(nil) // NewQuorumCertificates Creates QuorumCertificates instance which is a database of quorum certificates // which supports storing, caching and retrieving by block ID. func NewQuorumCertificates(collector module.CacheMetrics, db *badger.DB, cacheSize uint) *QuorumCertificates { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - qc := val.(*flow.QuorumCertificate) + store := func(_ flow.Identifier, qc *flow.QuorumCertificate) func(*transaction.Tx) error { return transaction.WithTx(operation.InsertQuorumCertificate(qc)) } - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - blockID := key.(flow.Identifier) - var qc flow.QuorumCertificate - return func(tx *badger.Txn) (interface{}, error) { + retrieve := func(blockID flow.Identifier) func(tx *badger.Txn) (*flow.QuorumCertificate, error) { + return func(tx *badger.Txn) (*flow.QuorumCertificate, error) { + var qc flow.QuorumCertificate err := operation.RetrieveQuorumCertificate(blockID, &qc)(tx) return &qc, err } @@ -38,8 +36,8 @@ func NewQuorumCertificates(collector module.CacheMetrics, db *badger.DB, cacheSi return &QuorumCertificates{ db: db, - cache: newCache(collector, metrics.ResourceQC, - withLimit(cacheSize), + cache: newCache[flow.Identifier, *flow.QuorumCertificate](collector, metrics.ResourceQC, + withLimit[flow.Identifier, *flow.QuorumCertificate](cacheSize), withStore(store), withRetrieve(retrieve)), } @@ -61,6 +59,6 @@ func (q *QuorumCertificates) retrieveTx(blockID flow.Identifier) func(*badger.Tx if err != nil { return nil, err } - return val.(*flow.QuorumCertificate), nil + return val, nil } } diff --git a/storage/badger/receipts.go b/storage/badger/receipts.go index fb4996b82d3..b92c3961048 100644 --- a/storage/badger/receipts.go +++ b/storage/badger/receipts.go @@ -18,14 +18,13 @@ import ( type ExecutionReceipts struct { db *badger.DB results *ExecutionResults - cache *Cache + cache *Cache[flow.Identifier, *flow.ExecutionReceipt] } // NewExecutionReceipts Creates ExecutionReceipts instance which is a database of receipts which // supports storing and indexing receipts by receipt ID and block ID. func NewExecutionReceipts(collector module.CacheMetrics, db *badger.DB, results *ExecutionResults, cacheSize uint) *ExecutionReceipts { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - receipt := val.(*flow.ExecutionReceipt) + store := func(receiptTD flow.Identifier, receipt *flow.ExecutionReceipt) func(*transaction.Tx) error { receiptID := receipt.ID() // assemble DB operations to store result (no execution) @@ -54,9 +53,8 @@ func NewExecutionReceipts(collector module.CacheMetrics, db *badger.DB, results } } - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - receiptID := key.(flow.Identifier) - return func(tx *badger.Txn) (interface{}, error) { + retrieve := func(receiptID flow.Identifier) func(tx *badger.Txn) (*flow.ExecutionReceipt, error) { + return func(tx *badger.Txn) (*flow.ExecutionReceipt, error) { var meta flow.ExecutionReceiptMeta err := operation.RetrieveExecutionReceiptMeta(receiptID, &meta)(tx) if err != nil { @@ -73,8 +71,8 @@ func NewExecutionReceipts(collector module.CacheMetrics, db *badger.DB, results return &ExecutionReceipts{ db: db, results: results, - cache: newCache(collector, metrics.ResourceReceipt, - withLimit(cacheSize), + cache: newCache[flow.Identifier, *flow.ExecutionReceipt](collector, metrics.ResourceReceipt, + withLimit[flow.Identifier, *flow.ExecutionReceipt](cacheSize), withStore(store), withRetrieve(retrieve)), } @@ -92,7 +90,7 @@ func (r *ExecutionReceipts) byID(receiptID flow.Identifier) func(*badger.Txn) (* if err != nil { return nil, err } - return val.(*flow.ExecutionReceipt), nil + return val, nil } } diff --git a/storage/badger/results.go b/storage/badger/results.go index c6160f5dcb5..d4d1a4525b0 100644 --- a/storage/badger/results.go +++ b/storage/badger/results.go @@ -17,22 +17,20 @@ import ( // ExecutionResults implements persistent storage for execution results. type ExecutionResults struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.ExecutionResult] } var _ storage.ExecutionResults = (*ExecutionResults)(nil) func NewExecutionResults(collector module.CacheMetrics, db *badger.DB) *ExecutionResults { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - result := val.(*flow.ExecutionResult) + store := func(_ flow.Identifier, result *flow.ExecutionResult) func(*transaction.Tx) error { return transaction.WithTx(operation.SkipDuplicates(operation.InsertExecutionResult(result))) } - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - resultID := key.(flow.Identifier) - var result flow.ExecutionResult - return func(tx *badger.Txn) (interface{}, error) { + retrieve := func(resultID flow.Identifier) func(tx *badger.Txn) (*flow.ExecutionResult, error) { + return func(tx *badger.Txn) (*flow.ExecutionResult, error) { + var result flow.ExecutionResult err := operation.RetrieveExecutionResult(resultID, &result)(tx) return &result, err } @@ -40,8 +38,8 @@ func NewExecutionResults(collector module.CacheMetrics, db *badger.DB) *Executio res := &ExecutionResults{ db: db, - cache: newCache(collector, metrics.ResourceResult, - withLimit(flow.DefaultTransactionExpiry+100), + cache: newCache[flow.Identifier, *flow.ExecutionResult](collector, metrics.ResourceResult, + withLimit[flow.Identifier, *flow.ExecutionResult](flow.DefaultTransactionExpiry+100), withStore(store), withRetrieve(retrieve)), } @@ -59,7 +57,7 @@ func (r *ExecutionResults) byID(resultID flow.Identifier) func(*badger.Txn) (*fl if err != nil { return nil, err } - return val.(*flow.ExecutionResult), nil + return val, nil } } diff --git a/storage/badger/seals.go b/storage/badger/seals.go index aa68511ed7e..5ae5cbe71af 100644 --- a/storage/badger/seals.go +++ b/storage/badger/seals.go @@ -16,21 +16,18 @@ import ( type Seals struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.Seal] } func NewSeals(collector module.CacheMetrics, db *badger.DB) *Seals { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - sealID := key.(flow.Identifier) - seal := val.(*flow.Seal) + store := func(sealID flow.Identifier, seal *flow.Seal) func(*transaction.Tx) error { return transaction.WithTx(operation.SkipDuplicates(operation.InsertSeal(sealID, seal))) } - retrieve := func(key interface{}) func(*badger.Txn) (interface{}, error) { - sealID := key.(flow.Identifier) - var seal flow.Seal - return func(tx *badger.Txn) (interface{}, error) { + retrieve := func(sealID flow.Identifier) func(*badger.Txn) (*flow.Seal, error) { + return func(tx *badger.Txn) (*flow.Seal, error) { + var seal flow.Seal err := operation.RetrieveSeal(sealID, &seal)(tx) return &seal, err } @@ -38,8 +35,8 @@ func NewSeals(collector module.CacheMetrics, db *badger.DB) *Seals { s := &Seals{ db: db, - cache: newCache(collector, metrics.ResourceSeal, - withLimit(flow.DefaultTransactionExpiry+100), + cache: newCache[flow.Identifier, *flow.Seal](collector, metrics.ResourceSeal, + withLimit[flow.Identifier, *flow.Seal](flow.DefaultTransactionExpiry+100), withStore(store), withRetrieve(retrieve)), } @@ -57,7 +54,7 @@ func (s *Seals) retrieveTx(sealID flow.Identifier) func(*badger.Txn) (*flow.Seal if err != nil { return nil, err } - return val.(*flow.Seal), err + return val, err } } diff --git a/storage/badger/transaction_results.go b/storage/badger/transaction_results.go index 77cc103e8b5..1869d5d5aea 100644 --- a/storage/badger/transaction_results.go +++ b/storage/badger/transaction_results.go @@ -16,9 +16,9 @@ import ( type TransactionResults struct { db *badger.DB - cache *Cache - indexCache *Cache - blockCache *Cache + cache *Cache[string, flow.TransactionResult] + indexCache *Cache[string, flow.TransactionResult] + blockCache *Cache[string, []flow.TransactionResult] } func KeyFromBlockIDTransactionID(blockID flow.Identifier, txID flow.Identifier) string { @@ -83,43 +83,43 @@ func KeyToBlockID(key string) (flow.Identifier, error) { } func NewTransactionResults(collector module.CacheMetrics, db *badger.DB, transactionResultsCacheSize uint) *TransactionResults { - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { + retrieve := func(key string) func(tx *badger.Txn) (flow.TransactionResult, error) { var txResult flow.TransactionResult - return func(tx *badger.Txn) (interface{}, error) { + return func(tx *badger.Txn) (flow.TransactionResult, error) { - blockID, txID, err := KeyToBlockIDTransactionID(key.(string)) + blockID, txID, err := KeyToBlockIDTransactionID(key) if err != nil { - return nil, fmt.Errorf("could not convert key: %w", err) + return flow.TransactionResult{}, fmt.Errorf("could not convert key: %w", err) } err = operation.RetrieveTransactionResult(blockID, txID, &txResult)(tx) if err != nil { - return nil, handleError(err, flow.TransactionResult{}) + return flow.TransactionResult{}, handleError(err, flow.TransactionResult{}) } return txResult, nil } } - retrieveIndex := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { + retrieveIndex := func(key string) func(tx *badger.Txn) (flow.TransactionResult, error) { var txResult flow.TransactionResult - return func(tx *badger.Txn) (interface{}, error) { + return func(tx *badger.Txn) (flow.TransactionResult, error) { - blockID, txIndex, err := KeyToBlockIDIndex(key.(string)) + blockID, txIndex, err := KeyToBlockIDIndex(key) if err != nil { - return nil, fmt.Errorf("could not convert index key: %w", err) + return flow.TransactionResult{}, fmt.Errorf("could not convert index key: %w", err) } err = operation.RetrieveTransactionResultByIndex(blockID, txIndex, &txResult)(tx) if err != nil { - return nil, handleError(err, flow.TransactionResult{}) + return flow.TransactionResult{}, handleError(err, flow.TransactionResult{}) } return txResult, nil } } - retrieveForBlock := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { + retrieveForBlock := func(key string) func(tx *badger.Txn) ([]flow.TransactionResult, error) { var txResults []flow.TransactionResult - return func(tx *badger.Txn) (interface{}, error) { + return func(tx *badger.Txn) ([]flow.TransactionResult, error) { - blockID, err := KeyToBlockID(key.(string)) + blockID, err := KeyToBlockID(key) if err != nil { return nil, fmt.Errorf("could not convert index key: %w", err) } @@ -133,19 +133,19 @@ func NewTransactionResults(collector module.CacheMetrics, db *badger.DB, transac } return &TransactionResults{ db: db, - cache: newCache(collector, metrics.ResourceTransactionResults, - withLimit(transactionResultsCacheSize), - withStore(noopStore), + cache: newCache[string, flow.TransactionResult](collector, metrics.ResourceTransactionResults, + withLimit[string, flow.TransactionResult](transactionResultsCacheSize), + withStore(noopStore[string, flow.TransactionResult]), withRetrieve(retrieve), ), - indexCache: newCache(collector, metrics.ResourceTransactionResultIndices, - withLimit(transactionResultsCacheSize), - withStore(noopStore), + indexCache: newCache[string, flow.TransactionResult](collector, metrics.ResourceTransactionResultIndices, + withLimit[string, flow.TransactionResult](transactionResultsCacheSize), + withStore(noopStore[string, flow.TransactionResult]), withRetrieve(retrieveIndex), ), - blockCache: newCache(collector, metrics.ResourceTransactionResultIndices, - withLimit(transactionResultsCacheSize), - withStore(noopStore), + blockCache: newCache[string, []flow.TransactionResult](collector, metrics.ResourceTransactionResultIndices, + withLimit[string, []flow.TransactionResult](transactionResultsCacheSize), + withStore(noopStore[string, []flow.TransactionResult]), withRetrieve(retrieveForBlock), ), } @@ -190,14 +190,10 @@ func (tr *TransactionResults) ByBlockIDTransactionID(blockID flow.Identifier, tx tx := tr.db.NewTransaction(false) defer tx.Discard() key := KeyFromBlockIDTransactionID(blockID, txID) - val, err := tr.cache.Get(key)(tx) + transactionResult, err := tr.cache.Get(key)(tx) if err != nil { return nil, err } - transactionResult, ok := val.(flow.TransactionResult) - if !ok { - return nil, fmt.Errorf("could not convert transaction result: %w", err) - } return &transactionResult, nil } @@ -206,14 +202,10 @@ func (tr *TransactionResults) ByBlockIDTransactionIndex(blockID flow.Identifier, tx := tr.db.NewTransaction(false) defer tx.Discard() key := KeyFromBlockIDIndex(blockID, txIndex) - val, err := tr.indexCache.Get(key)(tx) + transactionResult, err := tr.indexCache.Get(key)(tx) if err != nil { return nil, err } - transactionResult, ok := val.(flow.TransactionResult) - if !ok { - return nil, fmt.Errorf("could not convert transaction result: %w", err) - } return &transactionResult, nil } @@ -222,14 +214,10 @@ func (tr *TransactionResults) ByBlockID(blockID flow.Identifier) ([]flow.Transac tx := tr.db.NewTransaction(false) defer tx.Discard() key := KeyFromBlockID(blockID) - val, err := tr.blockCache.Get(key)(tx) + transactionResults, err := tr.blockCache.Get(key)(tx) if err != nil { return nil, err } - transactionResults, ok := val.([]flow.TransactionResult) - if !ok { - return nil, fmt.Errorf("could not convert transaction result: %w", err) - } return transactionResults, nil } diff --git a/storage/badger/transactions.go b/storage/badger/transactions.go index 97cd6e98293..eeca9c9477e 100644 --- a/storage/badger/transactions.go +++ b/storage/badger/transactions.go @@ -13,21 +13,18 @@ import ( // Transactions ... type Transactions struct { db *badger.DB - cache *Cache + cache *Cache[flow.Identifier, *flow.TransactionBody] } // NewTransactions ... func NewTransactions(cacheMetrics module.CacheMetrics, db *badger.DB) *Transactions { - store := func(key interface{}, val interface{}) func(*transaction.Tx) error { - txID := key.(flow.Identifier) - flowTx := val.(*flow.TransactionBody) - return transaction.WithTx(operation.SkipDuplicates(operation.InsertTransaction(txID, flowTx))) + store := func(txID flow.Identifier, flowTX *flow.TransactionBody) func(*transaction.Tx) error { + return transaction.WithTx(operation.SkipDuplicates(operation.InsertTransaction(txID, flowTX))) } - retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) { - txID := key.(flow.Identifier) - var flowTx flow.TransactionBody - return func(tx *badger.Txn) (interface{}, error) { + retrieve := func(txID flow.Identifier) func(tx *badger.Txn) (*flow.TransactionBody, error) { + return func(tx *badger.Txn) (*flow.TransactionBody, error) { + var flowTx flow.TransactionBody err := operation.RetrieveTransaction(txID, &flowTx)(tx) return &flowTx, err } @@ -35,8 +32,8 @@ func NewTransactions(cacheMetrics module.CacheMetrics, db *badger.DB) *Transacti t := &Transactions{ db: db, - cache: newCache(cacheMetrics, metrics.ResourceTransaction, - withLimit(flow.DefaultTransactionExpiry+100), + cache: newCache[flow.Identifier, *flow.TransactionBody](cacheMetrics, metrics.ResourceTransaction, + withLimit[flow.Identifier, *flow.TransactionBody](flow.DefaultTransactionExpiry+100), withStore(store), withRetrieve(retrieve)), } @@ -66,6 +63,6 @@ func (t *Transactions) retrieveTx(txID flow.Identifier) func(*badger.Txn) (*flow if err != nil { return nil, err } - return val.(*flow.TransactionBody), err + return val, err } } diff --git a/storage/badger/version_beacon.go b/storage/badger/version_beacon.go new file mode 100644 index 00000000000..7300c2fc568 --- /dev/null +++ b/storage/badger/version_beacon.go @@ -0,0 +1,43 @@ +package badger + +import ( + "errors" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +type VersionBeacons struct { + db *badger.DB +} + +var _ storage.VersionBeacons = (*VersionBeacons)(nil) + +func NewVersionBeacons(db *badger.DB) *VersionBeacons { + res := &VersionBeacons{ + db: db, + } + + return res +} + +func (r *VersionBeacons) Highest( + belowOrEqualTo uint64, +) (*flow.SealedVersionBeacon, error) { + tx := r.db.NewTransaction(false) + defer tx.Discard() + + var beacon flow.SealedVersionBeacon + + err := operation.LookupLastVersionBeaconByHeight(belowOrEqualTo, &beacon)(tx) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, nil + } + return nil, err + } + return &beacon, nil +} diff --git a/storage/headers.go b/storage/headers.go index 0035e12f2a0..ccd58899e94 100644 --- a/storage/headers.go +++ b/storage/headers.go @@ -24,8 +24,8 @@ type Headers interface { // No errors are expected during normal operation. Exists(blockID flow.Identifier) (bool, error) - // BlockIDByHeight the block ID that is finalized at the given height. It is an optimized version - // of `ByHeight` that skips retrieving the block. Expected errors during normal operations: + // BlockIDByHeight returns the block ID that is finalized at the given height. It is an optimized + // version of `ByHeight` that skips retrieving the block. Expected errors during normal operations: // * `storage.ErrNotFound` if no finalized block is known at given height BlockIDByHeight(height uint64) (flow.Identifier, error) @@ -33,18 +33,4 @@ type Headers interface { // might be unfinalized; if there is more than one, at least one of them has to // be unfinalized. ByParentID(parentID flow.Identifier) ([]*flow.Header, error) - - // IndexByChunkID indexes block ID by chunk ID. - IndexByChunkID(headerID, chunkID flow.Identifier) error - - // BatchIndexByChunkID indexes block ID by chunk ID in a given batch. - BatchIndexByChunkID(headerID, chunkID flow.Identifier, batch BatchStorage) error - - // IDByChunkID finds the ID of the block corresponding to given chunk ID. - IDByChunkID(chunkID flow.Identifier) (flow.Identifier, error) - - // BatchRemoveChunkBlockIndexByChunkID removes block to chunk index entry keyed by a blockID in a provided batch - // No errors are expected during normal operation, even if no entries are matched. - // If Badger unexpectedly fails to process the request, the error is wrapped in a generic error and returned. - BatchRemoveChunkBlockIndexByChunkID(chunkID flow.Identifier, batch BatchStorage) error } diff --git a/storage/ledger.go b/storage/ledger.go old mode 100755 new mode 100644 diff --git a/storage/merkle/proof_test.go b/storage/merkle/proof_test.go index 44e93a90bef..826b61b6ed8 100644 --- a/storage/merkle/proof_test.go +++ b/storage/merkle/proof_test.go @@ -3,7 +3,6 @@ package merkle import ( "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -142,7 +141,7 @@ func TestValidateFormat(t *testing.T) { // when trie includes many random keys. (only a random subset of keys are checked for proofs) func TestProofsWithRandomKeys(t *testing.T) { // initialize random generator, two trees and zero hash - rand.Seed(time.Now().UnixNano()) + keyLength := 32 numberOfInsertions := 10000 numberOfProofsToVerify := 100 diff --git a/storage/merkle/tree_test.go b/storage/merkle/tree_test.go index b20ee26d7e5..aea20cca8db 100644 --- a/storage/merkle/tree_test.go +++ b/storage/merkle/tree_test.go @@ -3,11 +3,11 @@ package merkle import ( + crand "crypto/rand" "encoding/hex" "fmt" - "math/rand" + mrand "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -64,7 +64,9 @@ func TestEmptyTreeHash(t *testing.T) { // generate random key-value pair key := make([]byte, keyLength) - rand.Read(key) + _, err := crand.Read(key) + require.NoError(t, err) + val := []byte{1} // add key-value pair: hash should be non-empty @@ -239,7 +241,7 @@ func Test_KeyLengthChecked(t *testing.T) { // of a _single_ key-value pair to an otherwise empty tree. func TestTreeSingle(t *testing.T) { // initialize the random generator, tree and zero hash - rand.Seed(time.Now().UnixNano()) + keyLength := 32 tree, err := NewTree(keyLength) assert.NoError(t, err) @@ -275,7 +277,7 @@ func TestTreeSingle(t *testing.T) { // Key-value pairs are added and deleted in the same order. func TestTreeBatch(t *testing.T) { // initialize random generator, tree, zero hash - rand.Seed(time.Now().UnixNano()) + keyLength := 32 tree, err := NewTree(keyLength) assert.NoError(t, err) @@ -321,7 +323,7 @@ func TestTreeBatch(t *testing.T) { // in which the elements were added. func TestRandomOrder(t *testing.T) { // initialize random generator, two trees and zero hash - rand.Seed(time.Now().UnixNano()) + keyLength := 32 tree1, err := NewTree(keyLength) assert.NoError(t, err) @@ -346,7 +348,7 @@ func TestRandomOrder(t *testing.T) { } // shuffle the keys and insert them with random order into the second tree - rand.Shuffle(len(keys), func(i int, j int) { + mrand.Shuffle(len(keys), func(i int, j int) { keys[i], keys[j] = keys[j], keys[i] }) for _, key := range keys { @@ -382,8 +384,8 @@ func BenchmarkTree(b *testing.B) { func randomKeyValuePair(keySize, valueSize int) ([]byte, []byte) { key := make([]byte, keySize) val := make([]byte, valueSize) - _, _ = rand.Read(key) - _, _ = rand.Read(val) + _, _ = crand.Read(key) + _, _ = crand.Read(val) return key, val } diff --git a/storage/mock/headers.go b/storage/mock/headers.go index 0c21e53fe07..f130a452946 100644 --- a/storage/mock/headers.go +++ b/storage/mock/headers.go @@ -5,8 +5,6 @@ package mock import ( flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" - - storage "github.com/onflow/flow-go/storage" ) // Headers is an autogenerated mock type for the Headers type @@ -14,34 +12,6 @@ type Headers struct { mock.Mock } -// BatchIndexByChunkID provides a mock function with given fields: headerID, chunkID, batch -func (_m *Headers) BatchIndexByChunkID(headerID flow.Identifier, chunkID flow.Identifier, batch storage.BatchStorage) error { - ret := _m.Called(headerID, chunkID, batch) - - var r0 error - if rf, ok := ret.Get(0).(func(flow.Identifier, flow.Identifier, storage.BatchStorage) error); ok { - r0 = rf(headerID, chunkID, batch) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// BatchRemoveChunkBlockIndexByChunkID provides a mock function with given fields: chunkID, batch -func (_m *Headers) BatchRemoveChunkBlockIndexByChunkID(chunkID flow.Identifier, batch storage.BatchStorage) error { - ret := _m.Called(chunkID, batch) - - var r0 error - if rf, ok := ret.Get(0).(func(flow.Identifier, storage.BatchStorage) error); ok { - r0 = rf(chunkID, batch) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // BlockIDByHeight provides a mock function with given fields: height func (_m *Headers) BlockIDByHeight(height uint64) (flow.Identifier, error) { ret := _m.Called(height) @@ -170,46 +140,6 @@ func (_m *Headers) Exists(blockID flow.Identifier) (bool, error) { return r0, r1 } -// IDByChunkID provides a mock function with given fields: chunkID -func (_m *Headers) IDByChunkID(chunkID flow.Identifier) (flow.Identifier, error) { - ret := _m.Called(chunkID) - - var r0 flow.Identifier - var r1 error - if rf, ok := ret.Get(0).(func(flow.Identifier) (flow.Identifier, error)); ok { - return rf(chunkID) - } - if rf, ok := ret.Get(0).(func(flow.Identifier) flow.Identifier); ok { - r0 = rf(chunkID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(flow.Identifier) - } - } - - if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { - r1 = rf(chunkID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// IndexByChunkID provides a mock function with given fields: headerID, chunkID -func (_m *Headers) IndexByChunkID(headerID flow.Identifier, chunkID flow.Identifier) error { - ret := _m.Called(headerID, chunkID) - - var r0 error - if rf, ok := ret.Get(0).(func(flow.Identifier, flow.Identifier) error); ok { - r0 = rf(headerID, chunkID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // Store provides a mock function with given fields: header func (_m *Headers) Store(header *flow.Header) error { ret := _m.Called(header) diff --git a/storage/mock/version_beacons.go b/storage/mock/version_beacons.go new file mode 100644 index 00000000000..dd06ce17dd2 --- /dev/null +++ b/storage/mock/version_beacons.go @@ -0,0 +1,54 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// VersionBeacons is an autogenerated mock type for the VersionBeacons type +type VersionBeacons struct { + mock.Mock +} + +// Highest provides a mock function with given fields: belowOrEqualTo +func (_m *VersionBeacons) Highest(belowOrEqualTo uint64) (*flow.SealedVersionBeacon, error) { + ret := _m.Called(belowOrEqualTo) + + var r0 *flow.SealedVersionBeacon + var r1 error + if rf, ok := ret.Get(0).(func(uint64) (*flow.SealedVersionBeacon, error)); ok { + return rf(belowOrEqualTo) + } + if rf, ok := ret.Get(0).(func(uint64) *flow.SealedVersionBeacon); ok { + r0 = rf(belowOrEqualTo) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.SealedVersionBeacon) + } + } + + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(belowOrEqualTo) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewVersionBeacons interface { + mock.TestingT + Cleanup(func()) +} + +// NewVersionBeacons creates a new instance of VersionBeacons. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewVersionBeacons(t mockConstructorTestingTNewVersionBeacons) *VersionBeacons { + mock := &VersionBeacons{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/storage/mocks/storage.go b/storage/mocks/storage.go index 49fdbe48c96..e8b1281377a 100644 --- a/storage/mocks/storage.go +++ b/storage/mocks/storage.go @@ -189,34 +189,6 @@ func (m *MockHeaders) EXPECT() *MockHeadersMockRecorder { return m.recorder } -// BatchIndexByChunkID mocks base method. -func (m *MockHeaders) BatchIndexByChunkID(arg0, arg1 flow.Identifier, arg2 storage.BatchStorage) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "BatchIndexByChunkID", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// BatchIndexByChunkID indicates an expected call of BatchIndexByChunkID. -func (mr *MockHeadersMockRecorder) BatchIndexByChunkID(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchIndexByChunkID", reflect.TypeOf((*MockHeaders)(nil).BatchIndexByChunkID), arg0, arg1, arg2) -} - -// BatchRemoveChunkBlockIndexByChunkID mocks base method. -func (m *MockHeaders) BatchRemoveChunkBlockIndexByChunkID(arg0 flow.Identifier, arg1 storage.BatchStorage) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "BatchRemoveChunkBlockIndexByChunkID", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// BatchRemoveChunkBlockIndexByChunkID indicates an expected call of BatchRemoveChunkBlockIndexByChunkID. -func (mr *MockHeadersMockRecorder) BatchRemoveChunkBlockIndexByChunkID(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchRemoveChunkBlockIndexByChunkID", reflect.TypeOf((*MockHeaders)(nil).BatchRemoveChunkBlockIndexByChunkID), arg0, arg1) -} - // BlockIDByHeight mocks base method. func (m *MockHeaders) BlockIDByHeight(arg0 uint64) (flow.Identifier, error) { m.ctrl.T.Helper() @@ -292,35 +264,6 @@ func (mr *MockHeadersMockRecorder) Exists(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockHeaders)(nil).Exists), arg0) } -// IDByChunkID mocks base method. -func (m *MockHeaders) IDByChunkID(arg0 flow.Identifier) (flow.Identifier, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IDByChunkID", arg0) - ret0, _ := ret[0].(flow.Identifier) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IDByChunkID indicates an expected call of IDByChunkID. -func (mr *MockHeadersMockRecorder) IDByChunkID(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IDByChunkID", reflect.TypeOf((*MockHeaders)(nil).IDByChunkID), arg0) -} - -// IndexByChunkID mocks base method. -func (m *MockHeaders) IndexByChunkID(arg0, arg1 flow.Identifier) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IndexByChunkID", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// IndexByChunkID indicates an expected call of IndexByChunkID. -func (mr *MockHeadersMockRecorder) IndexByChunkID(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IndexByChunkID", reflect.TypeOf((*MockHeaders)(nil).IndexByChunkID), arg0, arg1) -} - // Store mocks base method. func (m *MockHeaders) Store(arg0 *flow.Header) error { m.ctrl.T.Helper() diff --git a/storage/version_beacon.go b/storage/version_beacon.go new file mode 100644 index 00000000000..2a57c944aa4 --- /dev/null +++ b/storage/version_beacon.go @@ -0,0 +1,13 @@ +package storage + +import "github.com/onflow/flow-go/model/flow" + +// VersionBeacons represents persistent storage for Version Beacons. +type VersionBeacons interface { + + // Highest finds the highest flow.SealedVersionBeacon but no higher than + // belowOrEqualTo + // Returns nil if no version beacon has been sealed below or equal to the + // input height. + Highest(belowOrEqualTo uint64) (*flow.SealedVersionBeacon, error) +} diff --git a/utils/logging/consts.go b/utils/logging/consts.go index 31cfef3078a..46f48a3c937 100644 --- a/utils/logging/consts.go +++ b/utils/logging/consts.go @@ -1,5 +1,11 @@ package logging -// KeySuspicious is a logging label that is used to flag the log event as suspicious behavior -// This is used to add an easily searchable label to the log event -const KeySuspicious = "suspicious" +const ( + // KeySuspicious is a logging label that is used to flag the log event as suspicious behavior + // This is used to add an easily searchable label to the log event + KeySuspicious = "suspicious" + + // KeyNetworkingSecurity is a logging label that is used to flag the log event as a networking security issue. + // This is used to add an easily searchable label to the log events. + KeyNetworkingSecurity = "networking-security" +) diff --git a/utils/rand/rand.go b/utils/rand/rand.go index c589ae67868..9a577f7afec 100644 --- a/utils/rand/rand.go +++ b/utils/rand/rand.go @@ -1,3 +1,14 @@ +// Package rand is a wrapper around `crypto/rand` that uses the system RNG underneath +// to extract secure entropy. +// +// It implements useful tools that are not exported by the `crypto/rand` package. +// This package should be used instead of `math/rand` for any use-case requiring +// a secure randomness. It provides similar APIs to the ones provided by `math/rand`. +// This package does not implement any determinstic RNG (Pseudo-RNG) based on +// user input seeds. For the deterministic use-cases please use `flow-go/crypto/random`. +// +// Functions in this package may return an error if the underlying system implementation fails +// to read new randoms. When that happens, this package considers it an irrecoverable exception. package rand import ( @@ -6,13 +17,11 @@ import ( "fmt" ) -// This package is a wrppaer around true RNG crypto/rand. -// It implements useful tools using the true RNG and that -// are not exported by the crypto/rand package. -// This package does not implement any determinstic RNG (Pseudo RNG) -// unlike the package flow-go/crypto/random. - -// returns a random uint64 +// Uint64 returns a random uint64. +// +// It returns: +// - (0, exception) if crypto/rand fails to provide entropy which is likely a result of a system error. +// - (random, nil) otherwise func Uint64() (uint64, error) { // allocate a new memory at each call. Another possibility // is to use a global variable but that would make the package non thread safe @@ -24,8 +33,13 @@ func Uint64() (uint64, error) { return r, nil } -// returns a random uint64 strictly less than n -// errors if n==0 +// Uint64n returns a random uint64 strictly less than `n`. +// `n` has to be a strictly positive integer. +// +// It returns: +// - (0, exception) if `n==0` +// - (0, exception) if crypto/rand fails to provide entropy which is likely a result of a system error. +// - (random, nil) otherwise func Uint64n(n uint64) (uint64, error) { if n == 0 { return 0, fmt.Errorf("n should be strictly positive, got %d", n) @@ -66,7 +80,11 @@ func Uint64n(n uint64) (uint64, error) { return random, nil } -// returns a random uint32 +// Uint32 returns a random uint32. +// +// It returns: +// - (0, exception) if crypto/rand fails to provide entropy which is likely a result of a system error. +// - (random, nil) otherwise func Uint32() (uint32, error) { // for 64-bits machines, doing 64 bits operations and then casting // should be faster than dealing with 32 bits operations @@ -74,21 +92,35 @@ func Uint32() (uint32, error) { return uint32(r), err } -// returns a random uint32 strictly less than n -// errors if n==0 +// Uint32n returns a random uint32 strictly less than `n`. +// `n` has to be a strictly positive integer. +// +// It returns an error: +// - (0, exception) if `n==0` +// - (0, exception) if crypto/rand fails to provide entropy which is likely a result of a system error. +// - (random, nil) otherwise func Uint32n(n uint32) (uint32, error) { r, err := Uint64n(uint64(n)) return uint32(r), err } -// returns a random uint +// Uint returns a random uint. +// +// It returns: +// - (0, exception) if crypto/rand fails to provide entropy which is likely a result of a system error. +// - (random, nil) otherwise func Uint() (uint, error) { r, err := Uint64() return uint(r), err } -// returns a random uint strictly less than n -// errors if n==0 +// returns a random uint strictly less than `n`. +// `n` has to be a strictly positive integer. +// +// It returns an error: +// - (0, exception) if `n==0` +// - (0, exception) if crypto/rand fails to provide entropy which is likely a result of a system error. +// - (random, nil) otherwise func Uintn(n uint) (uint, error) { r, err := Uint64n(uint64(n)) return uint(r), err @@ -99,22 +131,29 @@ func Uintn(n uint) (uint, error) { // It is not deterministic. // // It implements Fisher-Yates Shuffle using crypto/rand as a source of randoms. +// It uses O(1) space and O(n) time // -// O(1) space and O(n) time +// It returns: +// - (exception) if crypto/rand fails to provide entropy which is likely a result of a system error. +// - (nil) otherwise func Shuffle(n uint, swap func(i, j uint)) error { return Samples(n, n, swap) } -// Samples picks randomly m elements out of n elemnts in a data structure +// Samples picks randomly `m` elements out of `n` elements in a data structure // and places them in random order at indices [0,m-1], // the swapping being implemented in place. The data structure is defined -// by the `swap` function. -// Sampling is not deterministic. +// by the `swap` function itself. +// Sampling is not deterministic like the other functions of the package. // -// It implements the first (m) elements of Fisher-Yates Shuffle using -// crypto/rand as a source of randoms. +// It implements the first `m` elements of Fisher-Yates Shuffle using +// crypto/rand as a source of randoms. `m` has to be less or equal to `n`. +// It uses O(1) space and O(m) time // -// O(1) space and O(m) time +// It returns: +// - (exception) if `n < m` +// - (exception) if crypto/rand fails to provide entropy which is likely a result of a system error. +// - (nil) otherwise func Samples(n uint, m uint, swap func(i, j uint)) error { if n < m { return fmt.Errorf("sample size (%d) cannot be larger than entire population (%d)", m, n) diff --git a/utils/rand/rand_test.go b/utils/rand/rand_test.go index 14f00559d62..73d7ca539ca 100644 --- a/utils/rand/rand_test.go +++ b/utils/rand/rand_test.go @@ -1,49 +1,16 @@ package rand import ( - "fmt" "math" mrand "math/rand" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gonum.org/v1/gonum/stat" - _ "github.com/onflow/flow-go/crypto/random" + "github.com/onflow/flow-go/crypto/random" ) -// TODO: these functions are copied from flow-go/crypto/rand -// Once the new flow-go/crypto/ module version is tagged, flow-go would upgrade -// to the new version and import these functions -func BasicDistributionTest(t *testing.T, n uint64, classWidth uint64, randf func() (uint64, error)) { - // sample size should ideally be a high number multiple of `n` - // but if `n` is too small, we could use a small sample size so that the test - // isn't too slow - sampleSize := 1000 * n - if n < 100 { - sampleSize = (80000 / n) * n // highest multiple of n less than 80000 - } - distribution := make([]float64, n) - // populate the distribution - for i := uint64(0); i < sampleSize; i++ { - r, err := randf() - require.NoError(t, err) - if n*classWidth != 0 { - require.Less(t, r, n*classWidth) - } - distribution[r/classWidth] += 1.0 - } - EvaluateDistributionUniformity(t, distribution) -} - -func EvaluateDistributionUniformity(t *testing.T, distribution []float64) { - tolerance := 0.05 - stdev := stat.StdDev(distribution, nil) - mean := stat.Mean(distribution, nil) - assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed: n: %d, stdev: %v, mean: %v", len(distribution), stdev, mean)) -} - func TestRandomIntegers(t *testing.T) { t.Run("basic uniformity", func(t *testing.T) { @@ -56,7 +23,7 @@ func TestRandomIntegers(t *testing.T) { r, err := Uint() return uint64(r), err } - BasicDistributionTest(t, uint64(n), uint64(classWidth), uintf) + random.BasicDistributionTest(t, uint64(n), uint64(classWidth), uintf) }) t.Run("Uint64", func(t *testing.T) { @@ -64,7 +31,7 @@ func TestRandomIntegers(t *testing.T) { // n is a random power of 2 (from 2 to 2^10) n := 1 << (1 + mrand.Intn(10)) classWidth := (math.MaxUint64 / uint64(n)) + 1 - BasicDistributionTest(t, uint64(n), uint64(classWidth), Uint64) + random.BasicDistributionTest(t, uint64(n), uint64(classWidth), Uint64) }) t.Run("Uint32", func(t *testing.T) { @@ -76,7 +43,7 @@ func TestRandomIntegers(t *testing.T) { r, err := Uint32() return uint64(r), err } - BasicDistributionTest(t, uint64(n), uint64(classWidth), uintf) + random.BasicDistributionTest(t, uint64(n), uint64(classWidth), uintf) }) t.Run("Uintn", func(t *testing.T) { @@ -86,7 +53,7 @@ func TestRandomIntegers(t *testing.T) { return uint64(r), err } // classWidth is 1 since `n` is small - BasicDistributionTest(t, uint64(n), uint64(1), uintf) + random.BasicDistributionTest(t, uint64(n), uint64(1), uintf) }) t.Run("Uint64n", func(t *testing.T) { @@ -95,7 +62,7 @@ func TestRandomIntegers(t *testing.T) { return Uint64n(uint64(n)) } // classWidth is 1 since `n` is small - BasicDistributionTest(t, uint64(n), uint64(1), uintf) + random.BasicDistributionTest(t, uint64(n), uint64(1), uintf) }) t.Run("Uint32n", func(t *testing.T) { @@ -105,7 +72,7 @@ func TestRandomIntegers(t *testing.T) { return uint64(r), err } // classWidth is 1 since `n` is small - BasicDistributionTest(t, uint64(n), uint64(1), uintf) + random.BasicDistributionTest(t, uint64(n), uint64(1), uintf) }) }) @@ -169,7 +136,7 @@ func TestShuffle(t *testing.T) { } // if the shuffle is uniform, the test element // should end up uniformly in all positions of the slice - EvaluateDistributionUniformity(t, distribution) + random.EvaluateDistributionUniformity(t, distribution) }) t.Run("shuffle a same permutation", func(t *testing.T) { @@ -182,7 +149,7 @@ func TestShuffle(t *testing.T) { } // if the shuffle is uniform, the test element // should end up uniformly in all positions of the slice - EvaluateDistributionUniformity(t, distribution) + random.EvaluateDistributionUniformity(t, distribution) }) }) @@ -232,10 +199,10 @@ func TestSamples(t *testing.T) { } // if the sampling is uniform, all elements // should end up being sampled an equivalent number of times - EvaluateDistributionUniformity(t, samplingDistribution) + random.EvaluateDistributionUniformity(t, samplingDistribution) // if the sampling is uniform, the test element // should end up uniformly in all positions of the sample slice - EvaluateDistributionUniformity(t, orderingDistribution) + random.EvaluateDistributionUniformity(t, orderingDistribution) }) t.Run("zero edge cases", func(t *testing.T) { diff --git a/utils/unittest/bytes.go b/utils/unittest/bytes.go new file mode 100644 index 00000000000..96238cc7ae0 --- /dev/null +++ b/utils/unittest/bytes.go @@ -0,0 +1,20 @@ +package unittest + +import ( + "crypto/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +// RandomByteSlice is a test helper that generates a cryptographically secure random byte slice of size n. +func RandomByteSlice(t *testing.T, n int) []byte { + require.Greater(t, n, 0, "size should be positive") + + byteSlice := make([]byte, n) + n, err := rand.Read(byteSlice) + require.NoErrorf(t, err, "failed to generate random byte slice of size %d", n) + require.Equalf(t, n, len(byteSlice), "failed to generate random byte slice of size %d", n) + + return byteSlice +} diff --git a/utils/unittest/chain_suite.go b/utils/unittest/chain_suite.go index bd7b97fe52b..a2ebc59f8d0 100644 --- a/utils/unittest/chain_suite.go +++ b/utils/unittest/chain_suite.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/model/chunks" @@ -504,7 +505,8 @@ func (bc *BaseChainSuite) ValidSubgraphFixture() subgraphFixture { assignedVerifiersPerChunk := uint(len(bc.Approvers) / 2) approvals := make(map[uint64]map[flow.Identifier]*flow.ResultApproval) for _, chunk := range incorporatedResult.Result.Chunks { - assignedVerifiers := bc.Approvers.Sample(assignedVerifiersPerChunk) + assignedVerifiers, err := bc.Approvers.Sample(assignedVerifiersPerChunk) + require.NoError(bc.T(), err) assignment.Add(chunk, assignedVerifiers.NodeIDs()) // generate approvals @@ -543,7 +545,8 @@ func (bc *BaseChainSuite) Extend(block *flow.Block) { assignedVerifiersPerChunk := uint(len(bc.Approvers) / 2) approvals := make(map[uint64]map[flow.Identifier]*flow.ResultApproval) for _, chunk := range incorporatedResult.Result.Chunks { - assignedVerifiers := bc.Approvers.Sample(assignedVerifiersPerChunk) + assignedVerifiers, err := bc.Approvers.Sample(assignedVerifiersPerChunk) + require.NoError(bc.T(), err) assignment.Add(chunk, assignedVerifiers.NodeIDs()) // generate approvals diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 36030632ffa..9e843576195 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -24,7 +24,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "25efe0670b8832f97147c1e6c7d5c8f3314c4f67e073c02364ff861c5fd22246" +const GenesisStateCommitmentHex = "517138d362602fb11b17524a654b00d8eecdfbf56406b1636a2c58dad7c5d144" var GenesisStateCommitment flow.StateCommitment @@ -68,3 +68,30 @@ func init() { // fvm.AccountKeyWeightThreshold here ServiceAccountPublicKey = ServiceAccountPrivateKey.PublicKey(1000) } + +// this is done by printing the state commitment in TestBootstrapLedger test with different chain ID +func GenesisStateCommitmentByChainID(chainID flow.ChainID) flow.StateCommitment { + commitString := genesisCommitHexByChainID(chainID) + bytes, err := hex.DecodeString(commitString) + if err != nil { + panic("error while hex decoding hardcoded state commitment") + } + commit, err := flow.ToStateCommitment(bytes) + if err != nil { + panic("genesis state commitment size is invalid") + } + return commit +} + +func genesisCommitHexByChainID(chainID flow.ChainID) string { + if chainID == flow.Mainnet { + return GenesisStateCommitmentHex + } + if chainID == flow.Testnet { + return "dd8c079b196fced93e4c541a8f6c49a0ee5fda01b2653c5a03cc165ab1015423" + } + if chainID == flow.Sandboxnet { + return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" + } + return "c6e7f204c774f4208e67451acfdf9783932df06e5ab29b7afc56d548a1573769" +} diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index b7517add2c3..999f090232b 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -1,6 +1,7 @@ package unittest import ( + "bytes" crand "crypto/rand" "fmt" "math/rand" @@ -8,8 +9,6 @@ import ( "testing" "time" - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" @@ -17,13 +16,15 @@ import ( sdk "github.com/onflow/flow-go-sdk" + hotstuff "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/crypto/hash" - - hotstuff "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/engine/access/rest/util" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/bitutils" + "github.com/onflow/flow-go/ledger/common/testutils" "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/chainsync" "github.com/onflow/flow-go/model/chunks" @@ -35,6 +36,7 @@ import ( "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/model/verification" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/module/mempool/entity" "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/module/updatable_configs" @@ -261,7 +263,10 @@ func WithAllTheFixins(payload *flow.Payload) { payload.Seals = Seal.Fixtures(3) payload.Guarantees = CollectionGuaranteesFixture(4) for i := 0; i < 10; i++ { - receipt := ExecutionReceiptFixture() + receipt := ExecutionReceiptFixture( + WithResult(ExecutionResultFixture(WithServiceEvents(3))), + WithSpocks(SignaturesFixture(3)), + ) payload.Receipts = flow.ExecutionReceiptMetaList{receipt.Meta()} payload.Results = flow.ExecutionResultList{&receipt.ExecutionResult} } @@ -331,11 +336,15 @@ func WithoutGuarantee(payload *flow.Payload) { payload.Guarantees = nil } -func StateInteractionsFixture() *state.ExecutionSnapshot { - return &state.ExecutionSnapshot{} +func StateInteractionsFixture() *snapshot.ExecutionSnapshot { + return &snapshot.ExecutionSnapshot{} } -func BlockWithParentAndProposerFixture(t *testing.T, parent *flow.Header, proposer flow.Identifier) flow.Block { +func BlockWithParentAndProposerFixture( + t *testing.T, + parent *flow.Header, + proposer flow.Identifier, +) flow.Block { block := BlockWithParentFixture(parent) indices, err := signature.EncodeSignersToIndices( @@ -405,13 +414,10 @@ func BlockHeaderFixture(opts ...func(header *flow.Header)) *flow.Header { return header } -func CidFixture() cid.Cid { - data := make([]byte, 1024) - _, _ = rand.Read(data) - return blocks.NewBlock(data).Cid() -} - -func BlockHeaderFixtureOnChain(chainID flow.ChainID, opts ...func(header *flow.Header)) *flow.Header { +func BlockHeaderFixtureOnChain( + chainID flow.ChainID, + opts ...func(header *flow.Header), +) *flow.Header { height := 1 + uint64(rand.Uint32()) // avoiding edge case of height = 0 (genesis block) view := height + uint64(rand.Intn(1000)) header := BlockHeaderWithParentFixture(&flow.Header{ @@ -460,6 +466,38 @@ func BlockHeaderWithParentFixture(parent *flow.Header) *flow.Header { } } +func BlockHeaderWithParentWithSoRFixture(parent *flow.Header, source []byte) *flow.Header { + height := parent.Height + 1 + view := parent.View + 1 + uint64(rand.Intn(10)) // Intn returns [0, n) + var lastViewTC *flow.TimeoutCertificate + if view != parent.View+1 { + newestQC := QuorumCertificateFixture(func(qc *flow.QuorumCertificate) { + qc.View = parent.View + }) + lastViewTC = &flow.TimeoutCertificate{ + View: view - 1, + NewestQCViews: []uint64{newestQC.View}, + NewestQC: newestQC, + SignerIndices: SignerIndicesFixture(4), + SigData: SignatureFixture(), + } + } + return &flow.Header{ + ChainID: parent.ChainID, + ParentID: parent.ID(), + Height: height, + PayloadHash: IdentifierFixture(), + Timestamp: time.Now().UTC(), + View: view, + ParentView: parent.View, + ParentVoterIndices: SignerIndicesFixture(4), + ParentVoterSigData: QCSigDataWithSoRFixture(source), + ProposerID: IdentifierFixture(), + ProposerSigData: SignatureFixture(), + LastViewTC: lastViewTC, + } +} + func ClusterPayloadFixture(n int) *cluster.Payload { transactions := make([]*flow.TransactionBody, n) for i := 0; i < n; i++ { @@ -482,6 +520,20 @@ func ClusterBlockFixture() cluster.Block { } } +func ClusterBlockChainFixture(n int) []cluster.Block { + clusterBlocks := make([]cluster.Block, 0, n) + + parent := ClusterBlockFixture() + + for i := 0; i < n; i++ { + block := ClusterBlockWithParent(&parent) + clusterBlocks = append(clusterBlocks, block) + parent = block + } + + return clusterBlocks +} + // ClusterBlockWithParent creates a new cluster consensus block that is valid // with respect to the given parent block. func ClusterBlockWithParent(parent *cluster.Block) cluster.Block { @@ -538,7 +590,10 @@ func CollectionGuaranteesWithCollectionIDFixture(collections []*flow.Collection) return guarantees } -func CollectionGuaranteesFixture(n int, options ...func(*flow.CollectionGuarantee)) []*flow.CollectionGuarantee { +func CollectionGuaranteesFixture( + n int, + options ...func(*flow.CollectionGuarantee), +) []*flow.CollectionGuarantee { guarantees := make([]*flow.CollectionGuarantee, 0, n) for i := 1; i <= n; i++ { guarantee := CollectionGuaranteeFixture(options...) @@ -612,13 +667,20 @@ func CompleteCollectionFromTransactions(txs []*flow.TransactionBody) *entity.Com } } -func ExecutableBlockFixture(collectionsSignerIDs [][]flow.Identifier) *entity.ExecutableBlock { +func ExecutableBlockFixture( + collectionsSignerIDs [][]flow.Identifier, + startState *flow.StateCommitment, +) *entity.ExecutableBlock { header := BlockHeaderFixture() - return ExecutableBlockFixtureWithParent(collectionsSignerIDs, header) + return ExecutableBlockFixtureWithParent(collectionsSignerIDs, header, startState) } -func ExecutableBlockFixtureWithParent(collectionsSignerIDs [][]flow.Identifier, parent *flow.Header) *entity.ExecutableBlock { +func ExecutableBlockFixtureWithParent( + collectionsSignerIDs [][]flow.Identifier, + parent *flow.Header, + startState *flow.StateCommitment, +) *entity.ExecutableBlock { completeCollections := make(map[flow.Identifier]*entity.CompleteCollection, len(collectionsSignerIDs)) block := BlockWithParentFixture(parent) @@ -635,11 +697,15 @@ func ExecutableBlockFixtureWithParent(collectionsSignerIDs [][]flow.Identifier, executableBlock := &entity.ExecutableBlock{ Block: block, CompleteCollections: completeCollections, + StartState: startState, } return executableBlock } -func ExecutableBlockFromTransactions(chain flow.ChainID, txss [][]*flow.TransactionBody) *entity.ExecutableBlock { +func ExecutableBlockFromTransactions( + chain flow.ChainID, + txss [][]*flow.TransactionBody, +) *entity.ExecutableBlock { completeCollections := make(map[flow.Identifier]*entity.CompleteCollection, len(txss)) blockHeader := BlockHeaderFixtureOnChain(chain) @@ -675,6 +741,12 @@ func WithResult(result *flow.ExecutionResult) func(*flow.ExecutionReceipt) { } } +func WithSpocks(spocks []crypto.Signature) func(*flow.ExecutionReceipt) { + return func(receipt *flow.ExecutionReceipt) { + receipt.Spocks = spocks + } +} + func ExecutionReceiptFixture(opts ...func(*flow.ExecutionReceipt)) *flow.ExecutionReceipt { receipt := &flow.ExecutionReceipt{ ExecutorID: IdentifierFixture(), @@ -694,13 +766,19 @@ func ReceiptForBlockFixture(block *flow.Block) *flow.ExecutionReceipt { return ReceiptForBlockExecutorFixture(block, IdentifierFixture()) } -func ReceiptForBlockExecutorFixture(block *flow.Block, executor flow.Identifier) *flow.ExecutionReceipt { +func ReceiptForBlockExecutorFixture( + block *flow.Block, + executor flow.Identifier, +) *flow.ExecutionReceipt { result := ExecutionResultFixture(WithBlock(block)) receipt := ExecutionReceiptFixture(WithResult(result), WithExecutorID(executor)) return receipt } -func ReceiptsForBlockFixture(block *flow.Block, ids []flow.Identifier) []*flow.ExecutionReceipt { +func ReceiptsForBlockFixture( + block *flow.Block, + ids []flow.Identifier, +) []*flow.ExecutionReceipt { result := ExecutionResultFixture(WithBlock(block)) var ers []*flow.ExecutionReceipt for _, id := range ids { @@ -743,7 +821,10 @@ func WithChunks(n uint) func(*flow.ExecutionResult) { } } -func ExecutionResultListFixture(n int, opts ...func(*flow.ExecutionResult)) []*flow.ExecutionResult { +func ExecutionResultListFixture( + n int, + opts ...func(*flow.ExecutionResult), +) []*flow.ExecutionResult { results := make([]*flow.ExecutionResult, 0, n) for i := 0; i < n; i++ { results = append(results, ExecutionResultFixture(opts...)) @@ -761,6 +842,12 @@ func WithExecutionResultBlockID(blockID flow.Identifier) func(*flow.ExecutionRes } } +func WithFinalState(commit flow.StateCommitment) func(*flow.ExecutionResult) { + return func(result *flow.ExecutionResult) { + result.Chunks[len(result.Chunks)-1].EndState = commit + } +} + func WithServiceEvents(n int) func(result *flow.ExecutionResult) { return func(result *flow.ExecutionResult) { result.ServiceEvents = ServiceEventsFixture(n) @@ -776,12 +863,14 @@ func WithExecutionDataID(id flow.Identifier) func(result *flow.ExecutionResult) func ServiceEventsFixture(n int) flow.ServiceEventList { sel := make(flow.ServiceEventList, n) - for ; n > 0; n-- { - switch rand.Intn(2) { + for i := 0; i < n; i++ { + switch i % 3 { case 0: - sel[n-1] = EpochCommitFixture().ServiceEvent() + sel[i] = EpochCommitFixture().ServiceEvent() case 1: - sel[n-1] = EpochSetupFixture().ServiceEvent() + sel[i] = EpochSetupFixture().ServiceEvent() + case 2: + sel[i] = VersionBeaconFixture().ServiceEvent() } } @@ -1013,7 +1102,10 @@ func IdentityFixture(opts ...func(*flow.Identity)) *flow.Identity { } // IdentityWithNetworkingKeyFixture returns a node identity and networking private key -func IdentityWithNetworkingKeyFixture(opts ...func(*flow.Identity)) (*flow.Identity, crypto.PrivateKey) { +func IdentityWithNetworkingKeyFixture(opts ...func(*flow.Identity)) ( + *flow.Identity, + crypto.PrivateKey, +) { networkKey := NetworkingPrivKeyFixture() opts = append(opts, WithNetworkingKey(networkKey.PublicKey())) id := IdentityFixture(opts...) @@ -1119,7 +1211,11 @@ func WithChunkStartState(startState flow.StateCommitment) func(chunk *flow.Chunk } } -func ChunkFixture(blockID flow.Identifier, collectionIndex uint, opts ...func(*flow.Chunk)) *flow.Chunk { +func ChunkFixture( + blockID flow.Identifier, + collectionIndex uint, + opts ...func(*flow.Chunk), +) *flow.Chunk { chunk := &flow.Chunk{ ChunkBody: flow.ChunkBody{ CollectionIndex: collectionIndex, @@ -1181,7 +1277,12 @@ func ChunkStatusListToChunkLocatorFixture(statuses []*verification.ChunkStatus) // ChunkStatusListFixture receives an execution result, samples `n` chunks out of it and // creates a chunk status for them. // It returns the list of sampled chunk statuses for the result. -func ChunkStatusListFixture(t *testing.T, blockHeight uint64, result *flow.ExecutionResult, n int) verification.ChunkStatusList { +func ChunkStatusListFixture( + t *testing.T, + blockHeight uint64, + result *flow.ExecutionResult, + n int, +) verification.ChunkStatusList { statuses := verification.ChunkStatusList{} // result should have enough chunk to sample @@ -1203,8 +1304,7 @@ func ChunkStatusListFixture(t *testing.T, blockHeight uint64, result *flow.Execu return statuses } -func QCSigDataFixture() []byte { - packer := hotstuff.SigDataPacker{} +func qcSignatureDataFixture() hotstuff.SignatureData { sigType := RandomBytes(5) for i := range sigType { sigType[i] = sigType[i] % 2 @@ -1215,6 +1315,20 @@ func QCSigDataFixture() []byte { AggregatedRandomBeaconSig: SignatureFixture(), ReconstructedRandomBeaconSig: SignatureFixture(), } + return sigData +} + +func QCSigDataFixture() []byte { + packer := hotstuff.SigDataPacker{} + sigData := qcSignatureDataFixture() + encoded, _ := packer.Encode(&sigData) + return encoded +} + +func QCSigDataWithSoRFixture(sor []byte) []byte { + packer := hotstuff.SigDataPacker{} + sigData := qcSignatureDataFixture() + sigData.ReconstructedRandomBeaconSig = sor encoded, _ := packer.Encode(&sigData) return encoded } @@ -1233,6 +1347,14 @@ func SignaturesFixture(n int) []crypto.Signature { return sigs } +func RandomSourcesFixture(n int) [][]byte { + var sigs [][]byte + for i := 0; i < n; i++ { + sigs = append(sigs, SignatureFixture()) + } + return sigs +} + func TransactionFixture(n ...func(t *flow.Transaction)) flow.Transaction { tx := flow.Transaction{TransactionBody: TransactionBodyFixture()} if len(n) > 0 { @@ -1360,7 +1482,10 @@ func VerifiableChunkDataFixture(chunkIndex uint64) *verification.VerifiableChunk // ChunkDataResponseMsgFixture creates a chunk data response message with a single-transaction collection, and random chunk ID. // Use options to customize the response. -func ChunkDataResponseMsgFixture(chunkID flow.Identifier, opts ...func(*messages.ChunkDataResponse)) *messages.ChunkDataResponse { +func ChunkDataResponseMsgFixture( + chunkID flow.Identifier, + opts ...func(*messages.ChunkDataResponse), +) *messages.ChunkDataResponse { cdp := &messages.ChunkDataResponse{ ChunkDataPack: *ChunkDataPackFixture(chunkID), Nonce: rand.Uint64(), @@ -1394,7 +1519,10 @@ func ChunkDataResponseMessageListFixture(chunkIDs flow.IdentifierList) []*messag } // ChunkDataPackRequestListFixture creates and returns a list of chunk data pack requests fixtures. -func ChunkDataPackRequestListFixture(n int, opts ...func(*verification.ChunkDataPackRequest)) verification.ChunkDataPackRequestList { +func ChunkDataPackRequestListFixture( + n int, + opts ...func(*verification.ChunkDataPackRequest), +) verification.ChunkDataPackRequestList { lst := make([]*verification.ChunkDataPackRequest, 0, n) for i := 0; i < n; i++ { lst = append(lst, ChunkDataPackRequestFixture(opts...)) @@ -1482,7 +1610,10 @@ func WithStartState(startState flow.StateCommitment) func(*flow.ChunkDataPack) { } } -func ChunkDataPackFixture(chunkID flow.Identifier, opts ...func(*flow.ChunkDataPack)) *flow.ChunkDataPack { +func ChunkDataPackFixture( + chunkID flow.Identifier, + opts ...func(*flow.ChunkDataPack), +) *flow.ChunkDataPack { coll := CollectionFixture(1) cdp := &flow.ChunkDataPack{ ChunkID: chunkID, @@ -1498,7 +1629,10 @@ func ChunkDataPackFixture(chunkID flow.Identifier, opts ...func(*flow.ChunkDataP return cdp } -func ChunkDataPacksFixture(count int, opts ...func(*flow.ChunkDataPack)) []*flow.ChunkDataPack { +func ChunkDataPacksFixture( + count int, + opts ...func(*flow.ChunkDataPack), +) []*flow.ChunkDataPack { chunkDataPacks := make([]*flow.ChunkDataPack, count) for i := 0; i < count; i++ { chunkDataPacks[i] = ChunkDataPackFixture(IdentifierFixture()) @@ -1524,7 +1658,23 @@ func SeedFixtures(m int, n int) [][]byte { } // BlockEventsFixture returns a block events model populated with random events of length n. -func BlockEventsFixture(header *flow.Header, n int, types ...flow.EventType) flow.BlockEvents { +func BlockEventsFixture( + header *flow.Header, + n int, + types ...flow.EventType, +) flow.BlockEvents { + return flow.BlockEvents{ + BlockID: header.ID(), + BlockHeight: header.Height, + BlockTimestamp: header.Timestamp, + Events: EventsFixture(n, types...), + } +} + +func EventsFixture( + n int, + types ...flow.EventType, +) []flow.Event { if len(types) == 0 { types = []flow.EventType{"A.0x1.Foo.Bar", "A.0x2.Zoo.Moo", "A.0x3.Goo.Hoo"} } @@ -1534,16 +1684,17 @@ func BlockEventsFixture(header *flow.Header, n int, types ...flow.EventType) flo events[i] = EventFixture(types[i%len(types)], 0, uint32(i), IdentifierFixture(), 0) } - return flow.BlockEvents{ - BlockID: header.ID(), - BlockHeight: header.Height, - BlockTimestamp: header.Timestamp, - Events: events, - } + return events } // EventFixture returns an event -func EventFixture(eType flow.EventType, transactionIndex uint32, eventIndex uint32, txID flow.Identifier, _ int) flow.Event { +func EventFixture( + eType flow.EventType, + transactionIndex uint32, + eventIndex uint32, + txID flow.Identifier, + _ int, +) flow.Event { return flow.Event{ Type: eType, TransactionIndex: transactionIndex, @@ -1608,7 +1759,10 @@ func BatchListFixture(n int) []chainsync.Batch { return batches } -func BootstrapExecutionResultFixture(block *flow.Block, commit flow.StateCommitment) *flow.ExecutionResult { +func BootstrapExecutionResultFixture( + block *flow.Block, + commit flow.StateCommitment, +) *flow.ExecutionResult { result := &flow.ExecutionResult{ BlockID: block.ID(), PreviousResultID: flow.ZeroID, @@ -1655,7 +1809,10 @@ func QuorumCertificateWithSignerIDsFixture(opts ...func(*flow.QuorumCertificateW return &qc } -func QuorumCertificatesWithSignerIDsFixtures(n uint, opts ...func(*flow.QuorumCertificateWithSignerIDs)) []*flow.QuorumCertificateWithSignerIDs { +func QuorumCertificatesWithSignerIDsFixtures( + n uint, + opts ...func(*flow.QuorumCertificateWithSignerIDs), +) []*flow.QuorumCertificateWithSignerIDs { qcs := make([]*flow.QuorumCertificateWithSignerIDs, 0, n) for i := 0; i < int(n); i++ { qcs = append(qcs, QuorumCertificateWithSignerIDsFixture(opts...)) @@ -1695,7 +1852,10 @@ func CertifyBlock(header *flow.Header) *flow.QuorumCertificate { return qc } -func QuorumCertificatesFixtures(n uint, opts ...func(*flow.QuorumCertificate)) []*flow.QuorumCertificate { +func QuorumCertificatesFixtures( + n uint, + opts ...func(*flow.QuorumCertificate), +) []*flow.QuorumCertificate { qcs := make([]*flow.QuorumCertificate, 0, n) for i := 0; i < int(n); i++ { qcs = append(qcs, QuorumCertificateFixture(opts...)) @@ -1755,7 +1915,10 @@ func WithVoteBlockID(blockID flow.Identifier) func(*hotstuff.Vote) { } } -func VoteForBlockFixture(block *hotstuff.Block, opts ...func(vote *hotstuff.Vote)) *hotstuff.Vote { +func VoteForBlockFixture( + block *hotstuff.Block, + opts ...func(vote *hotstuff.Vote), +) *hotstuff.Vote { vote := VoteFixture(WithVoteView(block.View), WithVoteBlockID(block.BlockID)) @@ -1901,11 +2064,52 @@ func EpochCommitFixture(opts ...func(*flow.EpochCommit)) *flow.EpochCommit { return commit } +func WithBoundaries(boundaries ...flow.VersionBoundary) func(*flow.VersionBeacon) { + return func(b *flow.VersionBeacon) { + b.VersionBoundaries = append(b.VersionBoundaries, boundaries...) + } +} + +func VersionBeaconFixture(options ...func(*flow.VersionBeacon)) *flow.VersionBeacon { + + versionTable := &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{}, + Sequence: uint64(0), + } + opts := options + + if len(opts) == 0 { + opts = []func(*flow.VersionBeacon){ + WithBoundaries(flow.VersionBoundary{ + Version: "0.0.0", + BlockHeight: 0, + }), + } + } + + for _, apply := range opts { + apply(versionTable) + } + + return versionTable +} + // BootstrapFixture generates all the artifacts necessary to bootstrap the // protocol state. -func BootstrapFixture(participants flow.IdentityList, opts ...func(*flow.Block)) (*flow.Block, *flow.ExecutionResult, *flow.Seal) { +func BootstrapFixture( + participants flow.IdentityList, + opts ...func(*flow.Block), +) (*flow.Block, *flow.ExecutionResult, *flow.Seal) { + return BootstrapFixtureWithChainID(participants, flow.Emulator, opts...) +} + +func BootstrapFixtureWithChainID( + participants flow.IdentityList, + chainID flow.ChainID, + opts ...func(*flow.Block), +) (*flow.Block, *flow.ExecutionResult, *flow.Seal) { - root := GenesisFixture() + root := flow.Genesis(chainID) for _, apply := range opts { apply(root) } @@ -1923,8 +2127,12 @@ func BootstrapFixture(participants flow.IdentityList, opts ...func(*flow.Block)) WithDKGFromParticipants(participants), ) - result := BootstrapExecutionResultFixture(root, GenesisStateCommitment) - result.ServiceEvents = []flow.ServiceEvent{setup.ServiceEvent(), commit.ServiceEvent()} + stateCommit := GenesisStateCommitmentByChainID(chainID) + result := BootstrapExecutionResultFixture(root, stateCommit) + result.ServiceEvents = []flow.ServiceEvent{ + setup.ServiceEvent(), + commit.ServiceEvent(), + } seal := Seal.Fixture(Seal.WithResult(result)) @@ -1933,8 +2141,19 @@ func BootstrapFixture(participants flow.IdentityList, opts ...func(*flow.Block)) // RootSnapshotFixture returns a snapshot representing a root chain state, for // example one as returned from BootstrapFixture. -func RootSnapshotFixture(participants flow.IdentityList, opts ...func(*flow.Block)) *inmem.Snapshot { - block, result, seal := BootstrapFixture(participants.Sort(order.Canonical), opts...) +func RootSnapshotFixture( + participants flow.IdentityList, + opts ...func(*flow.Block), +) *inmem.Snapshot { + return RootSnapshotFixtureWithChainID(participants, flow.Emulator, opts...) +} + +func RootSnapshotFixtureWithChainID( + participants flow.IdentityList, + chainID flow.ChainID, + opts ...func(*flow.Block), +) *inmem.Snapshot { + block, result, seal := BootstrapFixtureWithChainID(participants.Sort(order.Canonical), chainID, opts...) qc := QuorumCertificateFixture(QCWithRootBlockID(block.ID())) root, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) if err != nil { @@ -1943,7 +2162,10 @@ func RootSnapshotFixture(participants flow.IdentityList, opts ...func(*flow.Bloc return root } -func SnapshotClusterByIndex(snapshot *inmem.Snapshot, clusterIndex uint) (protocol.Cluster, error) { +func SnapshotClusterByIndex( + snapshot *inmem.Snapshot, + clusterIndex uint, +) (protocol.Cluster, error) { epochs := snapshot.Epochs() epoch := epochs.Current() cluster, err := epoch.Cluster(clusterIndex) @@ -1954,7 +2176,11 @@ func SnapshotClusterByIndex(snapshot *inmem.Snapshot, clusterIndex uint) (protoc } // ChainFixture creates a list of blocks that forms a chain -func ChainFixture(nonGenesisCount int) ([]*flow.Block, *flow.ExecutionResult, *flow.Seal) { +func ChainFixture(nonGenesisCount int) ( + []*flow.Block, + *flow.ExecutionResult, + *flow.Seal, +) { chain := make([]*flow.Block, 0, nonGenesisCount+1) participants := IdentityListFixture(5, WithAllRoles()) @@ -1980,7 +2206,10 @@ func ChainFixtureFrom(count int, parent *flow.Header) []*flow.Block { return blocks } -func ReceiptChainFor(blocks []*flow.Block, result0 *flow.ExecutionResult) []*flow.ExecutionReceipt { +func ReceiptChainFor( + blocks []*flow.Block, + result0 *flow.ExecutionResult, +) []*flow.ExecutionReceipt { receipts := make([]*flow.ExecutionReceipt, len(blocks)) receipts[0] = ExecutionReceiptFixture(WithResult(result0)) receipts[0].ExecutionResult.BlockID = blocks[0].ID() @@ -2058,7 +2287,11 @@ func PrivateKeyFixture(algo crypto.SigningAlgorithm, seedLength int) crypto.Priv // PrivateKeyFixtureByIdentifier returns a private key for a given node. // given the same identifier, it will always return the same private key -func PrivateKeyFixtureByIdentifier(algo crypto.SigningAlgorithm, seedLength int, id flow.Identifier) crypto.PrivateKey { +func PrivateKeyFixtureByIdentifier( + algo crypto.SigningAlgorithm, + seedLength int, + id flow.Identifier, +) crypto.PrivateKey { seed := append(id[:], id[:]...) sk, err := crypto.GeneratePrivateKey(algo, seed[:seedLength]) if err != nil { @@ -2091,7 +2324,10 @@ func NodeMachineAccountInfoFixture() bootstrap.NodeMachineAccountInfo { } } -func MachineAccountFixture(t *testing.T) (bootstrap.NodeMachineAccountInfo, *sdk.Account) { +func MachineAccountFixture(t *testing.T) ( + bootstrap.NodeMachineAccountInfo, + *sdk.Account, +) { info := NodeMachineAccountInfoFixture() bal, err := cadence.NewUFix64("0.5") @@ -2179,10 +2415,127 @@ func EngineMessageFixtures(count int) []*engine.Message { } // GetFlowProtocolEventID returns the event ID for the event provided. -func GetFlowProtocolEventID(t *testing.T, channel channels.Channel, event interface{}) flow.Identifier { +func GetFlowProtocolEventID( + t *testing.T, + channel channels.Channel, + event interface{}, +) flow.Identifier { payload, err := NetworkCodec().Encode(event) require.NoError(t, err) eventIDHash, err := network.EventId(channel, payload) require.NoError(t, err) return flow.HashToID(eventIDHash) } + +func WithBlockExecutionDataBlockID(blockID flow.Identifier) func(*execution_data.BlockExecutionData) { + return func(bed *execution_data.BlockExecutionData) { + bed.BlockID = blockID + } +} + +func WithChunkExecutionDatas(chunks ...*execution_data.ChunkExecutionData) func(*execution_data.BlockExecutionData) { + return func(bed *execution_data.BlockExecutionData) { + bed.ChunkExecutionDatas = chunks + } +} + +func BlockExecutionDataFixture(opts ...func(*execution_data.BlockExecutionData)) *execution_data.BlockExecutionData { + bed := &execution_data.BlockExecutionData{ + BlockID: IdentifierFixture(), + ChunkExecutionDatas: []*execution_data.ChunkExecutionData{}, + } + + for _, opt := range opts { + opt(bed) + } + + return bed +} + +func BlockExecutionDatEntityFixture(opts ...func(*execution_data.BlockExecutionData)) *execution_data.BlockExecutionDataEntity { + execData := BlockExecutionDataFixture(opts...) + return execution_data.NewBlockExecutionDataEntity(IdentifierFixture(), execData) +} + +func BlockExecutionDatEntityListFixture(n int) []*execution_data.BlockExecutionDataEntity { + l := make([]*execution_data.BlockExecutionDataEntity, n) + for i := 0; i < n; i++ { + l[i] = BlockExecutionDatEntityFixture() + } + + return l +} + +func WithChunkEvents(events flow.EventsList) func(*execution_data.ChunkExecutionData) { + return func(conf *execution_data.ChunkExecutionData) { + conf.Events = events + } +} + +func ChunkExecutionDataFixture(t *testing.T, minSize int, opts ...func(*execution_data.ChunkExecutionData)) *execution_data.ChunkExecutionData { + collection := CollectionFixture(5) + ced := &execution_data.ChunkExecutionData{ + Collection: &collection, + Events: flow.EventsList{}, + TrieUpdate: testutils.TrieUpdateFixture(1, 1, 8), + } + + for _, opt := range opts { + opt(ced) + } + + if minSize <= 1 || ced.TrieUpdate == nil { + return ced + } + + size := 1 + for { + buf := &bytes.Buffer{} + require.NoError(t, execution_data.DefaultSerializer.Serialize(buf, ced)) + if buf.Len() >= minSize { + return ced + } + + v := make([]byte, size) + _, err := crand.Read(v) + require.NoError(t, err) + + k, err := ced.TrieUpdate.Payloads[0].Key() + require.NoError(t, err) + + ced.TrieUpdate.Payloads[0] = ledger.NewPayload(k, v) + size *= 2 + } +} + +func CreateSendTxHttpPayload(tx flow.TransactionBody) map[string]interface{} { + tx.Arguments = [][]uint8{} // fix how fixture creates nil values + auth := make([]string, len(tx.Authorizers)) + for i, a := range tx.Authorizers { + auth[i] = a.String() + } + + return map[string]interface{}{ + "script": util.ToBase64(tx.Script), + "arguments": tx.Arguments, + "reference_block_id": tx.ReferenceBlockID.String(), + "gas_limit": fmt.Sprintf("%d", tx.GasLimit), + "payer": tx.Payer.String(), + "proposal_key": map[string]interface{}{ + "address": tx.ProposalKey.Address.String(), + "key_index": fmt.Sprintf("%d", tx.ProposalKey.KeyIndex), + "sequence_number": fmt.Sprintf("%d", tx.ProposalKey.SequenceNumber), + }, + "authorizers": auth, + "payload_signatures": []map[string]interface{}{{ + "address": tx.PayloadSignatures[0].Address.String(), + "key_index": fmt.Sprintf("%d", tx.PayloadSignatures[0].KeyIndex), + "signature": util.ToBase64(tx.PayloadSignatures[0].Signature), + }}, + "envelope_signatures": []map[string]interface{}{{ + "address": tx.EnvelopeSignatures[0].Address.String(), + "key_index": fmt.Sprintf("%d", tx.EnvelopeSignatures[0].KeyIndex), + "signature": util.ToBase64(tx.EnvelopeSignatures[0].Signature), + }}, + } +} diff --git a/utils/unittest/generator/events.go b/utils/unittest/generator/events.go index 117f3834007..014a9aaac9f 100644 --- a/utils/unittest/generator/events.go +++ b/utils/unittest/generator/events.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/onflow/cadence" - encoding "github.com/onflow/cadence/encoding/json" + "github.com/onflow/cadence/encoding/ccf" "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-go/model/flow" @@ -53,7 +53,7 @@ func (g *Events) New() flow.Event { fooString, }).WithType(testEventType) - payload, err := encoding.Encode(testEvent) + payload, err := ccf.Encode(testEvent) if err != nil { panic(fmt.Sprintf("unexpected error while encoding events: %s", err)) } diff --git a/utils/unittest/mocks/epoch_query.go b/utils/unittest/mocks/epoch_query.go index a624a655dd7..df71efb4073 100644 --- a/utils/unittest/mocks/epoch_query.go +++ b/utils/unittest/mocks/epoch_query.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/invalid" ) @@ -59,6 +60,17 @@ func (mock *EpochQuery) Previous() protocol.Epoch { return epoch } +// Phase returns a phase consistent with the current epoch state. +func (mock *EpochQuery) Phase() flow.EpochPhase { + mock.mu.RLock() + defer mock.mu.RUnlock() + _, exists := mock.byCounter[mock.counter+1] + if exists { + return flow.EpochPhaseSetup + } + return flow.EpochPhaseStaking +} + func (mock *EpochQuery) ByCounter(counter uint64) protocol.Epoch { mock.mu.RLock() defer mock.mu.RUnlock() diff --git a/utils/unittest/mocks/finalized_header_cache.go b/utils/unittest/mocks/finalized_header_cache.go new file mode 100644 index 00000000000..c5aa7a9bf28 --- /dev/null +++ b/utils/unittest/mocks/finalized_header_cache.go @@ -0,0 +1,28 @@ +package mocks + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" +) + +type finalizedHeaderCache struct { + t *testing.T + state protocol.State +} + +func NewFinalizedHeaderCache(t *testing.T, state protocol.State) *finalizedHeaderCache { + return &finalizedHeaderCache{ + t: t, + state: state, + } +} + +func (cache *finalizedHeaderCache) Get() *flow.Header { + head, err := cache.state.Final().Head() + require.NoError(cache.t, err) + return head +} diff --git a/utils/unittest/mocks/protocol_state.go b/utils/unittest/mocks/protocol_state.go index c2fa3421c13..91672b74419 100644 --- a/utils/unittest/mocks/protocol_state.go +++ b/utils/unittest/mocks/protocol_state.go @@ -66,10 +66,14 @@ func (p *Params) EpochFallbackTriggered() (bool, error) { return false, fmt.Errorf("not implemented") } -func (p *Params) Root() (*flow.Header, error) { +func (p *Params) FinalizedRoot() (*flow.Header, error) { return p.state.root.Header, nil } +func (p *Params) SealedRoot() (*flow.Header, error) { + return p.FinalizedRoot() +} + func (p *Params) Seal() (*flow.Seal, error) { return nil, fmt.Errorf("not implemented") } diff --git a/utils/unittest/network/conduit.go b/utils/unittest/network/conduit.go new file mode 100644 index 00000000000..5ce87ee1de6 --- /dev/null +++ b/utils/unittest/network/conduit.go @@ -0,0 +1,32 @@ +package network + +import ( + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/mocknetwork" +) + +type Conduit struct { + mocknetwork.Conduit + net *Network + channel channels.Channel +} + +var _ network.Conduit = (*Conduit)(nil) + +// Publish sends a message on this mock network, invoking any callback that has +// been specified. This will panic if no callback is found. +func (c *Conduit) Publish(event interface{}, targetIDs ...flow.Identifier) error { + if c.net.publishFunc != nil { + return c.net.publishFunc(c.channel, event, targetIDs...) + } + panic("Publish called but no callback function was found.") +} + +// ReportMisbehavior reports the misbehavior of a node on sending a message to the current node that appears valid +// based on the networking layer but is considered invalid by the current node based on the Flow protocol. +// This method is a no-op in the test helper implementation. +func (c *Conduit) ReportMisbehavior(_ network.MisbehaviorReport) { + // no-op +} diff --git a/utils/unittest/network/network.go b/utils/unittest/network/network.go index aa9541e57de..369e014f52a 100644 --- a/utils/unittest/network/network.go +++ b/utils/unittest/network/network.go @@ -12,32 +12,20 @@ import ( ) type EngineProcessFunc func(channels.Channel, flow.Identifier, interface{}) error -type NetworkPublishFunc func(channels.Channel, interface{}, ...flow.Identifier) error +type PublishFunc func(channels.Channel, interface{}, ...flow.Identifier) error // Conduit represents a mock conduit. -type Conduit struct { - mocknetwork.Conduit - net *Network - channel channels.Channel -} - -// Publish sends a message on this mock network, invoking any callback that has -// been specified. This will panic if no callback is found. -func (c *Conduit) Publish(event interface{}, targetIDs ...flow.Identifier) error { - if c.net.publishFunc != nil { - return c.net.publishFunc(c.channel, event, targetIDs...) - } - panic("Publish called but no callback function was found.") -} // Network represents a mock network. The implementation is not concurrency-safe. type Network struct { mocknetwork.Network conduits map[channels.Channel]*Conduit engines map[channels.Channel]network.MessageProcessor - publishFunc NetworkPublishFunc + publishFunc PublishFunc } +var _ network.Network = (*Network)(nil) + // NewNetwork returns a new mock network. func NewNetwork() *Network { return &Network{ @@ -73,7 +61,7 @@ func (n *Network) Send(channel channels.Channel, originID flow.Identifier, event // OnPublish specifies the callback that should be executed when `Publish` is called on any conduits // created by this mock network. -func (n *Network) OnPublish(publishFunc NetworkPublishFunc) *Network { +func (n *Network) OnPublish(publishFunc PublishFunc) *Network { n.publishFunc = publishFunc return n } diff --git a/utils/unittest/protected_map.go b/utils/unittest/protected_map.go index f0b4a65ad92..a2af2f5f513 100644 --- a/utils/unittest/protected_map.go +++ b/utils/unittest/protected_map.go @@ -57,3 +57,10 @@ func (p *ProtectedMap[K, V]) ForEach(fn func(k K, v V) error) error { } return nil } + +// Size returns the size of the map. +func (p *ProtectedMap[K, V]) Size() int { + p.mu.RLock() + defer p.mu.RUnlock() + return len(p.m) +} diff --git a/utils/unittest/service_events_fixtures.go b/utils/unittest/service_events_fixtures.go index 0f56bb4316c..74e5600c154 100644 --- a/utils/unittest/service_events_fixtures.go +++ b/utils/unittest/service_events_fixtures.go @@ -1,13 +1,16 @@ package unittest import ( + "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" + "github.com/onflow/cadence/runtime/common" + "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/model/flow" ) // This file contains service event fixtures for testing purposes. -// The Cadence form is represented by JSON-CDC-encoded string variables. // EpochSetupFixtureByChainID returns an EpochSetup service event as a Cadence event // representation and as a protocol model representation. @@ -18,7 +21,7 @@ func EpochSetupFixtureByChainID(chain flow.ChainID) (flow.Event, *flow.EpochSetu } event := EventFixture(events.EpochSetup.EventType(), 1, 1, IdentifierFixture(), 0) - event.Payload = []byte(EpochSetupFixtureJSON) + event.Payload = EpochSetupFixtureCCF // randomSource is [0,0,...,1,2,3,4] randomSource := make([]uint8, flow.EpochSetupRandomSourceLength) @@ -117,7 +120,7 @@ func EpochCommitFixtureByChainID(chain flow.ChainID) (flow.Event, *flow.EpochCom } event := EventFixture(events.EpochCommit.EventType(), 1, 1, IdentifierFixture(), 0) - event.Payload = []byte(EpochCommitFixtureJSON) + event.Payload = EpochCommitFixtureCCF expected := &flow.EpochCommit{ Counter: 1, @@ -146,1083 +149,917 @@ func EpochCommitFixtureByChainID(chain flow.ChainID) (flow.Event, *flow.EpochCom return event, expected } -var EpochSetupFixtureJSON = ` -{ - "type": "Event", - "value": { - "id": "A.01cf0e2f2f715450.FlowEpoch.EpochSetup", - "fields": [ - { - "name": "counter", - "value": { - "type": "UInt64", - "value": "1" - } - }, - { - "name": "nodeInfo", - "value": { - "type": "Array", - "value": [ - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowIDTableStaking.NodeInfo", - "fields": [ - { - "name": "id", - "value": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000001" - } - }, - { - "name": "role", - "value": { - "type": "UInt8", - "value": "1" - } - }, - { - "name": "networkingAddress", - "value": { - "type": "String", - "value": "1.flow.com" - } - }, - { - "name": "networkingKey", - "value": { - "type": "String", - "value": "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc" - } - }, - { - "name": "stakingKey", - "value": { - "type": "String", - "value": "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7" - } - }, - { - "name": "tokensStaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensCommitted", - "value": { - "type": "UFix64", - "value": "1350000.00000000" - } - }, - { - "name": "tokensUnstaking", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensUnstaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensRewarded", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "delegators", - "value": { - "type": "Array", - "value": [] - } - }, - { - "name": "delegatorIDCounter", - "value": { - "type": "UInt32", - "value": "0" - } - }, - { - "name": "tokensRequestedToUnstake", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "initialWeight", - "value": { - "type": "UInt64", - "value": "100" - } - } - ] - } - }, - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowIDTableStaking.NodeInfo", - "fields": [ - { - "name": "id", - "value": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000002" - } - }, - { - "name": "role", - "value": { - "type": "UInt8", - "value": "1" - } - }, - { - "name": "networkingAddress", - "value": { - "type": "String", - "value": "2.flow.com" - } - }, - { - "name": "networkingKey", - "value": { - "type": "String", - "value": "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc" - } - }, - { - "name": "stakingKey", - "value": { - "type": "String", - "value": "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7" - } - }, - { - "name": "tokensStaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensCommitted", - "value": { - "type": "UFix64", - "value": "1350000.00000000" - } - }, - { - "name": "tokensUnstaking", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensUnstaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensRewarded", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "delegators", - "value": { - "type": "Array", - "value": [] - } - }, - { - "name": "delegatorIDCounter", - "value": { - "type": "UInt32", - "value": "0" - } - }, - { - "name": "tokensRequestedToUnstake", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "initialWeight", - "value": { - "type": "UInt64", - "value": "100" - } - } - ] - } - }, - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowIDTableStaking.NodeInfo", - "fields": [ - { - "name": "id", - "value": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000003" - } - }, - { - "name": "role", - "value": { - "type": "UInt8", - "value": "1" - } - }, - { - "name": "networkingAddress", - "value": { - "type": "String", - "value": "3.flow.com" - } - }, - { - "name": "networkingKey", - "value": { - "type": "String", - "value": "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc" - } - }, - { - "name": "stakingKey", - "value": { - "type": "String", - "value": "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7" - } - }, - { - "name": "tokensStaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensCommitted", - "value": { - "type": "UFix64", - "value": "1350000.00000000" - } - }, - { - "name": "tokensUnstaking", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensUnstaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensRewarded", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "delegators", - "value": { - "type": "Array", - "value": [] - } - }, - { - "name": "delegatorIDCounter", - "value": { - "type": "UInt32", - "value": "0" - } - }, - { - "name": "tokensRequestedToUnstake", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "initialWeight", - "value": { - "type": "UInt64", - "value": "100" - } - } - ] - } - }, - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowIDTableStaking.NodeInfo", - "fields": [ - { - "name": "id", - "value": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000004" - } - }, - { - "name": "role", - "value": { - "type": "UInt8", - "value": "1" - } - }, - { - "name": "networkingAddress", - "value": { - "type": "String", - "value": "4.flow.com" - } - }, - { - "name": "networkingKey", - "value": { - "type": "String", - "value": "378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc" - } - }, - { - "name": "stakingKey", - "value": { - "type": "String", - "value": "af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7" - } - }, - { - "name": "tokensStaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensCommitted", - "value": { - "type": "UFix64", - "value": "1350000.00000000" - } - }, - { - "name": "tokensUnstaking", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensUnstaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensRewarded", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "delegators", - "value": { - "type": "Array", - "value": [] - } - }, - { - "name": "delegatorIDCounter", - "value": { - "type": "UInt32", - "value": "0" - } - }, - { - "name": "tokensRequestedToUnstake", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "initialWeight", - "value": { - "type": "UInt64", - "value": "100" - } - } - ] - } - }, - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowIDTableStaking.NodeInfo", - "fields": [ - { - "name": "id", - "value": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000011" - } - }, - { - "name": "role", - "value": { - "type": "UInt8", - "value": "2" - } - }, - { - "name": "networkingAddress", - "value": { - "type": "String", - "value": "11.flow.com" - } - }, - { - "name": "networkingKey", - "value": { - "type": "String", - "value": "cfdfe8e4362c8f79d11772cb7277ab16e5033a63e8dd5d34caf1b041b77e5b2d63c2072260949ccf8907486e4cfc733c8c42ca0e4e208f30470b0d950856cd47" - } - }, - { - "name": "stakingKey", - "value": { - "type": "String", - "value": "8207559cd7136af378bba53a8f0196dee3849a3ab02897c1995c3e3f6ca0c4a776c3ae869d1ddbb473090054be2400ad06d7910aa2c5d1780220fdf3765a3c1764bce10c6fe66a5a2be51a422e878518bd750424bb56b8a0ecf0f8ad2057e83f" - } - }, - { - "name": "tokensStaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensCommitted", - "value": { - "type": "UFix64", - "value": "1350000.00000000" - } - }, - { - "name": "tokensUnstaking", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensUnstaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensRewarded", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "delegators", - "value": { - "type": "Array", - "value": [] - } - }, - { - "name": "delegatorIDCounter", - "value": { - "type": "UInt32", - "value": "0" - } - }, - { - "name": "tokensRequestedToUnstake", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "initialWeight", - "value": { - "type": "UInt64", - "value": "100" - } - } - ] - } - }, - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowIDTableStaking.NodeInfo", - "fields": [ - { - "name": "id", - "value": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000021" - } - }, - { - "name": "role", - "value": { - "type": "UInt8", - "value": "3" - } - }, - { - "name": "networkingAddress", - "value": { - "type": "String", - "value": "21.flow.com" - } - }, - { - "name": "networkingKey", - "value": { - "type": "String", - "value": "d64318ba0dbf68f3788fc81c41d507c5822bf53154530673127c66f50fe4469ccf1a054a868a9f88506a8999f2386d86fcd2b901779718cba4fb53c2da258f9e" - } - }, - { - "name": "stakingKey", - "value": { - "type": "String", - "value": "880b162b7ec138b36af401d07868cb08d25746d905395edbb4625bdf105d4bb2b2f4b0f4ae273a296a6efefa7ce9ccb914e39947ce0e83745125cab05d62516076ff0173ed472d3791ccef937597c9ea12381d76f547a092a4981d77ff3fba83" - } - }, - { - "name": "tokensStaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensCommitted", - "value": { - "type": "UFix64", - "value": "1350000.00000000" - } - }, - { - "name": "tokensUnstaking", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensUnstaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensRewarded", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "delegators", - "value": { - "type": "Array", - "value": [] - } - }, - { - "name": "delegatorIDCounter", - "value": { - "type": "UInt32", - "value": "0" - } - }, - { - "name": "tokensRequestedToUnstake", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "initialWeight", - "value": { - "type": "UInt64", - "value": "100" - } - } - ] - } - }, - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowIDTableStaking.NodeInfo", - "fields": [ - { - "name": "id", - "value": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000031" - } - }, - { - "name": "role", - "value": { - "type": "UInt8", - "value": "4" - } - }, - { - "name": "networkingAddress", - "value": { - "type": "String", - "value": "31.flow.com" - } - }, - { - "name": "networkingKey", - "value": { - "type": "String", - "value": "697241208dcc9142b6f53064adc8ff1c95760c68beb2ba083c1d005d40181fd7a1b113274e0163c053a3addd47cd528ec6a1f190cf465aac87c415feaae011ae" - } - }, - { - "name": "stakingKey", - "value": { - "type": "String", - "value": "b1f97d0a06020eca97352e1adde72270ee713c7daf58da7e74bf72235321048b4841bdfc28227964bf18e371e266e32107d238358848bcc5d0977a0db4bda0b4c33d3874ff991e595e0f537c7b87b4ddce92038ebc7b295c9ea20a1492302aa7" - } - }, - { - "name": "tokensStaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensCommitted", - "value": { - "type": "UFix64", - "value": "1350000.00000000" - } - }, - { - "name": "tokensUnstaking", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensUnstaked", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "tokensRewarded", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "delegators", - "value": { - "type": "Array", - "value": [] - } - }, - { - "name": "delegatorIDCounter", - "value": { - "type": "UInt32", - "value": "0" - } - }, - { - "name": "tokensRequestedToUnstake", - "value": { - "type": "UFix64", - "value": "0.00000000" - } - }, - { - "name": "initialWeight", - "value": { - "type": "UInt64", - "value": "100" - } - } - ] - } - } - ] - } - }, - { - "name": "firstView", - "value": { - "type": "UInt64", - "value": "100" - } - }, - { - "name": "finalView", - "value": { - "type": "UInt64", - "value": "200" - } - }, - { - "name": "collectorClusters", - "value": { - "type": "Array", - "value": [ - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowClusterQC.Cluster", - "fields": [ - { - "name": "index", - "value": { - "type": "UInt16", - "value": "0" - } - }, - { - "name": "nodeWeights", - "value": { - "type": "Dictionary", - "value": [ - { - "key": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000001" - }, - "value": { - "type": "UInt64", - "value": "100" - } - }, - { - "key": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000002" - }, - "value": { - "type": "UInt64", - "value": "100" - } - } - ] - } - }, - { - "name": "totalWeight", - "value": { - "type": "UInt64", - "value": "100" - } - }, - { - "name": "votes", - "value": { - "type": "Array", - "value": [] - } - } - ] - } - }, - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowClusterQC.Cluster", - "fields": [ - { - "name": "index", - "value": { - "type": "UInt16", - "value": "1" - } - }, - { - "name": "nodeWeights", - "value": { - "type": "Dictionary", - "value": [ - { - "key": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000003" - }, - "value": { - "type": "UInt64", - "value": "100" - } - }, - { - "key": { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000004" - }, - "value": { - "type": "UInt64", - "value": "100" - } - } - ] - } - }, - { - "name": "totalWeight", - "value": { - "type": "UInt64", - "value": "0" - } - }, - { - "name": "votes", - "value": { - "type": "Array", - "value": [] - } - } - ] - } - } - ] - } - }, - { - "name": "randomSource", - "value": { - "type": "String", - "value": "01020304" - } - }, - { - "name": "DKGPhase1FinalView", - "value": { - "type": "UInt64", - "value": "150" - } - }, - { - "name": "DKGPhase2FinalView", - "value": { - "type": "UInt64", - "value": "160" - } - }, - { - "name": "DKGPhase3FinalView", - "value": { - "type": "UInt64", - "value": "170" - } - } - ] - } +// VersionBeaconFixtureByChainID returns a VersionTable service event as a Cadence event +// representation and as a protocol model representation. +func VersionBeaconFixtureByChainID(chain flow.ChainID) (flow.Event, *flow.VersionBeacon) { + + events, err := systemcontracts.ServiceEventsForChain(chain) + if err != nil { + panic(err) + } + + event := EventFixture(events.VersionBeacon.EventType(), 1, 1, IdentifierFixture(), 0) + event.Payload = VersionBeaconFixtureCCF + + expected := &flow.VersionBeacon{ + VersionBoundaries: []flow.VersionBoundary{ + { + BlockHeight: 44, + Version: "2.13.7-test", + }, + }, + Sequence: 5, + } + + return event, expected +} + +func createEpochSetupEvent() cadence.Event { + + return cadence.NewEvent([]cadence.Value{ + // counter + cadence.NewUInt64(1), + + // nodeInfo + createEpochNodes(), + + // firstView + cadence.NewUInt64(100), + + // finalView + cadence.NewUInt64(200), + + // collectorClusters + createEpochCollectors(), + + // randomSource + cadence.String("01020304"), + + // DKGPhase1FinalView + cadence.UInt64(150), + + // DKGPhase2FinalView + cadence.UInt64(160), + + // DKGPhase3FinalView + cadence.UInt64(170), + }).WithType(newFlowEpochEpochSetupEventType()) } -` - -var EpochCommitFixtureJSON = ` -{ - "type": "Event", - "value": { - "id": "A.01cf0e2f2f715450.FlowEpoch.EpochCommitted", - "fields": [ - { - "name": "counter", - "value": { - "type": "UInt64", - "value": "1" - } - }, - { - "name": "clusterQCs", - "value": { - "type": "Array", - "value": [ - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowClusterQC.ClusterQC", - "fields": [ - { - "name": "index", - "value": { - "type": "UInt16", - "value": "0" - } - }, - { - "name": "voteSignatures", - "value": { - "type": "Array", - "value": [ - { - "type": "String", - "value": "a39cd1e1bf7e2fb0609b7388ce5215a6a4c01eef2aee86e1a007faa28a6b2a3dc876e11bb97cdb26c3846231d2d01e4d" - }, - { - "type": "String", - "value": "91673ad9c717d396c9a0953617733c128049ac1a639653d4002ab245b121df1939430e313bcbfd06948f6a281f6bf853" - } - ] - } - }, - { - "name": "voteMessage", - "value": { - "type": "String", - "value": "irrelevant_for_these_purposes" - } - }, - { - "name": "voterIDs", - "value": { - "type": "Array", - "value": [ - { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000001" - }, - { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000002" - } - ] - } - } - ] - } - }, - { - "type": "Struct", - "value": { - "id": "A.01cf0e2f2f715450.FlowClusterQC.ClusterQC", - "fields": [ - { - "name": "index", - "value": { - "type": "UInt16", - "value": "1" - } - }, - { - "name": "voteSignatures", - "value": { - "type": "Array", - "value": [ - { - "type": "String", - "value": "b2bff159971852ed63e72c37991e62c94822e52d4fdcd7bf29aaf9fb178b1c5b4ce20dd9594e029f3574cb29533b857a" - }, - { - "type": "String", - "value": "9931562f0248c9195758da3de4fb92f24fa734cbc20c0cb80280163560e0e0348f843ac89ecbd3732e335940c1e8dccb" - } - ] - } - }, - { - "name": "voteMessage", - "value": { - "type": "String", - "value": "irrelevant_for_these_purposes" - } - }, - { - "name": "voterIDs", - "value": { - "type": "Array", - "value": [ - { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000003" - }, - { - "type": "String", - "value": "0000000000000000000000000000000000000000000000000000000000000004" - } - ] - } - } - ] - } - } - ] - } - }, - { - "name": "dkgPubKeys", - "value": { - "type": "Array", - "value": [ - { - "type": "String", - "value": "8c588266db5f5cda629e83f8aa04ae9413593fac19e4865d06d291c9d14fbdd9bdb86a7a12f9ef8590c79cb635e3163315d193087e9336092987150d0cd2b14ac6365f7dc93eec573752108b8c12368abb65f0652d9f644e5aed611c37926950" - }, - { - "type": "String", - "value": "87a339e4e5c74f089da20a33f515d8c8f4464ab53ede5a74aa2432cd1ae66d522da0c122249ee176cd747ddc83ca81090498389384201614caf51eac392c1c0a916dfdcfbbdf7363f9552b6468434add3d3f6dc91a92bbe3ee368b59b7828488" - } - ] - } - } - ] - } -}` + +func createEpochNodes() cadence.Array { + + nodeInfoType := newFlowIDTableStakingNodeInfoStructType() + + nodeInfo1 := cadence.NewStruct([]cadence.Value{ + // id + cadence.String("0000000000000000000000000000000000000000000000000000000000000001"), + + // role + cadence.UInt8(1), + + // networkingAddress + cadence.String("1.flow.com"), + + // networkingKey + cadence.String("378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + + // stakingKey + cadence.String("af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + + // tokensStaked + ufix64FromString("0.00000000"), + + // tokensCommitted + ufix64FromString("1350000.00000000"), + + // tokensUnstaking + ufix64FromString("0.00000000"), + + // tokensUnstaked + ufix64FromString("0.00000000"), + + // tokensRewarded + ufix64FromString("0.00000000"), + + // delegators + cadence.NewArray([]cadence.Value{}).WithType(cadence.NewVariableSizedArrayType(cadence.NewUInt32Type())), + + // delegatorIDCounter + cadence.UInt32(0), + + // tokensRequestedToUnstake + ufix64FromString("0.00000000"), + + // initialWeight + cadence.UInt64(100), + }).WithType(nodeInfoType) + + nodeInfo2 := cadence.NewStruct([]cadence.Value{ + // id + cadence.String("0000000000000000000000000000000000000000000000000000000000000002"), + + // role + cadence.UInt8(1), + + // networkingAddress + cadence.String("2.flow.com"), + + // networkingKey + cadence.String("378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + + // stakingKey + cadence.String("af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + + // tokensStaked + ufix64FromString("0.00000000"), + + // tokensCommitted + ufix64FromString("1350000.00000000"), + + // tokensUnstaking + ufix64FromString("0.00000000"), + + // tokensUnstaked + ufix64FromString("0.00000000"), + + // tokensRewarded + ufix64FromString("0.00000000"), + + // delegators + cadence.NewArray([]cadence.Value{}).WithType(cadence.NewVariableSizedArrayType(cadence.NewUInt32Type())), + + // delegatorIDCounter + cadence.UInt32(0), + + // tokensRequestedToUnstake + ufix64FromString("0.00000000"), + + // initialWeight + cadence.UInt64(100), + }).WithType(nodeInfoType) + + nodeInfo3 := cadence.NewStruct([]cadence.Value{ + // id + cadence.String("0000000000000000000000000000000000000000000000000000000000000003"), + + // role + cadence.UInt8(1), + + // networkingAddress + cadence.String("3.flow.com"), + + // networkingKey + cadence.String("378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + + // stakingKey + cadence.String("af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + + // tokensStaked + ufix64FromString("0.00000000"), + + // tokensCommitted + ufix64FromString("1350000.00000000"), + + // tokensUnstaking + ufix64FromString("0.00000000"), + + // tokensUnstaked + ufix64FromString("0.00000000"), + + // tokensRewarded + ufix64FromString("0.00000000"), + + // delegators + cadence.NewArray([]cadence.Value{}).WithType(cadence.NewVariableSizedArrayType(cadence.NewUInt32Type())), + + // delegatorIDCounter + cadence.UInt32(0), + + // tokensRequestedToUnstake + ufix64FromString("0.00000000"), + + // initialWeight + cadence.UInt64(100), + }).WithType(nodeInfoType) + + nodeInfo4 := cadence.NewStruct([]cadence.Value{ + // id + cadence.String("0000000000000000000000000000000000000000000000000000000000000004"), + + // role + cadence.UInt8(1), + + // networkingAddress + cadence.String("4.flow.com"), + + // networkingKey + cadence.String("378dbf45d85c614feb10d8bd4f78f4b6ef8eec7d987b937e123255444657fb3da031f232a507e323df3a6f6b8f50339c51d188e80c0e7a92420945cc6ca893fc"), + + // stakingKey + cadence.String("af4aade26d76bb2ab15dcc89adcef82a51f6f04b3cb5f4555214b40ec89813c7a5f95776ea4fe449de48166d0bbc59b919b7eabebaac9614cf6f9461fac257765415f4d8ef1376a2365ec9960121888ea5383d88a140c24c29962b0a14e4e4e7"), + + // tokensStaked + ufix64FromString("0.00000000"), + + // tokensCommitted + ufix64FromString("1350000.00000000"), + + // tokensUnstaking + ufix64FromString("0.00000000"), + + // tokensUnstaked + ufix64FromString("0.00000000"), + + // tokensRewarded + ufix64FromString("0.00000000"), + + // delegators + cadence.NewArray([]cadence.Value{}).WithType(cadence.NewVariableSizedArrayType(cadence.NewUInt32Type())), + + // delegatorIDCounter + cadence.UInt32(0), + + // tokensRequestedToUnstake + ufix64FromString("0.00000000"), + + // initialWeight + cadence.UInt64(100), + }).WithType(nodeInfoType) + + nodeInfo5 := cadence.NewStruct([]cadence.Value{ + // id + cadence.String("0000000000000000000000000000000000000000000000000000000000000011"), + + // role + cadence.UInt8(2), + + // networkingAddress + cadence.String("11.flow.com"), + + // networkingKey + cadence.String("cfdfe8e4362c8f79d11772cb7277ab16e5033a63e8dd5d34caf1b041b77e5b2d63c2072260949ccf8907486e4cfc733c8c42ca0e4e208f30470b0d950856cd47"), + + // stakingKey + cadence.String("8207559cd7136af378bba53a8f0196dee3849a3ab02897c1995c3e3f6ca0c4a776c3ae869d1ddbb473090054be2400ad06d7910aa2c5d1780220fdf3765a3c1764bce10c6fe66a5a2be51a422e878518bd750424bb56b8a0ecf0f8ad2057e83f"), + + // tokensStaked + ufix64FromString("0.00000000"), + + // tokensCommitted + ufix64FromString("1350000.00000000"), + + // tokensUnstaking + ufix64FromString("0.00000000"), + + // tokensUnstaked + ufix64FromString("0.00000000"), + + // tokensRewarded + ufix64FromString("0.00000000"), + + // delegators + cadence.NewArray([]cadence.Value{}).WithType(cadence.NewVariableSizedArrayType(cadence.NewUInt32Type())), + + // delegatorIDCounter + cadence.UInt32(0), + + // tokensRequestedToUnstake + ufix64FromString("0.00000000"), + + // initialWeight + cadence.UInt64(100), + }).WithType(nodeInfoType) + + nodeInfo6 := cadence.NewStruct([]cadence.Value{ + // id + cadence.String("0000000000000000000000000000000000000000000000000000000000000021"), + + // role + cadence.UInt8(3), + + // networkingAddress + cadence.String("21.flow.com"), + + // networkingKey + cadence.String("d64318ba0dbf68f3788fc81c41d507c5822bf53154530673127c66f50fe4469ccf1a054a868a9f88506a8999f2386d86fcd2b901779718cba4fb53c2da258f9e"), + + // stakingKey + cadence.String("880b162b7ec138b36af401d07868cb08d25746d905395edbb4625bdf105d4bb2b2f4b0f4ae273a296a6efefa7ce9ccb914e39947ce0e83745125cab05d62516076ff0173ed472d3791ccef937597c9ea12381d76f547a092a4981d77ff3fba83"), + + // tokensStaked + ufix64FromString("0.00000000"), + + // tokensCommitted + ufix64FromString("1350000.00000000"), + + // tokensUnstaking + ufix64FromString("0.00000000"), + + // tokensUnstaked + ufix64FromString("0.00000000"), + + // tokensRewarded + ufix64FromString("0.00000000"), + + // delegators + cadence.NewArray([]cadence.Value{}).WithType(cadence.NewVariableSizedArrayType(cadence.NewUInt32Type())), + + // delegatorIDCounter + cadence.UInt32(0), + + // tokensRequestedToUnstake + ufix64FromString("0.00000000"), + + // initialWeight + cadence.UInt64(100), + }).WithType(nodeInfoType) + + nodeInfo7 := cadence.NewStruct([]cadence.Value{ + // id + cadence.String("0000000000000000000000000000000000000000000000000000000000000031"), + + // role + cadence.UInt8(4), + + // networkingAddress + cadence.String("31.flow.com"), + + // networkingKey + cadence.String("697241208dcc9142b6f53064adc8ff1c95760c68beb2ba083c1d005d40181fd7a1b113274e0163c053a3addd47cd528ec6a1f190cf465aac87c415feaae011ae"), + + // stakingKey + cadence.String("b1f97d0a06020eca97352e1adde72270ee713c7daf58da7e74bf72235321048b4841bdfc28227964bf18e371e266e32107d238358848bcc5d0977a0db4bda0b4c33d3874ff991e595e0f537c7b87b4ddce92038ebc7b295c9ea20a1492302aa7"), + + // tokensStaked + ufix64FromString("0.00000000"), + + // tokensCommitted + ufix64FromString("1350000.00000000"), + + // tokensUnstaking + ufix64FromString("0.00000000"), + + // tokensUnstaked + ufix64FromString("0.00000000"), + + // tokensRewarded + ufix64FromString("0.00000000"), + + // delegators + cadence.NewArray([]cadence.Value{}).WithType(cadence.NewVariableSizedArrayType(cadence.NewUInt32Type())), + + // delegatorIDCounter + cadence.UInt32(0), + + // tokensRequestedToUnstake + ufix64FromString("0.00000000"), + + // initialWeight + cadence.UInt64(100), + }).WithType(nodeInfoType) + + return cadence.NewArray([]cadence.Value{ + nodeInfo1, + nodeInfo2, + nodeInfo3, + nodeInfo4, + nodeInfo5, + nodeInfo6, + nodeInfo7, + }).WithType(cadence.NewVariableSizedArrayType(nodeInfoType)) +} + +func createEpochCollectors() cadence.Array { + + clusterType := newFlowClusterQCClusterStructType() + + voteType := newFlowClusterQCVoteStructType() + + cluster1 := cadence.NewStruct([]cadence.Value{ + // index + cadence.NewUInt16(0), + + // nodeWeights + cadence.NewDictionary([]cadence.KeyValuePair{ + { + Key: cadence.String("0000000000000000000000000000000000000000000000000000000000000001"), + Value: cadence.UInt64(100), + }, + { + Key: cadence.String("0000000000000000000000000000000000000000000000000000000000000002"), + Value: cadence.UInt64(100), + }, + }).WithType(cadence.NewMeteredDictionaryType(nil, cadence.StringType{}, cadence.UInt64Type{})), + + // totalWeight + cadence.NewUInt64(100), + + // generatedVotes + cadence.NewDictionary(nil).WithType(cadence.NewDictionaryType(cadence.StringType{}, voteType)), + + // uniqueVoteMessageTotalWeights + cadence.NewDictionary(nil).WithType(cadence.NewDictionaryType(cadence.StringType{}, cadence.UInt64Type{})), + }).WithType(clusterType) + + cluster2 := cadence.NewStruct([]cadence.Value{ + // index + cadence.NewUInt16(1), + + // nodeWeights + cadence.NewDictionary([]cadence.KeyValuePair{ + { + Key: cadence.String("0000000000000000000000000000000000000000000000000000000000000003"), + Value: cadence.UInt64(100), + }, + { + Key: cadence.String("0000000000000000000000000000000000000000000000000000000000000004"), + Value: cadence.UInt64(100), + }, + }).WithType(cadence.NewMeteredDictionaryType(nil, cadence.StringType{}, cadence.UInt64Type{})), + + // totalWeight + cadence.NewUInt64(0), + + // generatedVotes + cadence.NewDictionary(nil).WithType(cadence.NewDictionaryType(cadence.StringType{}, voteType)), + + // uniqueVoteMessageTotalWeights + cadence.NewDictionary(nil).WithType(cadence.NewDictionaryType(cadence.StringType{}, cadence.UInt64Type{})), + }).WithType(clusterType) + + return cadence.NewArray([]cadence.Value{ + cluster1, + cluster2, + }).WithType(cadence.NewVariableSizedArrayType(clusterType)) +} + +func createEpochCommittedEvent() cadence.Event { + + clusterQCType := newFlowClusterQCClusterQCStructType() + + cluster1 := cadence.NewStruct([]cadence.Value{ + // index + cadence.UInt16(0), + + // voteSignatures + cadence.NewArray([]cadence.Value{ + cadence.String("a39cd1e1bf7e2fb0609b7388ce5215a6a4c01eef2aee86e1a007faa28a6b2a3dc876e11bb97cdb26c3846231d2d01e4d"), + cadence.String("91673ad9c717d396c9a0953617733c128049ac1a639653d4002ab245b121df1939430e313bcbfd06948f6a281f6bf853"), + }).WithType(cadence.NewVariableSizedArrayType(cadence.StringType{})), + + // voteMessage + cadence.String("irrelevant_for_these_purposes"), + + // voterIDs + cadence.NewArray([]cadence.Value{ + cadence.String("0000000000000000000000000000000000000000000000000000000000000001"), + cadence.String("0000000000000000000000000000000000000000000000000000000000000002"), + }).WithType(cadence.NewVariableSizedArrayType(cadence.StringType{})), + }).WithType(clusterQCType) + + cluster2 := cadence.NewStruct([]cadence.Value{ + // index + cadence.UInt16(1), + + // voteSignatures + cadence.NewArray([]cadence.Value{ + cadence.String("b2bff159971852ed63e72c37991e62c94822e52d4fdcd7bf29aaf9fb178b1c5b4ce20dd9594e029f3574cb29533b857a"), + cadence.String("9931562f0248c9195758da3de4fb92f24fa734cbc20c0cb80280163560e0e0348f843ac89ecbd3732e335940c1e8dccb"), + }).WithType(cadence.NewVariableSizedArrayType(cadence.StringType{})), + + // voteMessage + cadence.String("irrelevant_for_these_purposes"), + + // voterIDs + cadence.NewArray([]cadence.Value{ + cadence.String("0000000000000000000000000000000000000000000000000000000000000003"), + cadence.String("0000000000000000000000000000000000000000000000000000000000000004"), + }).WithType(cadence.NewVariableSizedArrayType(cadence.StringType{})), + }).WithType(clusterQCType) + + return cadence.NewEvent([]cadence.Value{ + // counter + cadence.NewUInt64(1), + + // clusterQCs + cadence.NewArray([]cadence.Value{ + cluster1, + cluster2, + }).WithType(cadence.NewVariableSizedArrayType(clusterQCType)), + + // dkgPubKeys + cadence.NewArray([]cadence.Value{ + cadence.String("8c588266db5f5cda629e83f8aa04ae9413593fac19e4865d06d291c9d14fbdd9bdb86a7a12f9ef8590c79cb635e3163315d193087e9336092987150d0cd2b14ac6365f7dc93eec573752108b8c12368abb65f0652d9f644e5aed611c37926950"), + cadence.String("87a339e4e5c74f089da20a33f515d8c8f4464ab53ede5a74aa2432cd1ae66d522da0c122249ee176cd747ddc83ca81090498389384201614caf51eac392c1c0a916dfdcfbbdf7363f9552b6468434add3d3f6dc91a92bbe3ee368b59b7828488"), + }).WithType(cadence.NewVariableSizedArrayType(cadence.StringType{})), + }).WithType(newFlowEpochEpochCommittedEventType()) +} + +func createVersionBeaconEvent() cadence.Event { + versionBoundaryType := NewNodeVersionBeaconVersionBoundaryStructType() + + semverType := NewNodeVersionBeaconSemverStructType() + + semver := cadence.NewStruct([]cadence.Value{ + // major + cadence.UInt8(2), + + // minor + cadence.UInt8(13), + + // patch + cadence.UInt8(7), + + // preRelease + cadence.NewOptional(cadence.String("test")), + }).WithType(semverType) + + versionBoundary := cadence.NewStruct([]cadence.Value{ + // blockHeight + cadence.UInt64(44), + + // version + semver, + }).WithType(versionBoundaryType) + + return cadence.NewEvent([]cadence.Value{ + // versionBoundaries + cadence.NewArray([]cadence.Value{ + versionBoundary, + }).WithType(cadence.NewVariableSizedArrayType(versionBoundaryType)), + + // sequence + cadence.UInt64(5), + }).WithType(NewNodeVersionBeaconVersionBeaconEventType()) +} + +func newFlowClusterQCVoteStructType() cadence.Type { + + // A.01cf0e2f2f715450.FlowClusterQC.Vote + + address, _ := common.HexToAddress("01cf0e2f2f715450") + location := common.NewAddressLocation(nil, address, "FlowClusterQC") + + return &cadence.StructType{ + Location: location, + QualifiedIdentifier: "FlowClusterQC.Vote", + Fields: []cadence.Field{ + { + Identifier: "nodeID", + Type: cadence.StringType{}, + }, + { + Identifier: "signature", + Type: cadence.NewOptionalType(cadence.StringType{}), + }, + { + Identifier: "message", + Type: cadence.NewOptionalType(cadence.StringType{}), + }, + { + Identifier: "clusterIndex", + Type: cadence.UInt16Type{}, + }, + { + Identifier: "weight", + Type: cadence.UInt64Type{}, + }, + }, + } +} + +func newFlowClusterQCClusterStructType() *cadence.StructType { + + // A.01cf0e2f2f715450.FlowClusterQC.Cluster + + address, _ := common.HexToAddress("01cf0e2f2f715450") + location := common.NewAddressLocation(nil, address, "FlowClusterQC") + + return &cadence.StructType{ + Location: location, + QualifiedIdentifier: "FlowClusterQC.Cluster", + Fields: []cadence.Field{ + { + Identifier: "index", + Type: cadence.UInt16Type{}, + }, + { + Identifier: "nodeWeights", + Type: cadence.NewDictionaryType(cadence.StringType{}, cadence.UInt64Type{}), + }, + { + Identifier: "totalWeight", + Type: cadence.UInt64Type{}, + }, + { + Identifier: "generatedVotes", + Type: cadence.NewDictionaryType(cadence.StringType{}, newFlowClusterQCVoteStructType()), + }, + { + Identifier: "uniqueVoteMessageTotalWeights", + Type: cadence.NewDictionaryType(cadence.StringType{}, cadence.UInt64Type{}), + }, + }, + } +} + +func newFlowIDTableStakingNodeInfoStructType() *cadence.StructType { + + // A.01cf0e2f2f715450.FlowIDTableStaking.NodeInfo + + address, _ := common.HexToAddress("01cf0e2f2f715450") + location := common.NewAddressLocation(nil, address, "FlowIDTableStaking") + + return &cadence.StructType{ + Location: location, + QualifiedIdentifier: "FlowIDTableStaking.NodeInfo", + Fields: []cadence.Field{ + { + Identifier: "id", + Type: cadence.StringType{}, + }, + { + Identifier: "role", + Type: cadence.UInt8Type{}, + }, + { + Identifier: "networkingAddress", + Type: cadence.StringType{}, + }, + { + Identifier: "networkingKey", + Type: cadence.StringType{}, + }, + { + Identifier: "stakingKey", + Type: cadence.StringType{}, + }, + { + Identifier: "tokensStaked", + Type: cadence.UFix64Type{}, + }, + { + Identifier: "tokensCommitted", + Type: cadence.UFix64Type{}, + }, + { + Identifier: "tokensUnstaking", + Type: cadence.UFix64Type{}, + }, + { + Identifier: "tokensUnstaked", + Type: cadence.UFix64Type{}, + }, + { + Identifier: "tokensRewarded", + Type: cadence.UFix64Type{}, + }, + { + Identifier: "delegators", + Type: cadence.NewVariableSizedArrayType(cadence.NewUInt32Type()), + }, + { + Identifier: "delegatorIDCounter", + Type: cadence.UInt32Type{}, + }, + { + Identifier: "tokensRequestedToUnstake", + Type: cadence.UFix64Type{}, + }, + { + Identifier: "initialWeight", + Type: cadence.UInt64Type{}, + }, + }, + } +} + +func newFlowEpochEpochSetupEventType() *cadence.EventType { + + // A.01cf0e2f2f715450.FlowEpoch.EpochSetup + + address, _ := common.HexToAddress("01cf0e2f2f715450") + location := common.NewAddressLocation(nil, address, "FlowEpoch") + + return &cadence.EventType{ + Location: location, + QualifiedIdentifier: "FlowEpoch.EpochSetup", + Fields: []cadence.Field{ + { + Identifier: "counter", + Type: cadence.UInt64Type{}, + }, + { + Identifier: "nodeInfo", + Type: cadence.NewVariableSizedArrayType(newFlowIDTableStakingNodeInfoStructType()), + }, + { + Identifier: "firstView", + Type: cadence.UInt64Type{}, + }, + { + Identifier: "finalView", + Type: cadence.UInt64Type{}, + }, + { + Identifier: "collectorClusters", + Type: cadence.NewVariableSizedArrayType(newFlowClusterQCClusterStructType()), + }, + { + Identifier: "randomSource", + Type: cadence.StringType{}, + }, + { + Identifier: "DKGPhase1FinalView", + Type: cadence.UInt64Type{}, + }, + { + Identifier: "DKGPhase2FinalView", + Type: cadence.UInt64Type{}, + }, + { + Identifier: "DKGPhase3FinalView", + Type: cadence.UInt64Type{}, + }, + }, + } +} + +func newFlowEpochEpochCommittedEventType() *cadence.EventType { + + // A.01cf0e2f2f715450.FlowEpoch.EpochCommitted + + address, _ := common.HexToAddress("01cf0e2f2f715450") + location := common.NewAddressLocation(nil, address, "FlowEpoch") + + return &cadence.EventType{ + Location: location, + QualifiedIdentifier: "FlowEpoch.EpochCommitted", + Fields: []cadence.Field{ + { + Identifier: "counter", + Type: cadence.UInt64Type{}, + }, + { + Identifier: "clusterQCs", + Type: cadence.NewVariableSizedArrayType(newFlowClusterQCClusterQCStructType()), + }, + { + Identifier: "dkgPubKeys", + Type: cadence.NewVariableSizedArrayType(cadence.StringType{}), + }, + }, + } +} + +func newFlowClusterQCClusterQCStructType() *cadence.StructType { + + // A.01cf0e2f2f715450.FlowClusterQC.ClusterQC" + + address, _ := common.HexToAddress("01cf0e2f2f715450") + location := common.NewAddressLocation(nil, address, "FlowClusterQC") + + return &cadence.StructType{ + Location: location, + QualifiedIdentifier: "FlowClusterQC.ClusterQC", + Fields: []cadence.Field{ + { + Identifier: "index", + Type: cadence.UInt16Type{}, + }, + { + Identifier: "voteSignatures", + Type: cadence.NewVariableSizedArrayType(cadence.StringType{}), + }, + { + Identifier: "voteMessage", + Type: cadence.StringType{}, + }, + { + Identifier: "voterIDs", + Type: cadence.NewVariableSizedArrayType(cadence.StringType{}), + }, + }, + } +} + +func NewNodeVersionBeaconVersionBeaconEventType() *cadence.EventType { + + // A.01cf0e2f2f715450.NodeVersionBeacon.VersionBeacon + + address, _ := common.HexToAddress("01cf0e2f2f715450") + location := common.NewAddressLocation(nil, address, "NodeVersionBeacon") + + return &cadence.EventType{ + Location: location, + QualifiedIdentifier: "NodeVersionBeacon.VersionBeacon", + Fields: []cadence.Field{ + { + Identifier: "versionBoundaries", + Type: cadence.NewVariableSizedArrayType(NewNodeVersionBeaconVersionBoundaryStructType()), + }, + { + Identifier: "sequence", + Type: cadence.UInt64Type{}, + }, + }, + } +} + +func NewNodeVersionBeaconVersionBoundaryStructType() *cadence.StructType { + + // A.01cf0e2f2f715450.NodeVersionBeacon.VersionBoundary + + address, _ := common.HexToAddress("01cf0e2f2f715450") + location := common.NewAddressLocation(nil, address, "NodeVersionBeacon") + + return &cadence.StructType{ + Location: location, + QualifiedIdentifier: "NodeVersionBeacon.VersionBoundary", + Fields: []cadence.Field{ + { + Identifier: "blockHeight", + Type: cadence.UInt64Type{}, + }, + { + Identifier: "version", + Type: NewNodeVersionBeaconSemverStructType(), + }, + }, + } +} + +func NewNodeVersionBeaconSemverStructType() *cadence.StructType { + + // A.01cf0e2f2f715450.NodeVersionBeacon.Semver + + address, _ := common.HexToAddress("01cf0e2f2f715450") + location := common.NewAddressLocation(nil, address, "NodeVersionBeacon") + + return &cadence.StructType{ + Location: location, + QualifiedIdentifier: "NodeVersionBeacon.Semver", + Fields: []cadence.Field{ + { + Identifier: "major", + Type: cadence.UInt8Type{}, + }, + { + Identifier: "minor", + Type: cadence.UInt8Type{}, + }, + { + Identifier: "patch", + Type: cadence.UInt8Type{}, + }, + { + Identifier: "preRelease", + Type: cadence.NewOptionalType(cadence.StringType{}), + }, + }, + } +} + +func ufix64FromString(s string) cadence.UFix64 { + f, err := cadence.NewUFix64(s) + if err != nil { + panic(err) + } + return f +} + +var EpochSetupFixtureCCF = func() []byte { + b, err := ccf.Encode(createEpochSetupEvent()) + if err != nil { + panic(err) + } + _, err = ccf.Decode(nil, b) + if err != nil { + panic(err) + } + return b +}() + +var EpochCommitFixtureCCF = func() []byte { + b, err := ccf.Encode(createEpochCommittedEvent()) + if err != nil { + panic(err) + } + _, err = ccf.Decode(nil, b) + if err != nil { + panic(err) + } + return b +}() + +var VersionBeaconFixtureCCF = func() []byte { + b, err := ccf.Encode(createVersionBeaconEvent()) + if err != nil { + panic(err) + } + _, err = ccf.Decode(nil, b) + if err != nil { + panic(err) + } + return b +}() diff --git a/utils/unittest/strings.go b/utils/unittest/strings.go new file mode 100644 index 00000000000..9c7b39839af --- /dev/null +++ b/utils/unittest/strings.go @@ -0,0 +1,36 @@ +package unittest + +import ( + "crypto/rand" + "encoding/base64" + "testing" + + "github.com/stretchr/testify/require" +) + +// RandomStringFixture is a test helper that generates a cryptographically secure random string of size n. +func RandomStringFixture(t *testing.T, n int) string { + require.Greater(t, n, 0, "size should be positive") + + // The base64 encoding uses 64 different characters to represent data in + // strings, which makes it possible to represent 6 bits of data with each + // character (as 2^6 is 64). This means that every 3 bytes (24 bits) of + // input data will be represented by 4 characters (4 * 6 bits) in the + // base64 encoding. Consequently, base64 encoding increases the size of + // the data by approximately 1/3 compared to the original input data. + // + // 1. (n+3) / 4 - This calculates how many groups of 4 characters are needed + // in the base64 encoded output to represent at least 'n' characters. + // The +3 ensures rounding up, as integer division truncates the result. + // + // 2. ... * 3 - Each group of 4 base64 characters represents 3 bytes + // of input data. This multiplication calculates the number of bytes + // needed to produce the required length of the base64 string. + byteSlice := make([]byte, (n+3)/4*3) + n, err := rand.Read(byteSlice) + require.NoError(t, err) + require.Equal(t, n, len(byteSlice)) + + encodedString := base64.URLEncoding.EncodeToString(byteSlice) + return encodedString[:n] +} diff --git a/utils/unittest/unittest.go b/utils/unittest/unittest.go index 6c693d678a9..0d9949ffc2d 100644 --- a/utils/unittest/unittest.go +++ b/utils/unittest/unittest.go @@ -21,6 +21,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/util" "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" cborcodec "github.com/onflow/flow-go/network/codec/cbor" "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/topology" @@ -122,17 +123,6 @@ func SkipBenchmarkUnless(b *testing.B, reason SkipBenchmarkReason, message strin } } -func ExpectPanic(expectedMsg string, t *testing.T) { - if r := recover(); r != nil { - err := r.(error) - if err.Error() != expectedMsg { - t.Errorf("expected %v to be %v", err, expectedMsg) - } - return - } - t.Errorf("Expected to panic with `%s`, but did not panic", expectedMsg) -} - // AssertReturnsBefore asserts that the given function returns before the // duration expires. func AssertReturnsBefore(t *testing.T, f func(), duration time.Duration, msgAndArgs ...interface{}) bool { @@ -449,6 +439,18 @@ func GenerateRandomStringWithLen(commentLen uint) string { } // NetworkSlashingViolationsConsumer returns a slashing violations consumer for network middleware -func NetworkSlashingViolationsConsumer(logger zerolog.Logger, metrics module.NetworkSecurityMetrics) slashing.ViolationsConsumer { - return slashing.NewSlashingViolationsConsumer(logger, metrics) +func NetworkSlashingViolationsConsumer(logger zerolog.Logger, metrics module.NetworkSecurityMetrics, consumer network.MisbehaviorReportConsumer) network.ViolationsConsumer { + return slashing.NewSlashingViolationsConsumer(logger, metrics, consumer) +} + +type MisbehaviorReportConsumerFixture struct { + network.MisbehaviorReportManager +} + +func (c *MisbehaviorReportConsumerFixture) ReportMisbehaviorOnChannel(channel channels.Channel, report network.MisbehaviorReport) { + c.HandleMisbehaviorReport(channel, report) +} + +func NewMisbehaviorReportConsumerFixture(manager network.MisbehaviorReportManager) *MisbehaviorReportConsumerFixture { + return &MisbehaviorReportConsumerFixture{manager} } diff --git a/network/internal/testutils/updatable_provider.go b/utils/unittest/updatable_provider.go similarity index 98% rename from network/internal/testutils/updatable_provider.go rename to utils/unittest/updatable_provider.go index 014ae696b99..9661f7039a6 100644 --- a/network/internal/testutils/updatable_provider.go +++ b/utils/unittest/updatable_provider.go @@ -1,4 +1,4 @@ -package testutils +package unittest import ( "sync" diff --git a/utils/unittest/version_beacon.go b/utils/unittest/version_beacon.go new file mode 100644 index 00000000000..6518de747ef --- /dev/null +++ b/utils/unittest/version_beacon.go @@ -0,0 +1,60 @@ +package unittest + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" +) + +// AddVersionBeacon adds blocks sequence with given VersionBeacon so this +// service events takes effect in Flow protocol. +// This means execution result where event was emitted is sealed, and the seal is +// finalized by a valid block. +// This assumes state is bootstrapped with a root block, as it does NOT produce +// results for final block of the state +// Root <- A <- B(result(A(VB))) <- C(seal(B)) +func AddVersionBeacon(t *testing.T, beacon *flow.VersionBeacon, state protocol.FollowerState) { + + final, err := state.Final().Head() + + require.NoError(t, err) + + A := BlockWithParentFixture(final) + A.SetPayload(flow.Payload{}) + addToState(t, state, A, true) + + receiptA := ReceiptForBlockFixture(A) + receiptA.ExecutionResult.ServiceEvents = []flow.ServiceEvent{beacon.ServiceEvent()} + + B := BlockWithParentFixture(A.Header) + B.SetPayload(flow.Payload{ + Receipts: []*flow.ExecutionReceiptMeta{receiptA.Meta()}, + Results: []*flow.ExecutionResult{&receiptA.ExecutionResult}, + }) + addToState(t, state, B, true) + + sealsForB := []*flow.Seal{ + Seal.Fixture(Seal.WithResult(&receiptA.ExecutionResult)), + } + + C := BlockWithParentFixture(B.Header) + C.SetPayload(flow.Payload{ + Seals: sealsForB, + }) + addToState(t, state, C, true) +} + +func addToState(t *testing.T, state protocol.FollowerState, block *flow.Block, finalize bool) { + + err := state.ExtendCertified(context.Background(), block, CertifyBlock(block.Header)) + require.NoError(t, err) + + if finalize { + err = state.Finalize(context.Background(), block.ID()) + require.NoError(t, err) + } +}